diff --git a/.github/workflows/llm-benchmark-update.yml b/.github/workflows/llm-benchmark-update.yml
index b2789a71ae7..8d16cd662ed 100644
--- a/.github/workflows/llm-benchmark-update.yml
+++ b/.github/workflows/llm-benchmark-update.yml
@@ -302,4 +302,11 @@ jobs:
GH_TOKEN: ${{ secrets.CLOCKWORK_LABS_BOT_PAT }}
run: |
git remote set-url origin "https://x-access-token:${GH_TOKEN}@github.com/${{ steps.pr.outputs.head_repo_full_name }}.git"
+ # Fetch and rebase in case branch moved since workflow started (e.g., previous benchmark run)
+ git fetch origin "${{ steps.pr.outputs.head_ref }}"
+ if ! git rebase "origin/${{ steps.pr.outputs.head_ref }}"; then
+ git rebase --abort
+ echo "::error::Rebase failed due to conflicts. The PR branch may have been updated during the benchmark run. Please re-run /update-llm-benchmark."
+ exit 1
+ fi
git push origin "HEAD:${{ steps.pr.outputs.head_ref }}"
diff --git a/docs/docs/00100-intro/00100-getting-started/00250-zen-of-spacetimedb.md b/docs/docs/00100-intro/00100-getting-started/00250-zen-of-spacetimedb.md
index 3f3685fc070..bcf7da63793 100644
--- a/docs/docs/00100-intro/00100-getting-started/00250-zen-of-spacetimedb.md
+++ b/docs/docs/00100-intro/00100-getting-started/00250-zen-of-spacetimedb.md
@@ -26,7 +26,7 @@ Traditional stack: SpacetimeDB:
## Everything is Persistent
-You will ask, does everything need to be persistent? Won't that be a lot of data? Won't that be slow. The answers are yes, no, and no. SpacetimeDB persists everything by default, even the full history of everything that has every changed. It should be **your choice** to delete data, not the databases. SpacetimeDB gives you that choice.
+You will ask, does everything need to be persistent? Won't that be a lot of data? Won't that be slow. The answers are yes, no, and no. SpacetimeDB persists everything by default, even the full history of everything that has every changed. It should be **your choice** to delete data, not the database's. SpacetimeDB gives you that choice.
SpacetimeDB holds all your data in memory for blazing-fast access, but automatically persists everything to disk. You get the speed of in-memory computing with the durability of a traditional database.
diff --git a/docs/docs/00200-core-concepts/00300-tables/00500-schedule-tables.md b/docs/docs/00200-core-concepts/00300-tables/00500-schedule-tables.md
index 0c3c816e34d..d6307880e77 100644
--- a/docs/docs/00200-core-concepts/00300-tables/00500-schedule-tables.md
+++ b/docs/docs/00200-core-concepts/00300-tables/00500-schedule-tables.md
@@ -80,9 +80,156 @@ fn send_reminder(ctx: &ReducerContext, reminder: Reminder) -> Result<(), String>
+## Inserting Schedules
+
+To schedule an action, insert a row into the schedule table with a `scheduled_at` value. You can schedule actions to run:
+
+- **At intervals** - Execute repeatedly at fixed time intervals (e.g., every 5 seconds)
+- **At specific times** - Execute once at an absolute timestamp
+
+### Scheduling at Intervals
+
+Use intervals for periodic tasks like game ticks, heartbeats, or recurring maintenance:
+
+
+
+
+```typescript
+import { ScheduleAt } from 'spacetimedb';
+
+// Schedule to run every 5 seconds (5,000,000 microseconds)
+ctx.db.reminder.insert({
+ scheduled_id: 0n,
+ scheduled_at: ScheduleAt.interval(5_000_000n),
+ message: "Check for updates",
+});
+
+// Schedule to run every 100 milliseconds
+ctx.db.reminder.insert({
+ scheduled_id: 0n,
+ scheduled_at: ScheduleAt.interval(100_000n), // 100ms in microseconds
+ message: "Game tick",
+});
+```
+
+
+
+
+```csharp
+// Schedule to run every 5 seconds
+ctx.Db.Reminder.Insert(new Reminder
+{
+ Message = "Check for updates",
+ ScheduleAt = new ScheduleAt.Interval(TimeSpan.FromSeconds(5))
+});
+
+// Schedule to run every 100 milliseconds
+ctx.Db.Reminder.Insert(new Reminder
+{
+ Message = "Game tick",
+ ScheduleAt = new ScheduleAt.Interval(TimeSpan.FromMilliseconds(100))
+});
+```
+
+
+
+
+```rust
+use spacetimedb::{ScheduleAt, Duration};
+
+// Schedule to run every 5 seconds
+ctx.db.reminder().insert(Reminder {
+ id: 0,
+ message: "Check for updates".to_string(),
+ scheduled_at: ScheduleAt::Interval(Duration::from_secs(5).into()),
+});
+
+// Schedule to run every 100 milliseconds
+ctx.db.reminder().insert(Reminder {
+ id: 0,
+ message: "Game tick".to_string(),
+ scheduled_at: ScheduleAt::Interval(Duration::from_millis(100).into()),
+});
+```
+
+
+
+
+### Scheduling at Specific Times
+
+Use specific times for one-shot actions like sending a reminder at a particular moment or expiring content:
+
+
+
+
+```typescript
+import { ScheduleAt } from 'spacetimedb';
+
+// Schedule for 10 seconds from now
+const tenSecondsFromNow = ctx.timestamp.microseconds + 10_000_000n;
+ctx.db.reminder.insert({
+ scheduled_id: 0n,
+ scheduled_at: ScheduleAt.time(tenSecondsFromNow),
+ message: "Your auction has ended",
+});
+
+// Schedule for a specific Unix timestamp (microseconds since epoch)
+const targetTime = 1735689600_000_000n; // Jan 1, 2025 00:00:00 UTC
+ctx.db.reminder.insert({
+ scheduled_id: 0n,
+ scheduled_at: ScheduleAt.time(targetTime),
+ message: "Happy New Year!",
+});
+```
+
+
+
+
+```csharp
+// Schedule for 10 seconds from now
+ctx.Db.Reminder.Insert(new Reminder
+{
+ Message = "Your auction has ended",
+ ScheduleAt = new ScheduleAt.Time(DateTimeOffset.UtcNow.AddSeconds(10))
+});
+
+// Schedule for a specific time
+var targetTime = new DateTimeOffset(2025, 1, 1, 0, 0, 0, TimeSpan.Zero);
+ctx.Db.Reminder.Insert(new Reminder
+{
+ Message = "Happy New Year!",
+ ScheduleAt = new ScheduleAt.Time(targetTime)
+});
+```
+
+
+
+
+```rust
+use spacetimedb::{ScheduleAt, Duration};
+
+// Schedule for 10 seconds from now
+let ten_seconds_from_now = ctx.timestamp + Duration::from_secs(10);
+ctx.db.reminder().insert(Reminder {
+ id: 0,
+ message: "Your auction has ended".to_string(),
+ scheduled_at: ScheduleAt::Time(ten_seconds_from_now),
+});
+
+// Schedule for immediate execution (current timestamp)
+ctx.db.reminder().insert(Reminder {
+ id: 0,
+ message: "Process now".to_string(),
+ scheduled_at: ScheduleAt::Time(ctx.timestamp.clone()),
+});
+```
+
+
+
+
## How It Works
-1. **Insert a row** with a `schedule_at` time
+1. **Insert a row** with a `ScheduleAt` value
2. **SpacetimeDB monitors** the schedule table
3. **When the time arrives**, the specified reducer/procedure is automatically called with the row as a parameter
4. **The row is typically deleted** or updated by the reducer after processing
diff --git a/docs/llms/docs-benchmark-analysis.md b/docs/llms/docs-benchmark-analysis.md
index ee0e02bf837..f5275f0b197 100644
--- a/docs/llms/docs-benchmark-analysis.md
+++ b/docs/llms/docs-benchmark-analysis.md
@@ -1,80 +1,64 @@
# Benchmark Failure Analysis
-Generated from: `C:\Users\Tyler\Developer\SpacetimeDB\tools\xtask-llm-benchmark\../../docs/llms/docs-benchmark-details.json`
+Generated from: `/__w/SpacetimeDB/SpacetimeDB/tools/xtask-llm-benchmark/../../docs/llms/docs-benchmark-details.json`
## Summary
-- **Total failures analyzed**: 26
+- **Total failures analyzed**: 59
## Analysis
-# SpacetimeDB Benchmark Test Failures Analysis
+# Analysis of SpacetimeDB Benchmark Test Failures: Rust and C#
## Rust Failures
### 1. Root Causes
-- **Compile/Publish Errors (3 failures)**:
- - The primary issue across the failures is related to the use of `ScheduleAt::every_micros` versus `ScheduleAt::RepeatMicros`, which indicates a lack of clarity in the documentation about the correct method of using scheduled types.
- - Another issue is the incorrect implementation of `pub` for some fields and missing `#[derive(SpacetimeType)]` for structs, which has led to schema mismatches.
-
-- **Other Failures (1 failure)**:
- - The test `t_003_struct_in_table` has a mismatch where the expected reducer setup differs from what's provided. This highlights insufficient documentation around initial setup requirements for reducers.
+- **Inconsistent Table Names**: Many failures arise from incorrect or inconsistent table names used in the code segments compared to what is expected in the benchmarks. Tables such as `users`, `drawings`, `event`, etc., are referenced with incorrect names leading to errors.
+- **Lack of `pub` Keyword**: Many struct fields are missing the `pub` keyword, causing issues with accessing these fields outside the module.
+- **Unstable API Warnings**: Numerous tests are failing due to reliance on unstable methods or API changes.
+- **Missing Error Handling**: Functions that should return `Result` types do not, leading to issues when error handling is assumed.
### 2. Recommendations
-- **Documentation Updates**:
- - **Scheduled Types Documentation**: Enhance the section on scheduled types in the documentation to clarify the use of `ScheduleAt::every_micros` and `ScheduleAt::RepeatMicros`. Example for addition:
- ```markdown
- ### Scheduled Types
- - Use `ScheduleAt::every_micros(interval)` for non-repeating intervals.
- - Use `ScheduleAt::RepeatMicros(interval)` for repeating intervals. Ensure proper usage to avoid publishing errors.
- ```
-
- - **Section on Structs and Reducers**: Update the section dealing with struct fields to illustrate the necessity of using `pub` where it applies and clarifying how reducers must align:
- ```markdown
- ### Struct Definitions
- - Struct fields must be marked as `pub` to ensure they are accessible within the SpacetimeDB context.
- - Reducers must be defined properly; ensure that each reducer matches expected configurations in your schemas.
- ```
-
-- **Example Code Alignment**: Revise example code throughout documentation to align with the latest syntax and ensure that all required attributes are included.
+- **Update Table Names for Consistency**:
+ - In the table definitions and usages, ensure names are consistent throughout the documentation. For instance:
+ - `event` should replace all instances of `events`.
+ - `primitive` should replace all instances of `primitives`.
+ - `drawing` should replace all instances of `drawings`.
+- **Add `pub` Keyword for Structs and Fields**:
+ - Documentation should specify that structs and their fields must be public for access.
+- **Document API Stability**:
+ - Clearly mark all APIs that are unstable and subject to change. Provide a dedicated section for upcoming breaking changes, if possible.
+- **Error Handling**:
+ - Example code should consistently include error handling to return results or handle errors gracefully. This should be highlighted in the documentation.
### 3. Priority
-- **High Impact Fixes**:
- 1. Scheduled Types Documentation (to prevent compile errors).
- 2. Structs and Reducers Section (to ensure schema and function alignment).
+1. **Fix Table Name Consistencies**: This will directly resolve numerous failures and prevent potential confusion.
+2. **Add `pub` Keyword Requirement**: Ensuring access to fields would significantly improve usability and reduce errors in testing.
+3. **Document API Stability**: Prevent future issues arising from unexpected API changes.
---
## C# Failures
### 1. Root Causes
-- **Table Naming Issues (19 failures)**:
- - The primary issue causing the failures is the inconsistency in the use of table names (e.g., `entities` vs. `Entity`). Lack of clear guidelines on naming conventions has led to widespread discrepancies.
-
-- **Timeout Issues (3 failures)**:
- - Additionally, the timeout failures indicate that certain processes aren’t being documented well in terms of what expectations exist for execution time and potential pitfalls leading to these issues.
+- **Table Name Consistency**: Like Rust, several tests in C# fail due to improper table names, particularly for `users`, `results`, and `accounts`.
+- **Lack of `public` Modifiers**: Many structs and fields are missing the `public` modifier, which can restrict access from the context in which they are used.
+- **API Instability Documentation**: References to unstable API methods are rampant, causing uncertainty in method usage and expectations.
### 2. Recommendations
-- **Documentation Updates**:
- - **Table Naming Conventions**: Introduce a comprehensive section specifying the naming conventions for tables. Example for addition:
- ```markdown
- ### Table Naming Conventions
- - Table names should be singular and PascalCase (e.g., `User` instead of `users`).
- - Ensure that when creating and querying tables, the names are consistently used to avoid schema parity issues.
- ```
-
- - **Timeout Handling Guidance**: Provide clearer information on how to handle potential timeout issues within operations:
- ```markdown
- ### Handling Timeouts
- - If encountering timeout errors during transactions, consider optimizing the initial data load or query processes.
- - Implement logging to help identify which part of your transaction is leading to the timeout.
- ```
+- **Align Table Names**:
+ - Replace all instances of `Users` with `User`, both in definitions and references.
+ - Standardize other table names like `Results` and `Accounts` similarly.
+- **Include `public` Modifiers**:
+ - The documentation should specify the need for `public` qualifiers on structs and fields explicitly, with examples demonstrating proper usage.
+- **Mark Unstable APIs**:
+ - Document all APIs that are unstable and document their expected changes. Provide clear notes in the sections where these APIs are discussed, especially around construction methods and reducer configurations.
### 3. Priority
-- **High Impact Fixes**:
- 1. Table Naming Conventions (most immediate to fix widespread errors).
- 2. Timeout Handling Guidance (to improve performance and reliability in operations).
+1. **Align Table Names**: This will immediately tackle a significant number of test failures.
+2. **Public Modifier Guidelines**: Enhancing understanding of access levels will greatly improve code quality and tests.
+3. **Unstable API Documentation**: This is a long-term fix but is critical for preventing confusion around API usage in future versions.
----
+---
-This structured approach will help improve the accessibility and clarity of the SpacetimeDB documentation, directly addressing the root causes of current benchmark test failures.
+These actionable insights, with specific attention to documentation updates, can help mitigate the current benchmark failures and improve user experience in both Rust and C# implementations of the SpacetimeDB.
diff --git a/docs/llms/docs-benchmark-comment.md b/docs/llms/docs-benchmark-comment.md
index 00356a7efde..cd086ec3933 100644
--- a/docs/llms/docs-benchmark-comment.md
+++ b/docs/llms/docs-benchmark-comment.md
@@ -2,13 +2,13 @@
| Language | Mode | Category | Tests Passed | Task Pass % |
|----------|------|----------|--------------|-------------|
-| Rust | rustdoc_json | basics | 25/27 | 83.3% ⬇️ -8.3% |
-| Rust | rustdoc_json | schema | 29/34 | 80.0% |
-| Rust | rustdoc_json | **total** | 54/61 | **81.8%** ⬇️ -4.5% |
-| C# | docs | basics | 7/27 | 16.0% ⬇️ -84.0% |
-| C# | docs | schema | 18/34 | 48.5% ⬇️ -31.5% |
-| C# | docs | **total** | 25/61 | **30.8%** ⬇️ -60.2% |
+| Rust | rustdoc_json | basics | 9/27 | 36.1% ⬇️ -47.2% |
+| Rust | rustdoc_json | schema | 9/34 | 26.5% ⬇️ -53.5% |
+| Rust | rustdoc_json | **total** | 18/61 | **31.7%** ⬇️ -50.1% |
+| C# | docs | basics | 12/27 | 61.1% |
+| C# | docs | schema | 10/34 | 31.5% |
+| C# | docs | **total** | 22/61 | **47.7%** |
_Compared against master branch baseline_
-Generated at: 2026-01-16T19:34:55.608Z
+Generated at: 2026-01-20T01:46:16.880Z
diff --git a/docs/llms/docs-benchmark-details.json b/docs/llms/docs-benchmark-details.json
index e7d25c4bce3..c406c4526e3 100644
--- a/docs/llms/docs-benchmark-details.json
+++ b/docs/llms/docs-benchmark-details.json
@@ -19,7 +19,7 @@
"model_name": "GPT-5",
"total_tests": 1,
"passed_tests": 1,
- "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::reducer]\nfn empty_reducer_no_args(_ctx: &ReducerContext) {\n}\n\n#[spacetimedb::reducer]\nfn empty_reducer_with_int(_ctx: &ReducerContext, count: i32) {\n let _ = count;\n}\n\n#[spacetimedb::reducer]\nfn empty_reducer_with_string(_ctx: &ReducerContext, name: String) {\n let _ = name;\n}\n\n#[spacetimedb::reducer]\nfn empty_reducer_with_two_args(_ctx: &ReducerContext, count: i32, name: String) {\n let _ = (count, name);\n}\n\n#[spacetimedb::reducer]\nfn empty_reducer_with_three_args(_ctx: &ReducerContext, active: bool, ratio: f32, label: String) {\n let _ = (active, ratio, label);\n}",
+ "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::reducer]\npub fn empty_reducer_no_args(ctx: &ReducerContext) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_int(ctx: &ReducerContext, count: i32) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_string(ctx: &ReducerContext, name: String) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_two_args(ctx: &ReducerContext, count: i32, name: String) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_three_args(ctx: &ReducerContext, active: bool, ratio: f32, label: String) {\n}",
"category": "basics",
"route_api_model": "gpt-5",
"golden_db": "basics-t-000-empty-reducers-golden",
@@ -35,15 +35,15 @@
"llm_db": "basics-t-000-empty-reducers-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:46587",
+ "server": "http://127.0.0.1:40169",
"tables_diff": null,
"tables_equal": true
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:28:16.167437181Z",
- "finished_at": "2026-01-17T02:30:49.255347669Z"
+ "started_at": "2026-01-20T01:37:53.719374115Z",
+ "finished_at": "2026-01-20T01:38:26.935588621Z"
},
"t_001_basic_tables": {
"hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a",
@@ -53,7 +53,7 @@
"model_name": "GPT-5",
"total_tests": 1,
"passed_tests": 1,
- "llm_output": "use spacetimedb::table;\n\n#[table(name = users)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[table(name = products)]\npub struct Product {\n #[primary_key]\n id: i32,\n title: String,\n price: f32,\n in_stock: bool,\n}\n\n#[table(name = notes)]\npub struct Note {\n #[primary_key]\n id: i32,\n body: String,\n rating: i64,\n pinned: bool,\n}",
+ "llm_output": "use spacetimedb::table;\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[table(name = product)]\npub struct Product {\n #[primary_key]\n id: i32,\n title: String,\n price: f32,\n in_stock: bool,\n}\n\n#[table(name = note)]\npub struct Note {\n #[primary_key]\n id: i32,\n body: String,\n rating: i64,\n pinned: bool,\n}",
"category": "basics",
"route_api_model": "gpt-5",
"golden_db": "basics-t-001-basic-tables-golden",
@@ -69,15 +69,15 @@
"llm_db": "basics-t-001-basic-tables-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:46587",
+ "server": "http://127.0.0.1:40169",
"tables_diff": null,
"tables_equal": true
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:30:52.376540604Z",
- "finished_at": "2026-01-17T02:32:37.875429994Z"
+ "started_at": "2026-01-20T01:37:53.718230540Z",
+ "finished_at": "2026-01-20T01:38:21.752655472Z"
},
"t_002_scheduled_table": {
"hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a",
@@ -87,7 +87,7 @@
"model_name": "GPT-5",
"total_tests": 1,
"passed_tests": 0,
- "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt};\n\n#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n if ctx.db.tick_timer().count() == 0 {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::from_now_micros(50_000),\n });\n }\n}\n\n#[reducer]\npub fn tick(ctx: &ReducerContext, scheduled_id: u64) {\n if let Some(mut row) = ctx.db.tick_timer().scheduled_id().find(scheduled_id) {\n row.scheduled_at = ScheduleAt::from_now_micros(50_000);\n let _ = ctx.db.tick_timer().scheduled_id().update(row);\n }\n}",
+ "llm_output": "use spacetimedb::{ReducerContext, Table, ScheduleAt};\n\n#[spacetimedb::table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n if ctx.db.tick_timer().count() == 0 {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::interval_us(50_000),\n });\n }\n}\n\n#[spacetimedb::reducer(scheduled)]\npub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n}",
"category": "basics",
"route_api_model": "gpt-5",
"golden_db": "basics-t-002-scheduled-table-golden",
@@ -99,14 +99,14 @@
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime publish failed (exit=1)\n--- stderr ---\n\u001b[1m\u001b[32m Updating\u001b[0m crates.io index\n\u001b[1m\u001b[32m Locking\u001b[0m 72 packages to latest compatible versions\n\u001b[1m\u001b[36m Adding\u001b[0m generic-array v0.14.7 \u001b[1m\u001b[33m(available: v0.14.9)\u001b[0m\n\u001b[1m\u001b[36m Adding\u001b[0m spacetimedb v1.11.1 \u001b[1m\u001b[33m(available: v1.11.3)\u001b[0m\n\u001b[1m\u001b[36m Adding\u001b[0m spacetimedb-bindings-macro v1.11.1 \u001b[1m\u001b[33m(available: v1.11.3)\u001b[0m\n\u001b[1m\u001b[36m Adding\u001b[0m spacetimedb-bindings-sys v1.11.1 \u001b[1m\u001b[33m(available: v1.11.3)\u001b[0m\n\u001b[1m\u001b[36m Adding\u001b[0m spacetimedb-lib v1.11.1 \u001b[1m\u001b[33m(available: v1.11.3)\u001b[0m\n\u001b[1m\u001b[36m Adding\u001b[0m spacetimedb-primitives v1.11.1 \u001b[1m\u001b[33m(available: v1.11.3)\u001b[0m\n\u001b[1m\u001b[36m Adding\u001b[0m spacetimedb-sats v1.11.1 \u001b[1m\u001b[33m(available: v1.11.3)\u001b[0m\n\u001b[1m\u001b[32m Compiling\u001b[0m proc-macro2 v1.0.105\n\u001b[1m\u001b[32m Compiling\u001b[0m quote v1.0.43\n\u001b[1m\u001b[32m Compiling\u001b[0m unicode-ident v1.0.22\n\u001b[1m\u001b[32m Compiling\u001b[0m typenum v1.19.0\n\u001b[1m\u001b[32m Compiling\u001b[0m version_check v0.9.5\n\u001b[1m\u001b[32m Compiling\u001b[0m autocfg v1.5.0\n\u001b[1m\u001b[32m Compiling\u001b[0m generic-array v0.14.7\n\u001b[1m\u001b[32m Compiling\u001b[0m heck v0.5.0\n\u001b[1m\u001b[32m Compiling\u001b[0m num-traits v0.2.19\n\u001b[1m\u001b[32m Compiling\u001b[0m serde_core v1.0.228\n\u001b[1m\u001b[32m Compiling\u001b[0m cfg-if v1.0.4\n\u001b[1m\u001b[32m Compiling\u001b[0m syn v2.0.114\n\u001b[1m\u001b[32m Compiling\u001b[0m either v1.15.0\n\u001b[1m\u001b[32m Compiling\u001b[0m find-msvc-tools v0.1.8\n\u001b[1m\u001b[32m Compiling\u001b[0m serde v1.0.228\n\u001b[1m\u001b[32m Compiling\u001b[0m shlex v1.3.0\n\u001b[1m\u001b[32m Compiling\u001b[0m zerocopy v0.8.33\n\u001b[1m\u001b[32m Compiling\u001b[0m cc v1.2.53\n\u001b[1m\u001b[32m Compiling\u001b[0m itertools v0.12.1\n\u001b[1m\u001b[32m Compiling\u001b[0m crypto-common v0.1.7\n\u001b[1m\u001b[32m Compiling\u001b[0m block-buffer v0.10.4\n\u001b[1m\u001b[32m Compiling\u001b[0m thiserror v1.0.69\n\u001b[1m\u001b[32m Compiling\u001b[0m nohash-hasher v0.2.0\n\u001b[1m\u001b[32m Compiling\u001b[0m bitflags v2.10.0\n\u001b[1m\u001b[32m Compiling\u001b[0m anyhow v1.0.100\n\u001b[1m\u001b[32m Compiling\u001b[0m digest v0.10.7\n\u001b[1m\u001b[32m Compiling\u001b[0m blake3 v1.8.3\n\u001b[1m\u001b[32m Compiling\u001b[0m approx v0.3.2\n\u001b[1m\u001b[32m Compiling\u001b[0m getrandom v0.2.17\n\u001b[1m\u001b[32m Compiling\u001b[0m humantime v2.3.0\n\u001b[1m\u001b[32m Compiling\u001b[0m bytes v1.11.0\n\u001b[1m\u001b[32m Compiling\u001b[0m zmij v1.0.14\n\u001b[1m\u001b[32m Compiling\u001b[0m convert_case v0.4.0\n\u001b[1m\u001b[32m Compiling\u001b[0m heck v0.4.1\n\u001b[1m\u001b[32m Compiling\u001b[0m enum-as-inner v0.6.1\n\u001b[1m\u001b[32m Compiling\u001b[0m thiserror-impl v1.0.69\n\u001b[1m\u001b[32m Compiling\u001b[0m arrayvec v0.7.6\n\u001b[1m\u001b[32m Compiling\u001b[0m spacetimedb-primitives v1.11.1\n\u001b[1m\u001b[32m Compiling\u001b[0m keccak v0.1.5\n\u001b[1m\u001b[32m Compiling\u001b[0m spacetimedb-bindings-macro v1.11.1\n\u001b[1m\u001b[32m Compiling\u001b[0m sha3 v0.10.8\n\u001b[1m\u001b[32m Compiling\u001b[0m ppv-lite86 v0.2.21\n\u001b[1m\u001b[32m Compiling\u001b[0m derive_more v0.99.20\n\u001b[1m\u001b[32m Compiling\u001b[0m rand_core v0.6.4\n\u001b[1m\u001b[32m Compiling\u001b[0m decorum v0.3.1\n\u001b[1m\u001b[32m Compiling\u001b[0m ethnum v1.5.2\n\u001b[1m\u001b[32m Compiling\u001b[0m chrono v0.4.43\n\u001b[1m\u001b[32m Compiling\u001b[0m constant_time_eq v0.4.2\n\u001b[1m\u001b[32m Compiling\u001b[0m bytemuck v1.24.0\n\u001b[1m\u001b[32m Compiling\u001b[0m spacetimedb-lib v1.11.1\n\u001b[1m\u001b[32m Compiling\u001b[0m smallvec v1.15.1\n\u001b[1m\u001b[32m Compiling\u001b[0m serde_json v1.0.149\n\u001b[1m\u001b[32m Compiling\u001b[0m itoa v1.0.17\n\u001b[1m\u001b[32m Compiling\u001b[0m hex v0.4.3\n\u001b[1m\u001b[32m Compiling\u001b[0m arrayref v0.3.9\n\u001b[1m\u001b[32m Compiling\u001b[0m second-stack v0.3.5\n\u001b[1m\u001b[32m Compiling\u001b[0m rand_chacha v0.3.1\n\u001b[1m\u001b[32m Compiling\u001b[0m spacetimedb-sats v1.11.1\n\u001b[1m\u001b[32m Compiling\u001b[0m memchr v2.7.6\n\u001b[1m\u001b[32m Compiling\u001b[0m log v0.4.29\n\u001b[1m\u001b[32m Compiling\u001b[0m rand v0.8.5\n\u001b[1m\u001b[32m Compiling\u001b[0m http v1.4.0\n\u001b[1m\u001b[32m Compiling\u001b[0m spacetimedb-bindings-sys v1.11.1\n\u001b[1m\u001b[32m Compiling\u001b[0m scoped-tls v1.0.1\n\u001b[1m\u001b[32m Compiling\u001b[0m spacetimedb v1.11.1\n\u001b[1m\u001b[32m Compiling\u001b[0m spacetime-module v0.1.0 (/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_002_scheduled_table/rust/server/gpt-5/llm)\n\u001b[0m\u001b[1m\u001b[38;5;9merror\u001b[0m\u001b[0m\u001b[1m: expected one of: `public`, `private`, `name`, `index`, `scheduled`\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--> \u001b[0m\u001b[0msrc/lib.rs:4:28\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m4\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9m^^^^^^^^\u001b[0m\n\n\u001b[0m\u001b[1m\u001b[38;5;9merror[E0422]\u001b[0m\u001b[0m\u001b[1m: cannot find struct, variant or union type `TickTimer` in this scope\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--> \u001b[0m\u001b[0msrc/lib.rs:15:36\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m15\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m ctx.db.tick_timer().insert(TickTimer {\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9m^^^^^^^^^\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9mnot found in this scope\u001b[0m\n\n\u001b[0m\u001b[1m\u001b[38;5;9merror[E0599]\u001b[0m\u001b[0m\u001b[1m: no method named `tick_timer` found for struct `Local` in the current scope\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--> \u001b[0m\u001b[0msrc/lib.rs:14:15\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m14\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m if ctx.db.tick_timer().count() == 0 {\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9m^^^^^^^^^^\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9mmethod not found in `Local`\u001b[0m\n\n\u001b[0m\u001b[1m\u001b[38;5;9merror[E0599]\u001b[0m\u001b[0m\u001b[1m: no method named `tick_timer` found for struct `Local` in the current scope\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--> \u001b[0m\u001b[0msrc/lib.rs:15:16\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m15\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m ctx.db.tick_timer().insert(TickTimer {\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9m^^^^^^^^^^\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9mmethod not found in `Local`\u001b[0m\n\n\u001b[0m\u001b[1m\u001b[38;5;9merror[E0599]\u001b[0m\u001b[0m\u001b[1m: no variant or associated item named `from_now_micros` found for enum `ScheduleAt` in the current scope\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--> \u001b[0m\u001b[0msrc/lib.rs:17:39\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m17\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m scheduled_at: ScheduleAt::from_now_micros(50_000),\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9m^^^^^^^^^^^^^^^\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9mvariant or associated item not found in `ScheduleAt`\u001b[0m\n\n\u001b[0m\u001b[1m\u001b[38;5;9merror[E0599]\u001b[0m\u001b[0m\u001b[1m: no method named `tick_timer` found for struct `Local` in the current scope\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--> \u001b[0m\u001b[0msrc/lib.rs:24:35\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m24\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m if let Some(mut row) = ctx.db.tick_timer().scheduled_id().find(scheduled_id) {\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9m^^^^^^^^^^\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9mmethod not found in `Local`\u001b[0m\n\n\u001b[0m\u001b[1m\u001b[38;5;9merror[E0599]\u001b[0m\u001b[0m\u001b[1m: no variant or associated item named `from_now_micros` found for enum `ScheduleAt` in the current scope\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--> \u001b[0m\u001b[0msrc/lib.rs:25:40\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m25\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m row.scheduled_at = ScheduleAt::from_now_micros(50_000);\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9m^^^^^^^^^^^^^^^\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9mvariant or associated item not found in `ScheduleAt`\u001b[0m\n\n\u001b[0m\u001b[1m\u001b[38;5;9merror[E0599]\u001b[0m\u001b[0m\u001b[1m: no method named `tick_timer` found for struct `Local` in the current scope\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--> \u001b[0m\u001b[0msrc/lib.rs:26:24\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m26\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m let _ = ctx.db.tick_timer().scheduled_id().update(row);\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9m^^^^^^^^^^\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9mmethod not found in `Local`\u001b[0m\n\n\u001b[0m\u001b[1mSome errors have detailed explanations: E0422, E0599.\u001b[0m\n\u001b[0m\u001b[1mFor more information about an error, try `rustc --explain E0422`.\u001b[0m\n\u001b[1m\u001b[31merror\u001b[0m\u001b[1m:\u001b[0m could not compile `spacetime-module` (lib) due to 8 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n",
+ "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.105\n Compiling quote v1.0.43\n Compiling unicode-ident v1.0.22\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling heck v0.5.0\n Compiling serde_core v1.0.228\n Compiling cfg-if v1.0.4\n Compiling serde v1.0.228\n Compiling find-msvc-tools v0.1.8\n Compiling either v1.15.0\n Compiling zerocopy v0.8.33\n Compiling shlex v1.3.0\n Compiling bitflags v2.10.0\n Compiling thiserror v1.0.69\n Compiling nohash-hasher v0.2.0\n Compiling anyhow v1.0.100\n Compiling bytes v1.11.0\n Compiling zmij v1.0.15\n Compiling arrayvec v0.7.6\n Compiling humantime v2.3.0\n Compiling convert_case v0.4.0\n Compiling heck v0.4.1\n Compiling keccak v0.1.5\n Compiling constant_time_eq v0.4.2\n Compiling itoa v1.0.17\n Compiling second-stack v0.3.5\n Compiling smallvec v1.15.1\n Compiling getrandom v0.2.17\n Compiling bytemuck v1.24.0\n Compiling rand_core v0.6.4\n Compiling arrayref v0.3.9\n Compiling hex v0.4.3\n Compiling serde_json v1.0.149\n Compiling spacetimedb-lib v1.11.1\n Compiling log v0.4.29\n Compiling cc v1.2.53\n Compiling memchr v2.7.6\n Compiling itertools v0.12.1\n Compiling scoped-tls v1.0.1\n Compiling generic-array v0.14.7\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling blake3 v1.8.3\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling crypto-common v0.1.7\n Compiling block-buffer v0.10.4\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling ethnum v1.5.2\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_002_scheduled_table/rust/server/gpt-5/llm)\nerror: expected one of: `public`, `private`, `name`, `index`, `scheduled`\n --> src/lib.rs:4:41\n |\n4 | #[spacetimedb::table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\n | ^^^^^^^^\n\nerror: expected one of: `init`, `client_connected`, `client_disconnected`, `update`, `name`\n --> src/lib.rs:22:24\n |\n22 | #[spacetimedb::reducer(scheduled)]\n | ^^^^^^^^^\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:15:36\n |\n15 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0412]: cannot find type `TickTimer` in this scope\n --> src/lib.rs:23:42\n |\n23 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:14:15\n |\n14 | if ctx.db.tick_timer().count() == 0 {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:15:16\n |\n15 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `interval_us` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:17:39\n |\n17 | scheduled_at: ScheduleAt::interval_us(50_000),\n | ^^^^^^^^^^^ variant or associated item not found in `ScheduleAt`\n\nSome errors have detailed explanations: E0412, E0422, E0599.\nFor more information about an error, try `rustc --explain E0412`.\nerror: could not compile `spacetime-module` (lib) due to 7 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n",
"phase": "build_or_publish"
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:30:42.103317349Z",
- "finished_at": "2026-01-17T02:32:41.832312901Z"
+ "started_at": "2026-01-20T01:37:53.719615718Z",
+ "finished_at": "2026-01-20T01:38:51.848826124Z"
},
"t_003_struct_in_table": {
"hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a",
@@ -116,7 +116,7 @@
"model_name": "GPT-5",
"total_tests": 1,
"passed_tests": 0,
- "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[table(name = entities)]\npub struct Entity {\n #[primary_key]\n id: i32,\n pos: Position,\n}\n\n#[reducer]\npub fn add_entity(ctx: &ReducerContext, id: i32, x: i32, y: i32) {\n ctx.db.entities().insert(Entity { id, pos: Position { x, y } });\n}",
+ "llm_output": "use spacetimedb::{ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n pos: Position,\n}\n\n#[spacetimedb::reducer]\npub fn add_entity(ctx: &ReducerContext, id: i32, x: i32, y: i32) {\n ctx.db.entity().insert(Entity { id, pos: Position { x, y } });\n}",
"category": "basics",
"route_api_model": "gpt-5",
"golden_db": "basics-t-003-struct-in-table-golden",
@@ -137,15 +137,15 @@
]
},
"reducers_equal": false,
- "server": "http://127.0.0.1:46587",
+ "server": "http://127.0.0.1:40169",
"tables_diff": null,
"tables_equal": true
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:30:49.255385009Z",
- "finished_at": "2026-01-17T02:32:37.661244218Z"
+ "started_at": "2026-01-20T01:37:53.718572027Z",
+ "finished_at": "2026-01-20T01:38:23.204099278Z"
},
"t_004_insert": {
"hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a",
@@ -154,8 +154,8 @@
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 2,
- "passed_tests": 2,
- "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = users)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn insert_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n ctx.db.users().insert(User { id, name, age, active });\n}",
+ "passed_tests": 1,
+ "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn insert_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n ctx.db.user().insert(User { id, name, age, active });\n}",
"category": "basics",
"route_api_model": "gpt-5",
"golden_db": "basics-t-004-insert-golden",
@@ -163,25 +163,6 @@
"work_dir_golden": "target/llm-runs/basics/t_004_insert/rust/server/golden",
"work_dir_llm": "target/llm-runs/basics/t_004_insert/rust/server/gpt-5/llm",
"scorer_details": {
- "data_parity_insert_user": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "args": [
- 1,
- "Alice",
- 30,
- true
- ],
- "golden_db": "basics-t-004-insert-golden",
- "golden_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true",
- "llm_db": "basics-t-004-insert-gpt-5-llm",
- "llm_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true",
- "query": "SELECT id, name, age, active FROM users WHERE id=1",
- "reducer": "insert_user",
- "server": "http://127.0.0.1:46587"
- }
- },
"schema_parity": {
"pass": true,
"partial": 1.0,
@@ -190,15 +171,23 @@
"llm_db": "basics-t-004-insert-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:46587",
+ "server": "http://127.0.0.1:40169",
"tables_diff": null,
"tables_equal": true
}
+ },
+ "data_parity_insert_user": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40169/v1/database/c2003edd8f08ea2a3ce12332cee8aecdc8ae76399dd7a8276c213edf822adddb/sql)\n",
+ "phase": "sql_golden"
+ }
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:30:53.628765951Z",
- "finished_at": "2026-01-17T02:32:37.475748331Z"
+ "started_at": "2026-01-20T01:37:53.719159103Z",
+ "finished_at": "2026-01-20T01:38:19.328531147Z"
},
"t_005_update": {
"hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a",
@@ -207,8 +196,8 @@
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 3,
- "passed_tests": 3,
- "llm_output": "use spacetimedb::{table, reducer, ReducerContext};\n\n#[table(name = users)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n let user = User { id, name, age, active };\n ctx.db.users().id().update(user);\n}",
+ "passed_tests": 1,
+ "llm_output": "use spacetimedb::{table, reducer, ReducerContext};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n let users = ctx.db.user();\n let updated = User { id, name, age, active };\n users.id().update(updated);\n}",
"category": "basics",
"route_api_model": "gpt-5",
"golden_db": "basics-t-005-update-golden",
@@ -217,28 +206,19 @@
"work_dir_llm": "target/llm-runs/basics/t_005_update/rust/server/gpt-5/llm",
"scorer_details": {
"data_parity_update_user": {
- "pass": true,
- "partial": 1.0,
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "args": [
- 1,
- "Alice2",
- 31,
- false
- ],
- "golden_db": "basics-t-005-update-golden",
- "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
- "llm_db": "basics-t-005-update-gpt-5-llm",
- "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
- "query": "SELECT id, name, age, active FROM users WHERE id=1",
- "reducer": "update_user",
- "server": "http://127.0.0.1:46587"
+ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: The instance encountered a fatal error.\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:40169/v1/database/c200dec624659e9d32ea59b2a7217e2104b7fd68e2dbebb872559c4f5f0c0be2/call/update_user)\n",
+ "phase": "call_reducer_golden"
}
},
"seed_users_row": {
- "pass": true,
- "partial": 1.0,
+ "pass": false,
+ "partial": 0.0,
"notes": {
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40169/v1/database/c200dec624659e9d32ea59b2a7217e2104b7fd68e2dbebb872559c4f5f0c0be2/sql)\n",
+ "phase": "sql_golden",
"sql": "INSERT INTO users(id, name, age, active) VALUES (1, 'Alice', 30, true)"
}
},
@@ -250,15 +230,15 @@
"llm_db": "basics-t-005-update-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:46587",
+ "server": "http://127.0.0.1:40169",
"tables_diff": null,
"tables_equal": true
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:28:20.539096519Z",
- "finished_at": "2026-01-17T02:30:53.033151846Z"
+ "started_at": "2026-01-20T01:38:19.328919795Z",
+ "finished_at": "2026-01-20T01:38:56.779253262Z"
},
"t_006_delete": {
"hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a",
@@ -267,8 +247,8 @@
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 3,
- "passed_tests": 3,
- "llm_output": "use spacetimedb::{table, reducer, ReducerContext};\n\n#[table(name = users)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn delete_user(ctx: &ReducerContext, id: i32) {\n ctx.db.users().id().delete(&id);\n}",
+ "passed_tests": 1,
+ "llm_output": "use spacetimedb::{table, reducer, ReducerContext};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn delete_user(ctx: &ReducerContext, id: i32) {\n ctx.db.user().id().delete(&id);\n}",
"category": "basics",
"route_api_model": "gpt-5",
"golden_db": "basics-t-006-delete-golden",
@@ -276,20 +256,12 @@
"work_dir_golden": "target/llm-runs/basics/t_006_delete/rust/server/golden",
"work_dir_llm": "target/llm-runs/basics/t_006_delete/rust/server/gpt-5/llm",
"scorer_details": {
- "seed_users_row": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "sql": "INSERT INTO users(id, name, age, active) VALUES (1, 'Alice', 30, true)"
- }
- },
"delete_user_count_zero": {
- "pass": true,
- "partial": 1.0,
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "actual": 0,
- "expected": 0,
- "sql": "SELECT COUNT(*) AS n FROM users WHERE id=1"
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40169/v1/database/c20069ec5fbbbe5d6776b212b8a61457be4bf185ebc1fb09e92c83a672eafcb0/sql)\n",
+ "phase": "sql"
}
},
"schema_parity": {
@@ -300,15 +272,24 @@
"llm_db": "basics-t-006-delete-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:46587",
+ "server": "http://127.0.0.1:40169",
"tables_diff": null,
"tables_equal": true
}
+ },
+ "seed_users_row": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40169/v1/database/c200f842d919e3cec49c28d8258acd70580bdde387c04d64356d20fab03988be/sql)\n",
+ "phase": "sql_golden",
+ "sql": "INSERT INTO users(id, name, age, active) VALUES (1, 'Alice', 30, true)"
+ }
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:28:17.746281172Z",
- "finished_at": "2026-01-17T02:30:42.103247549Z"
+ "started_at": "2026-01-20T01:38:22.830268100Z",
+ "finished_at": "2026-01-20T01:38:57.132760254Z"
},
"t_007_crud": {
"hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a",
@@ -317,8 +298,8 @@
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 4,
- "passed_tests": 4,
- "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = users)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn crud(ctx: &ReducerContext) {\n let mut u1 = ctx.db.users().insert(User {\n id: 1,\n name: \"Alice\".to_string(),\n age: 30,\n active: true,\n });\n ctx.db.users().insert(User {\n id: 2,\n name: \"Bob\".to_string(),\n age: 22,\n active: false,\n });\n u1.name = \"Alice2\".to_string();\n u1.age = 31;\n u1.active = false;\n let _ = ctx.db.users().id().update(u1);\n ctx.db.users().id().delete(&2);\n}",
+ "passed_tests": 1,
+ "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn crud(ctx: &ReducerContext) {\n let users = ctx.db.user();\n\n users.insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n users.insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n\n if let Some(mut u1) = users.id().find(1) {\n u1.name = \"Alice2\".to_string();\n u1.age = 31;\n u1.active = false;\n users.id().update(u1);\n }\n\n users.id().delete(&2);\n}",
"category": "basics",
"route_api_model": "gpt-5",
"golden_db": "basics-t-007-crud-golden",
@@ -326,13 +307,20 @@
"work_dir_golden": "target/llm-runs/basics/t_007_crud/rust/server/golden",
"work_dir_llm": "target/llm-runs/basics/t_007_crud/rust/server/gpt-5/llm",
"scorer_details": {
- "crud_row_id2_deleted": {
- "pass": true,
- "partial": 1.0,
+ "crud_total_count_one": {
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "actual": 0,
- "expected": 0,
- "sql": "SELECT COUNT(*) AS n FROM users WHERE id=2"
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40169/v1/database/c20066de29f200472dee55afeb7df24c65e1842a602130d3a7ed7c8634df0e02/sql)\n",
+ "phase": "sql"
+ }
+ },
+ "crud_row_id1_parity": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40169/v1/database/c20078038024aed97bb4b5da16a4014ec08b58881a3550910b086b6718d8e75e/sql)\n",
+ "phase": "sql_golden"
}
},
"schema_parity": {
@@ -343,38 +331,23 @@
"llm_db": "basics-t-007-crud-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:46587",
+ "server": "http://127.0.0.1:40169",
"tables_diff": null,
"tables_equal": true
}
},
- "crud_total_count_one": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM users"
- }
- },
- "crud_row_id1_parity": {
- "pass": true,
- "partial": 1.0,
+ "crud_row_id2_deleted": {
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "args": [],
- "golden_db": "basics-t-007-crud-golden",
- "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
- "llm_db": "basics-t-007-crud-gpt-5-llm",
- "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
- "query": "SELECT id, name, age, active FROM users WHERE id=1",
- "reducer": "crud",
- "server": "http://127.0.0.1:46587"
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40169/v1/database/c20066de29f200472dee55afeb7df24c65e1842a602130d3a7ed7c8634df0e02/sql)\n",
+ "phase": "sql"
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:28:16.167798252Z",
- "finished_at": "2026-01-17T02:30:53.400172120Z"
+ "started_at": "2026-01-20T01:37:53.720028514Z",
+ "finished_at": "2026-01-20T01:38:22.830225666Z"
},
"t_008_index_lookup": {
"hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a",
@@ -383,8 +356,8 @@
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 3,
- "passed_tests": 3,
- "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = users)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[table(name = results)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[reducer]\nfn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(u) = ctx.db.users().id().find(id) {\n ctx.db.results().insert(ResultRow { id: u.id, name: u.name });\n }\n}",
+ "passed_tests": 1,
+ "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(u) = ctx.db.user().id().find(id) {\n let result_table = ctx.db.result();\n if let Some(_) = result_table.id().find(u.id) {\n result_table.id().delete(&u.id);\n }\n result_table.insert(ResultRow { id: u.id, name: u.name });\n }\n}",
"category": "basics",
"route_api_model": "gpt-5",
"golden_db": "basics-t-008-index-lookup-golden",
@@ -393,25 +366,19 @@
"work_dir_llm": "target/llm-runs/basics/t_008_index_lookup/rust/server/gpt-5/llm",
"scorer_details": {
"index_lookup_projection_parity": {
- "pass": true,
- "partial": 1.0,
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "args": [
- 1
- ],
- "golden_db": "basics-t-008-index-lookup-golden",
- "golden_out": "id | name ----+--------- 1 | \"Alice\"",
- "llm_db": "basics-t-008-index-lookup-gpt-5-llm",
- "llm_out": "id | name ----+--------- 1 | \"Alice\"",
- "query": "SELECT id, name FROM results WHERE id=1",
- "reducer": "lookup_user_name",
- "server": "http://127.0.0.1:46587"
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `results`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40169/v1/database/c2006068ef79dd448a9cc07e39a336da978f15292f6e32efffa616385dfb2947/sql)\n",
+ "phase": "sql_golden"
}
},
"seed_user_row": {
- "pass": true,
- "partial": 1.0,
+ "pass": false,
+ "partial": 0.0,
"notes": {
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40169/v1/database/c2006068ef79dd448a9cc07e39a336da978f15292f6e32efffa616385dfb2947/sql)\n",
+ "phase": "sql_golden",
"sql": "INSERT INTO users(id, name, age, active) VALUES (1, 'Alice', 30, true)"
}
},
@@ -423,15 +390,15 @@
"llm_db": "basics-t-008-index-lookup-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:46587",
+ "server": "http://127.0.0.1:40169",
"tables_diff": null,
"tables_equal": true
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:28:17.748044021Z",
- "finished_at": "2026-01-17T02:30:54.245888026Z"
+ "started_at": "2026-01-20T01:38:21.752700657Z",
+ "finished_at": "2026-01-20T01:39:09.906819088Z"
},
"t_009_init": {
"hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a",
@@ -440,8 +407,8 @@
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 4,
- "passed_tests": 4,
- "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = users)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.users().insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n ctx.db.users().insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n}",
+ "passed_tests": 1,
+ "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.user().insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n ctx.db.user().insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n}",
"category": "basics",
"route_api_model": "gpt-5",
"golden_db": "basics-t-009-init-golden",
@@ -449,31 +416,28 @@
"work_dir_golden": "target/llm-runs/basics/t_009_init/rust/server/golden",
"work_dir_llm": "target/llm-runs/basics/t_009_init/rust/server/gpt-5/llm",
"scorer_details": {
- "init_total_two": {
- "pass": true,
- "partial": 1.0,
+ "init_seed_bob": {
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "actual": 2,
- "expected": 2,
- "sql": "SELECT COUNT(*) AS n FROM users"
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40169/v1/database/c200fc7acece1da6b0afc73ca9f6d055c75c5c1bf2f35423e6f8f4eae5cf9fa0/sql)\n",
+ "phase": "sql"
}
},
- "init_seed_bob": {
- "pass": true,
- "partial": 1.0,
+ "init_total_two": {
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM users WHERE id=2 AND name='Bob' AND age=22 AND active=false"
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40169/v1/database/c200fc7acece1da6b0afc73ca9f6d055c75c5c1bf2f35423e6f8f4eae5cf9fa0/sql)\n",
+ "phase": "sql"
}
},
"init_seed_alice": {
- "pass": true,
- "partial": 1.0,
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM users WHERE id=1 AND name='Alice' AND age=30 AND active=true"
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40169/v1/database/c200fc7acece1da6b0afc73ca9f6d055c75c5c1bf2f35423e6f8f4eae5cf9fa0/sql)\n",
+ "phase": "sql"
}
},
"schema_parity": {
@@ -484,15 +448,15 @@
"llm_db": "basics-t-009-init-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:46587",
+ "server": "http://127.0.0.1:40169",
"tables_diff": null,
"tables_equal": true
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:28:17.930019173Z",
- "finished_at": "2026-01-17T02:30:41.209696143Z"
+ "started_at": "2026-01-20T01:38:20.042902379Z",
+ "finished_at": "2026-01-20T01:38:59.046285899Z"
},
"t_010_connect": {
"hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a",
@@ -501,8 +465,8 @@
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 1,
- "passed_tests": 1,
- "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = events)]\npub struct Event {\n #[primary_key]\n #[auto_inc]\n id: i32,\n kind: String,\n}\n\n#[reducer(client_connected)]\npub fn client_connected(ctx: &ReducerContext) {\n ctx.db.events().insert(Event { id: 0, kind: \"connected\".to_string() });\n}\n\n#[reducer(client_disconnected)]\npub fn client_disconnected(ctx: &ReducerContext) {\n ctx.db.events().insert(Event { id: 0, kind: \"disconnected\".to_string() });\n}",
+ "passed_tests": 0,
+ "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = events)]\npub struct Event {\n #[primary_key]\n #[auto_inc]\n id: i32,\n kind: String,\n}\n\n#[spacetimedb::reducer(client_connected)]\npub fn client_connected(ctx: &ReducerContext) {\n ctx.db.events().insert(Event { id: 0, kind: \"connected\".to_string() });\n}\n\n#[spacetimedb::reducer(client_disconnected)]\npub fn client_disconnected(ctx: &ReducerContext) {\n ctx.db.events().insert(Event { id: 0, kind: \"disconnected\".to_string() });\n}",
"category": "basics",
"route_api_model": "gpt-5",
"golden_db": "basics-t-010-connect-golden",
@@ -511,22 +475,30 @@
"work_dir_llm": "target/llm-runs/basics/t_010_connect/rust/server/gpt-5/llm",
"scorer_details": {
"schema_parity": {
- "pass": true,
- "partial": 1.0,
+ "pass": false,
+ "partial": 0.0,
"notes": {
"golden_db": "basics-t-010-connect-golden",
"llm_db": "basics-t-010-connect-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:46587",
- "tables_diff": null,
- "tables_equal": true
+ "server": "http://127.0.0.1:40169",
+ "tables_diff": {
+ "changed": {},
+ "only_golden": {
+ "event": {}
+ },
+ "only_llm": {
+ "events": {}
+ }
+ },
+ "tables_equal": false
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:30:53.626651147Z",
- "finished_at": "2026-01-17T02:32:36.216348757Z"
+ "started_at": "2026-01-20T01:37:53.719821484Z",
+ "finished_at": "2026-01-20T01:38:26.116583750Z"
},
"t_011_helper_function": {
"hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a",
@@ -535,8 +507,8 @@
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 3,
- "passed_tests": 3,
- "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = results)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n sum: i32,\n}\n\nfn add(a: i32, b: i32) -> i32 {\n a + b\n}\n\n#[reducer]\nfn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) {\n let sum = add(a, b);\n ctx.db.results().insert(ResultRow { id, sum });\n}",
+ "passed_tests": 1,
+ "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n sum: i32,\n}\n\nfn add(a: i32, b: i32) -> i32 {\n a + b\n}\n\n#[reducer]\nfn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) {\n let sum = add(a, b);\n ctx.db.result().insert(ResultRow { id, sum });\n}",
"category": "basics",
"route_api_model": "gpt-5",
"golden_db": "basics-t-011-helper-function-golden",
@@ -545,21 +517,11 @@
"work_dir_llm": "target/llm-runs/basics/t_011_helper_function/rust/server/gpt-5/llm",
"scorer_details": {
"helper_func_sum_parity": {
- "pass": true,
- "partial": 1.0,
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "args": [
- 1,
- 2,
- 3
- ],
- "golden_db": "basics-t-011-helper-function-golden",
- "golden_out": "id | sum ----+----- 1 | 5",
- "llm_db": "basics-t-011-helper-function-gpt-5-llm",
- "llm_out": "id | sum ----+----- 1 | 5",
- "query": "SELECT id, sum FROM results WHERE id=1",
- "reducer": "compute_sum",
- "server": "http://127.0.0.1:46587"
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `results`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40169/v1/database/c200af79c18dbe6b5baf26d7143e83063d991c36fc7305a70bcd6d59851875cc/sql)\n",
+ "phase": "sql_golden"
}
},
"schema_parity": {
@@ -570,24 +532,23 @@
"llm_db": "basics-t-011-helper-function-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:46587",
+ "server": "http://127.0.0.1:40169",
"tables_diff": null,
"tables_equal": true
}
},
"helper_func_sum_abs": {
- "pass": true,
- "partial": 1.0,
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM results WHERE id=1 AND sum=5"
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `results`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40169/v1/database/c200e67cbd9b3fbc47e1e267af8ba280cc220d8af5bdaeae915d132ba4c00a54/sql)\n",
+ "phase": "sql"
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:30:42.105283707Z",
- "finished_at": "2026-01-17T02:31:52.745496413Z"
+ "started_at": "2026-01-20T01:37:53.718900316Z",
+ "finished_at": "2026-01-20T01:38:20.042798579Z"
},
"t_012_spacetime_product_type": {
"hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a",
@@ -596,8 +557,8 @@
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 3,
- "passed_tests": 2,
- "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[derive(spacetimedb::SpacetimeType)]\npub struct Score {\n left: i32,\n right: i32,\n}\n\n#[spacetimedb::table(name = results)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Score,\n}\n\n#[spacetimedb::reducer]\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db.results().insert(ResultRow { id, value: Score { left, right } });\n}",
+ "passed_tests": 1,
+ "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType)]\nstruct Score {\n left: i32,\n right: i32,\n}\n\n#[spacetimedb::table(name = result)]\nstruct ResultRow {\n #[primary_key]\n id: i32,\n value: Score,\n}\n\n#[spacetimedb::reducer]\nfn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Score { left, right },\n });\n}",
"category": "schema",
"route_api_model": "gpt-5",
"golden_db": "schema-t-012-spacetime-product-type-golden",
@@ -605,40 +566,39 @@
"work_dir_golden": "target/llm-runs/schema/t_012_spacetime_product_type/rust/server/golden",
"work_dir_llm": "target/llm-runs/schema/t_012_spacetime_product_type/rust/server/gpt-5/llm",
"scorer_details": {
- "product_type_row_count": {
+ "schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM results WHERE id=1"
+ "golden_db": "schema-t-012-spacetime-product-type-golden",
+ "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "http://127.0.0.1:40169",
+ "tables_diff": null,
+ "tables_equal": true
}
},
- "product_type_row_parity": {
+ "product_type_row_count": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `results`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46587/v1/database/c20064919d5938e3c53955f56eff54581cc88db76e6afe6cedff286135db8edb/sql)\n",
- "phase": "sql_golden"
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `results`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40169/v1/database/c200b9b3f3aff71e7b634e31fd11d64412dc8a92b962a0493a4de63f11a02303/sql)\n",
+ "phase": "sql"
}
},
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
+ "product_type_row_parity": {
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "golden_db": "schema-t-012-spacetime-product-type-golden",
- "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "http://127.0.0.1:46587",
- "tables_diff": null,
- "tables_equal": true
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `results`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40169/v1/database/c2009882fabff694084850b3130329e2a907f262bc076fa6a61bf82c7aa7ed16/sql)\n",
+ "phase": "sql_golden"
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:25:38.992696821Z",
- "finished_at": "2026-01-17T02:28:16.167373502Z"
+ "started_at": "2026-01-20T01:38:56.779299897Z",
+ "finished_at": "2026-01-20T01:39:32.670215985Z"
},
"t_013_spacetime_sum_type": {
"hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a",
@@ -647,8 +607,8 @@
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 3,
- "passed_tests": 3,
- "llm_output": "use spacetimedb::{table, reducer, ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = results)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Shape,\n}\n\n#[reducer]\npub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db.results().id().delete(&id);\n ctx.db.results().insert(ResultRow { id, value: Shape::Circle(radius) });\n}",
+ "passed_tests": 1,
+ "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Shape,\n}\n\n#[reducer]\npub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Shape::Circle(radius),\n });\n}",
"category": "schema",
"route_api_model": "gpt-5",
"golden_db": "schema-t-013-spacetime-sum-type-golden",
@@ -656,49 +616,39 @@
"work_dir_golden": "target/llm-runs/schema/t_013_spacetime_sum_type/rust/server/golden",
"work_dir_llm": "target/llm-runs/schema/t_013_spacetime_sum_type/rust/server/gpt-5/llm",
"scorer_details": {
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
+ "sum_type_row_parity": {
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "golden_db": "schema-t-013-spacetime-sum-type-golden",
- "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "http://127.0.0.1:46587",
- "tables_diff": null,
- "tables_equal": true
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `results`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40169/v1/database/c200ff92def9d694cbae6ff5bbd223834238e65b34da96fb0db6b97fe6f79a60/sql)\n",
+ "phase": "sql_golden"
}
},
"sum_type_row_count": {
- "pass": true,
- "partial": 1.0,
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM results WHERE id=1"
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `results`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40169/v1/database/c200e3238b13394ad822c736e0ee93ce6aea393843c479048b2a00d28d80db79/sql)\n",
+ "phase": "sql"
}
},
- "sum_type_row_parity": {
+ "schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "args": [
- 1,
- 10
- ],
"golden_db": "schema-t-013-spacetime-sum-type-golden",
- "golden_out": "id | value ----+--------------- 1 | (Circle = 10)",
"llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm",
- "llm_out": "id | value ----+--------------- 1 | (Circle = 10)",
- "query": "SELECT id, value FROM results WHERE id=1",
- "reducer": "set_circle",
- "server": "http://127.0.0.1:46587"
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "http://127.0.0.1:40169",
+ "tables_diff": null,
+ "tables_equal": true
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:28:12.198644503Z",
- "finished_at": "2026-01-17T02:30:53.626583041Z"
+ "started_at": "2026-01-20T01:38:59.046328175Z",
+ "finished_at": "2026-01-20T01:39:32.231241139Z"
},
"t_014_elementary_columns": {
"hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a",
@@ -707,8 +657,8 @@
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 3,
- "passed_tests": 3,
- "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = primitives)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitives().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000i64,\n price: 1.5f32,\n ratio: 2.25f64,\n active: true,\n name: \"Alice\".to_string(),\n });\n}",
+ "passed_tests": 1,
+ "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = primitives)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitives().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000_i64,\n price: 1.5_f32,\n ratio: 2.25_f64,\n active: true,\n name: \"Alice\".to_string(),\n });\n}",
"category": "schema",
"route_api_model": "gpt-5",
"golden_db": "schema-t-014-elementary-columns-golden",
@@ -716,258 +666,1254 @@
"work_dir_golden": "target/llm-runs/schema/t_014_elementary_columns/rust/server/golden",
"work_dir_llm": "target/llm-runs/schema/t_014_elementary_columns/rust/server/gpt-5/llm",
"scorer_details": {
- "elementary_columns_row_count": {
- "pass": true,
- "partial": 1.0,
+ "elementary_columns_row_parity": {
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM primitives WHERE id=1"
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitives`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40169/v1/database/c2003ab1edea4b403b2887382106ab51cf26428fd73e37d6a77f2c4586800090/sql)\n",
+ "phase": "sql_golden"
}
},
"schema_parity": {
- "pass": true,
- "partial": 1.0,
+ "pass": false,
+ "partial": 0.0,
"notes": {
"golden_db": "schema-t-014-elementary-columns-golden",
"llm_db": "schema-t-014-elementary-columns-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:46587",
- "tables_diff": null,
- "tables_equal": true
+ "server": "http://127.0.0.1:40169",
+ "tables_diff": {
+ "changed": {},
+ "only_golden": {
+ "primitive": {}
+ },
+ "only_llm": {
+ "primitives": {}
+ }
+ },
+ "tables_equal": false
+ }
+ },
+ "elementary_columns_row_count": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM primitives WHERE id=1"
+ }
+ }
+ },
+ "vendor": "openai",
+ "started_at": "2026-01-20T01:38:57.132884681Z",
+ "finished_at": "2026-01-20T01:39:32.403925195Z"
+ },
+ "t_015_product_type_columns": {
+ "hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a",
+ "task": "t_015_product_type_columns",
+ "lang": "rust",
+ "golden_published": true,
+ "model_name": "GPT-5",
+ "total_tests": 3,
+ "passed_tests": 1,
+ "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[table(name = profile)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.profile().insert(Profile {\n id: 1,\n home: Address { street: \"1 Main\".to_string(), zip: 11111 },\n work: Address { street: \"2 Broad\".to_string(), zip: 22222 },\n pos: Position { x: 7, y: 9 },\n });\n}",
+ "category": "schema",
+ "route_api_model": "gpt-5",
+ "golden_db": "schema-t-015-product-type-columns-golden",
+ "llm_db": "schema-t-015-product-type-columns-gpt-5-llm",
+ "work_dir_golden": "target/llm-runs/schema/t_015_product_type_columns/rust/server/golden",
+ "work_dir_llm": "target/llm-runs/schema/t_015_product_type_columns/rust/server/gpt-5/llm",
+ "scorer_details": {
+ "product_type_columns_row_count": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `profiles`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40169/v1/database/c200aad3838b0b1e666c7b6302fba4876ca07a33e50611dfdbda06e123a65f6b/sql)\n",
+ "phase": "sql"
+ }
+ },
+ "product_type_columns_row_parity": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `profiles`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40169/v1/database/c2006d0dc00f0a2fd91742b9fb3affddb52c9b97a8c54dfd8b21f407efb4060a/sql)\n",
+ "phase": "sql_golden"
+ }
+ },
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "golden_db": "schema-t-015-product-type-columns-golden",
+ "llm_db": "schema-t-015-product-type-columns-gpt-5-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "http://127.0.0.1:40169",
+ "tables_diff": null,
+ "tables_equal": true
+ }
+ }
+ },
+ "vendor": "openai",
+ "started_at": "2026-01-20T01:38:59.046663368Z",
+ "finished_at": "2026-01-20T01:39:32.093791922Z"
+ },
+ "t_016_sum_type_columns": {
+ "hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a",
+ "task": "t_016_sum_type_columns",
+ "lang": "rust",
+ "golden_published": true,
+ "model_name": "GPT-5",
+ "total_tests": 3,
+ "passed_tests": 1,
+ "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\nstruct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\nenum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = drawings)]\nstruct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[reducer]\nfn seed(ctx: &ReducerContext) {\n ctx.db.drawings().insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n}",
+ "category": "schema",
+ "route_api_model": "gpt-5",
+ "golden_db": "schema-t-016-sum-type-columns-golden",
+ "llm_db": "schema-t-016-sum-type-columns-gpt-5-llm",
+ "work_dir_golden": "target/llm-runs/schema/t_016_sum_type_columns/rust/server/golden",
+ "work_dir_llm": "target/llm-runs/schema/t_016_sum_type_columns/rust/server/gpt-5/llm",
+ "scorer_details": {
+ "sum_type_columns_row_count": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM drawings WHERE id=1"
+ }
+ },
+ "schema_parity": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "golden_db": "schema-t-016-sum-type-columns-golden",
+ "llm_db": "schema-t-016-sum-type-columns-gpt-5-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "http://127.0.0.1:40169",
+ "tables_diff": {
+ "changed": {},
+ "only_golden": {
+ "drawing": {}
+ },
+ "only_llm": {
+ "drawings": {}
+ }
+ },
+ "tables_equal": false
+ }
+ },
+ "sum_type_columns_row_parity": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40169/v1/database/c200c1f153dbe0f97f16725b237b0fa3171f71af389c56994d21a03a3a585689/sql)\n",
+ "phase": "sql_golden"
+ }
+ }
+ },
+ "vendor": "openai",
+ "started_at": "2026-01-20T01:39:09.906871504Z",
+ "finished_at": "2026-01-20T01:39:44.286357862Z"
+ },
+ "t_017_scheduled_columns": {
+ "hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a",
+ "task": "t_017_scheduled_columns",
+ "lang": "rust",
+ "golden_published": false,
+ "model_name": "GPT-5",
+ "total_tests": 2,
+ "passed_tests": 0,
+ "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt};\n\n#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n if ctx.db.tick_timer().count() == 0 {\n ctx.db\n .tick_timer()\n .insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::repeat_micros(50_000),\n });\n }\n}\n\n#[reducer]\npub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n}",
+ "category": "schema",
+ "route_api_model": "gpt-5",
+ "golden_db": "schema-t-017-scheduled-columns-golden",
+ "llm_db": "schema-t-017-scheduled-columns-gpt-5-llm",
+ "work_dir_golden": "target/llm-runs/schema/t_017_scheduled_columns/rust/server/golden",
+ "work_dir_llm": "target/llm-runs/schema/t_017_scheduled_columns/rust/server/gpt-5/llm",
+ "scorer_details": {
+ "publish_error": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.105\n Compiling quote v1.0.43\n Compiling unicode-ident v1.0.22\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling cfg-if v1.0.4\n Compiling shlex v1.3.0\n Compiling either v1.15.0\n Compiling serde v1.0.228\n Compiling find-msvc-tools v0.1.8\n Compiling zerocopy v0.8.33\n Compiling bitflags v2.10.0\n Compiling nohash-hasher v0.2.0\n Compiling anyhow v1.0.100\n Compiling thiserror v1.0.69\n Compiling convert_case v0.4.0\n Compiling heck v0.4.1\n Compiling bytes v1.11.0\n Compiling keccak v0.1.5\n Compiling arrayvec v0.7.6\n Compiling humantime v2.3.0\n Compiling zmij v1.0.15\n Compiling spacetimedb-lib v1.11.1\n Compiling itoa v1.0.17\n Compiling constant_time_eq v0.4.2\n Compiling serde_json v1.0.149\n Compiling getrandom v0.2.17\n Compiling hex v0.4.3\n Compiling cc v1.2.53\n Compiling itertools v0.12.1\n Compiling arrayref v0.3.9\n Compiling bytemuck v1.24.0\n Compiling second-stack v0.3.5\n Compiling smallvec v1.15.1\n Compiling memchr v2.7.6\n Compiling log v0.4.29\n Compiling scoped-tls v1.0.1\n Compiling rand_core v0.6.4\n Compiling generic-array v0.14.7\n Compiling num-traits v0.2.19\n Compiling syn v2.0.114\n Compiling http v1.4.0\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling blake3 v1.8.3\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling ethnum v1.5.2\n Compiling rand v0.8.5\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_017_scheduled_columns/rust/server/gpt-5/llm)\nerror: expected one of: `public`, `private`, `name`, `index`, `scheduled`\n --> src/lib.rs:4:28\n |\n4 | #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\n | ^^^^^^^^\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:17:21\n |\n17 | .insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0412]: cannot find type `TickTimer` in this scope\n --> src/lib.rs:25:42\n |\n25 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:14:15\n |\n14 | if ctx.db.tick_timer().count() == 0 {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:16:14\n |\n15 | / ctx.db\n16 | | .tick_timer()\n | | -^^^^^^^^^^ method not found in `Local`\n | |_____________|\n |\n\nerror[E0599]: no variant or associated item named `repeat_micros` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:19:43\n |\n19 | scheduled_at: ScheduleAt::repeat_micros(50_000),\n | ^^^^^^^^^^^^^ variant or associated item not found in `ScheduleAt`\n\nerror[E0277]: invalid reducer signature\n --> src/lib.rs:25:8\n |\n 24 | #[reducer]\n | ---------- required by a bound introduced by this call\n 25 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n | ^^^^ this reducer signature is not valid\n |\n = help: the trait `Reducer<'_, _>` is not implemented for fn item `for<'a> fn(&'a ReducerContext, {type error}) {tick}`\n = note: \n = note: reducer signatures must match the following pattern:\n = note: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\n = note: where each `Ti` type implements `SpacetimeType`.\n = note: \nnote: required by a bound in `register_reducer`\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/rt.rs:746:81\n |\n746 | pub fn register_reducer<'a, A: Args<'a>, I: FnInfo>(_: impl Reducer<'a, A>) {\n | ^^^^^^^^^^^^^^ required by this bound in `register_reducer`\n\nerror[E0277]: invalid reducer signature\n --> src/lib.rs:25:8\n |\n24 | #[reducer]\n | ---------- required by a bound introduced by this call\n25 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n | ^^^^ this reducer signature is not valid\n |\n = help: the trait `Reducer<'_, _>` is not implemented for fn item `for<'a> fn(&'a ReducerContext, {type error}) {tick}`\n = note: \n = note: reducer signatures must match the following pattern:\n = note: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\n = note: where each `Ti` type implements `SpacetimeType`.\n = note: \nnote: required by a bound in `invoke_reducer`\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/rt.rs:45:19\n |\n44 | pub fn invoke_reducer<'a, A: Args<'a>>(\n | -------------- required by a bound in this function\n45 | reducer: impl Reducer<'a, A>,\n | ^^^^^^^^^^^^^^ required by this bound in `invoke_reducer`\n\nSome errors have detailed explanations: E0277, E0412, E0422, E0599.\nFor more information about an error, try `rustc --explain E0277`.\nerror: could not compile `spacetime-module` (lib) due to 8 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n",
+ "phase": "build_or_publish"
+ }
+ }
+ },
+ "vendor": "openai",
+ "started_at": "2026-01-20T01:38:56.387361658Z",
+ "finished_at": "2026-01-20T01:40:02.304670766Z"
+ },
+ "t_018_constraints": {
+ "hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a",
+ "task": "t_018_constraints",
+ "lang": "rust",
+ "golden_published": true,
+ "model_name": "GPT-5",
+ "total_tests": 3,
+ "passed_tests": 1,
+ "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = account, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let account = ctx.db.account();\n\n for row in account.iter() {\n account.id().delete(&row.id);\n }\n\n account.insert(Account {\n id: 1,\n email: \"a@example.com\".to_string(),\n name: \"Alice\".to_string(),\n });\n account.insert(Account {\n id: 2,\n email: \"b@example.com\".to_string(),\n name: \"Bob\".to_string(),\n });\n}",
+ "category": "schema",
+ "route_api_model": "gpt-5",
+ "golden_db": "schema-t-018-constraints-golden",
+ "llm_db": "schema-t-018-constraints-gpt-5-llm",
+ "work_dir_golden": "target/llm-runs/schema/t_018_constraints/rust/server/golden",
+ "work_dir_llm": "target/llm-runs/schema/t_018_constraints/rust/server/gpt-5/llm",
+ "scorer_details": {
+ "constraints_seed_two_rows": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `accounts`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40169/v1/database/c200316727c132515dabe507a809977f0aa84c15be9e796686011760d2807e11/sql)\n",
+ "phase": "sql"
+ }
+ },
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "golden_db": "schema-t-018-constraints-golden",
+ "llm_db": "schema-t-018-constraints-gpt-5-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "http://127.0.0.1:40169",
+ "tables_diff": null,
+ "tables_equal": true
+ }
+ },
+ "constraints_row_parity_after_seed": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `accounts`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40169/v1/database/c2001a45039c1ff691843ec0135be6a2e72152318d1738beabafc5c35ac327c9/sql)\n",
+ "phase": "sql_golden"
+ }
+ }
+ },
+ "vendor": "openai",
+ "started_at": "2026-01-20T01:38:26.935618947Z",
+ "finished_at": "2026-01-20T01:38:58.896420052Z"
+ },
+ "t_019_many_to_many": {
+ "hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a",
+ "task": "t_019_many_to_many",
+ "lang": "rust",
+ "golden_published": true,
+ "model_name": "GPT-5",
+ "total_tests": 5,
+ "passed_tests": 1,
+ "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n user_id: i32,\n name: String,\n}\n\n#[table(name = group)]\npub struct Group {\n #[primary_key]\n group_id: i32,\n title: String,\n}\n\n#[table(\n name = membership,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id]))\n)]\npub struct Membership {\n #[primary_key]\n id: i32,\n user_id: i32,\n group_id: i32,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let mids: Vec = ctx.db.membership().iter().map(|r| r.id).collect();\n for id in mids {\n ctx.db.membership().id().delete(&id);\n }\n let uids: Vec = ctx.db.user().iter().map(|r| r.user_id).collect();\n for id in uids {\n ctx.db.user().user_id().delete(&id);\n }\n let gids: Vec = ctx.db.group().iter().map(|r| r.group_id).collect();\n for id in gids {\n ctx.db.group().group_id().delete(&id);\n }\n\n ctx.db.user().insert(User { user_id: 1, name: \"Alice\".to_string() });\n ctx.db.user().insert(User { user_id: 2, name: \"Bob\".to_string() });\n\n ctx.db.group().insert(Group { group_id: 10, title: \"Admin\".to_string() });\n ctx.db.group().insert(Group { group_id: 20, title: \"Dev\".to_string() });\n\n ctx.db.membership().insert(Membership { id: 1, user_id: 1, group_id: 10 });\n ctx.db.membership().insert(Membership { id: 2, user_id: 1, group_id: 20 });\n ctx.db.membership().insert(Membership { id: 3, user_id: 2, group_id: 20 });\n}",
+ "category": "schema",
+ "route_api_model": "gpt-5",
+ "golden_db": "schema-t-019-many-to-many-golden",
+ "llm_db": "schema-t-019-many-to-many-gpt-5-llm",
+ "work_dir_golden": "target/llm-runs/schema/t_019_many_to_many/rust/server/golden",
+ "work_dir_llm": "target/llm-runs/schema/t_019_many_to_many/rust/server/gpt-5/llm",
+ "scorer_details": {
+ "m2m_has_2_20": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `memberships`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40169/v1/database/c200c62151ddc609fd2694241da65fd3881fd74e6d2df7881db259e8bec11a59/sql)\n",
+ "phase": "sql"
+ }
+ },
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "golden_db": "schema-t-019-many-to-many-golden",
+ "llm_db": "schema-t-019-many-to-many-gpt-5-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "http://127.0.0.1:40169",
+ "tables_diff": null,
+ "tables_equal": true
+ }
+ },
+ "m2m_has_1_20": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `memberships`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40169/v1/database/c200c62151ddc609fd2694241da65fd3881fd74e6d2df7881db259e8bec11a59/sql)\n",
+ "phase": "sql"
+ }
+ },
+ "memberships_three_rows": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `memberships`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40169/v1/database/c200c62151ddc609fd2694241da65fd3881fd74e6d2df7881db259e8bec11a59/sql)\n",
+ "phase": "sql"
+ }
+ },
+ "m2m_has_1_10": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `memberships`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40169/v1/database/c200c62151ddc609fd2694241da65fd3881fd74e6d2df7881db259e8bec11a59/sql)\n",
+ "phase": "sql"
+ }
+ }
+ },
+ "vendor": "openai",
+ "started_at": "2026-01-20T01:38:51.848851935Z",
+ "finished_at": "2026-01-20T01:39:40.120841096Z"
+ },
+ "t_020_ecs": {
+ "hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a",
+ "task": "t_020_ecs",
+ "lang": "rust",
+ "golden_published": true,
+ "model_name": "GPT-5",
+ "total_tests": 5,
+ "passed_tests": 1,
+ "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[table(name = position)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let entities = ctx.db.entity();\n let positions = ctx.db.position();\n let velocities = ctx.db.velocity();\n\n if entities.id().find(1).is_none() {\n entities.insert(Entity { id: 1 });\n }\n if positions.entity_id().find(1).is_none() {\n positions.insert(Position { entity_id: 1, x: 0, y: 0 });\n }\n if velocities.entity_id().find(1).is_none() {\n velocities.insert(Velocity { entity_id: 1, vx: 1, vy: 0 });\n }\n\n if entities.id().find(2).is_none() {\n entities.insert(Entity { id: 2 });\n }\n if positions.entity_id().find(2).is_none() {\n positions.insert(Position { entity_id: 2, x: 10, y: 0 });\n }\n if velocities.entity_id().find(2).is_none() {\n velocities.insert(Velocity { entity_id: 2, vx: -2, vy: 3 });\n }\n}\n\n#[reducer]\npub fn step(ctx: &ReducerContext) {\n let positions = ctx.db.position();\n let velocities = ctx.db.velocity();\n let next_positions = ctx.db.next_position();\n\n for p in positions.iter() {\n if let Some(v) = velocities.entity_id().find(p.entity_id) {\n let nx = p.x + v.vx;\n let ny = p.y + v.vy;\n\n if let Some(mut existing) = next_positions.entity_id().find(p.entity_id) {\n existing.x = nx;\n existing.y = ny;\n let _ = next_positions.entity_id().update(existing);\n } else {\n next_positions.insert(NextPosition { entity_id: p.entity_id, x: nx, y: ny });\n }\n }\n }\n}",
+ "category": "schema",
+ "route_api_model": "gpt-5",
+ "golden_db": "schema-t-020-ecs-golden",
+ "llm_db": "schema-t-020-ecs-gpt-5-llm",
+ "work_dir_golden": "target/llm-runs/schema/t_020_ecs/rust/server/golden",
+ "work_dir_llm": "target/llm-runs/schema/t_020_ecs/rust/server/gpt-5/llm",
+ "scorer_details": {
+ "ecs_step_next_positions_count": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40169/v1/database/c200748fc346c54923110d3c56752ec88be85732e7215fbd3c5ec12ace636878/sql)\n",
+ "phase": "sql"
+ }
+ },
+ "ecs_seed_positions_count": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40169/v1/database/c200748fc346c54923110d3c56752ec88be85732e7215fbd3c5ec12ace636878/sql)\n",
+ "phase": "sql"
+ }
+ },
+ "ecs_next_pos_entity1": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40169/v1/database/c200748fc346c54923110d3c56752ec88be85732e7215fbd3c5ec12ace636878/sql)\n",
+ "phase": "sql"
+ }
+ },
+ "ecs_next_pos_entity2": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40169/v1/database/c200748fc346c54923110d3c56752ec88be85732e7215fbd3c5ec12ace636878/sql)\n",
+ "phase": "sql"
+ }
+ },
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "golden_db": "schema-t-020-ecs-golden",
+ "llm_db": "schema-t-020-ecs-gpt-5-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "http://127.0.0.1:40169",
+ "tables_diff": null,
+ "tables_equal": true
+ }
+ }
+ },
+ "vendor": "openai",
+ "started_at": "2026-01-20T01:38:23.204129823Z",
+ "finished_at": "2026-01-20T01:39:09.705047037Z"
+ },
+ "t_021_multi_column_index": {
+ "hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a",
+ "task": "t_021_multi_column_index",
+ "lang": "rust",
+ "golden_published": true,
+ "model_name": "GPT-5",
+ "total_tests": 4,
+ "passed_tests": 1,
+ "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = log, index(name = by_user_day, btree(columns = [user_id, day])))]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let log = ctx.db.log();\n log.insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".to_string() });\n log.insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".to_string() });\n log.insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".to_string() });\n}",
+ "category": "schema",
+ "route_api_model": "gpt-5",
+ "golden_db": "schema-t-021-multi-column-index-golden",
+ "llm_db": "schema-t-021-multi-column-index-gpt-5-llm",
+ "work_dir_golden": "target/llm-runs/schema/t_021_multi_column_index/rust/server/golden",
+ "work_dir_llm": "target/llm-runs/schema/t_021_multi_column_index/rust/server/gpt-5/llm",
+ "scorer_details": {
+ "mcindex_lookup_u7_d1": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `logs`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40169/v1/database/c2005e396a0d5b0adb357daac18c864e41f05d215322b2c123e10167603cbdb8/sql)\n",
+ "phase": "sql"
+ }
+ },
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "golden_db": "schema-t-021-multi-column-index-golden",
+ "llm_db": "schema-t-021-multi-column-index-gpt-5-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "http://127.0.0.1:40169",
+ "tables_diff": null,
+ "tables_equal": true
+ }
+ },
+ "mcindex_seed_count": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `logs`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40169/v1/database/c2005e396a0d5b0adb357daac18c864e41f05d215322b2c123e10167603cbdb8/sql)\n",
+ "phase": "sql"
+ }
+ },
+ "mcindex_lookup_u7_d2": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `logs`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40169/v1/database/c2005e396a0d5b0adb357daac18c864e41f05d215322b2c123e10167603cbdb8/sql)\n",
+ "phase": "sql"
+ }
+ }
+ },
+ "vendor": "openai",
+ "started_at": "2026-01-20T01:38:26.116614204Z",
+ "finished_at": "2026-01-20T01:38:56.387316141Z"
+ }
+ }
+ }
+ ]
+ },
+ {
+ "mode": "docs",
+ "hash": "1df24d7e0efd9d88ff9cc77fa5c951ac075acc22e1c3cd844adb37b87d805489",
+ "models": [
+ {
+ "name": "GPT-5",
+ "route_api_model": "gpt-5",
+ "tasks": {
+ "t_000_empty_reducers": {
+ "hash": "1df24d7e0efd9d88ff9cc77fa5c951ac075acc22e1c3cd844adb37b87d805489",
+ "task": "t_000_empty_reducers",
+ "lang": "rust",
+ "golden_published": true,
+ "model_name": "GPT-5",
+ "total_tests": 1,
+ "passed_tests": 0,
+ "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::reducer]\npub fn empty_reducer_no_args(_ctx: &ReducerContext) -> () {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_int(_ctx: &ReducerContext, _count: i32) -> () {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_string(_ctx: &ReducerContext, _name: String) -> () {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_two_args(_ctx: &ReducerContext, _count: i32, _name: String) -> () {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_three_args(_ctx: &ReducerContext, _active: bool, _ratio: f32, _label: String) -> () {\n}",
+ "category": "basics",
+ "route_api_model": "gpt-5",
+ "golden_db": "basics-t-000-empty-reducers-golden",
+ "llm_db": "basics-t-000-empty-reducers-gpt-5-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_000_empty_reducers\\rust\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_000_empty_reducers\\rust\\server\\gpt-5\\llm",
+ "scorer_details": {
+ "schema_parity": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-000-empty-reducers-golden`.\n",
+ "phase": "describe_golden"
+ }
+ }
+ },
+ "vendor": "openai",
+ "started_at": "2026-01-17T17:23:23.847332100Z",
+ "finished_at": "2026-01-17T17:24:32.531014600Z"
+ },
+ "t_001_basic_tables": {
+ "hash": "1df24d7e0efd9d88ff9cc77fa5c951ac075acc22e1c3cd844adb37b87d805489",
+ "task": "t_001_basic_tables",
+ "lang": "rust",
+ "golden_published": true,
+ "model_name": "GPT-5",
+ "total_tests": 1,
+ "passed_tests": 0,
+ "llm_output": "use spacetimedb::Table;\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = product)]\npub struct Product {\n #[primary_key]\n id: i32,\n title: String,\n price: f32,\n in_stock: bool,\n}\n\n#[spacetimedb::table(name = note)]\npub struct Note {\n #[primary_key]\n id: i32,\n body: String,\n rating: i64,\n pinned: bool,\n}",
+ "category": "basics",
+ "route_api_model": "gpt-5",
+ "golden_db": "basics-t-001-basic-tables-golden",
+ "llm_db": "basics-t-001-basic-tables-gpt-5-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_001_basic_tables\\rust\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_001_basic_tables\\rust\\server\\gpt-5\\llm",
+ "scorer_details": {
+ "schema_parity": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-001-basic-tables-golden`.\n",
+ "phase": "describe_golden"
+ }
+ }
+ },
+ "vendor": "openai",
+ "started_at": "2026-01-17T17:23:23.902249Z",
+ "finished_at": "2026-01-17T17:24:32.761372900Z"
+ },
+ "t_002_scheduled_table": {
+ "hash": "1df24d7e0efd9d88ff9cc77fa5c951ac075acc22e1c3cd844adb37b87d805489",
+ "task": "t_002_scheduled_table",
+ "lang": "rust",
+ "golden_published": true,
+ "model_name": "GPT-5",
+ "total_tests": 1,
+ "passed_tests": 0,
+ "llm_output": "use std::time::Duration;\nuse spacetimedb::{ReducerContext, ScheduleAt, Table};\n\n#[spacetimedb::table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) -> Result<(), String> {\n Ok(())\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) -> Result<(), String> {\n if ctx.db.tick_timer().count() == 0 {\n ctx.db.tick_timer().try_insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::Interval(Duration::from_micros(50_000).into()),\n })?;\n }\n Ok(())\n}",
+ "category": "basics",
+ "route_api_model": "gpt-5",
+ "golden_db": "basics-t-002-scheduled-table-golden",
+ "llm_db": "basics-t-002-scheduled-table-gpt-5-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_002_scheduled_table\\rust\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_002_scheduled_table\\rust\\server\\gpt-5\\llm",
+ "scorer_details": {
+ "schema_parity": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-002-scheduled-table-golden`.\n",
+ "phase": "describe_golden"
+ }
+ }
+ },
+ "vendor": "openai",
+ "started_at": "2026-01-17T17:23:23.953682500Z",
+ "finished_at": "2026-01-17T17:24:33.012912700Z"
+ },
+ "t_003_struct_in_table": {
+ "hash": "1df24d7e0efd9d88ff9cc77fa5c951ac075acc22e1c3cd844adb37b87d805489",
+ "task": "t_003_struct_in_table",
+ "lang": "rust",
+ "golden_published": true,
+ "model_name": "GPT-5",
+ "total_tests": 1,
+ "passed_tests": 0,
+ "llm_output": "#[derive(spacetimedb::SpacetimeType, Clone, Debug)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n pos: Position,\n}",
+ "category": "basics",
+ "route_api_model": "gpt-5",
+ "golden_db": "basics-t-003-struct-in-table-golden",
+ "llm_db": "basics-t-003-struct-in-table-gpt-5-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_003_struct_in_table\\rust\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_003_struct_in_table\\rust\\server\\gpt-5\\llm",
+ "scorer_details": {
+ "schema_parity": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-003-struct-in-table-golden`.\n",
+ "phase": "describe_golden"
+ }
+ }
+ },
+ "vendor": "openai",
+ "started_at": "2026-01-17T17:23:24.009687400Z",
+ "finished_at": "2026-01-17T17:24:32.595062100Z"
+ },
+ "t_004_insert": {
+ "hash": "1df24d7e0efd9d88ff9cc77fa5c951ac075acc22e1c3cd844adb37b87d805489",
+ "task": "t_004_insert",
+ "lang": "rust",
+ "golden_published": true,
+ "model_name": "GPT-5",
+ "total_tests": 2,
+ "passed_tests": 0,
+ "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn insert_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n ctx.db.user().insert(User { id, name, age, active });\n}",
+ "category": "basics",
+ "route_api_model": "gpt-5",
+ "golden_db": "basics-t-004-insert-golden",
+ "llm_db": "basics-t-004-insert-gpt-5-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_004_insert\\rust\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_004_insert\\rust\\server\\gpt-5\\llm",
+ "scorer_details": {
+ "schema_parity": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-004-insert-golden`.\n",
+ "phase": "describe_golden"
+ }
+ },
+ "data_parity_insert_user": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-004-insert-golden`.\n",
+ "phase": "call_reducer_golden"
+ }
+ }
+ },
+ "vendor": "openai",
+ "started_at": "2026-01-17T17:23:24.061380200Z",
+ "finished_at": "2026-01-17T17:24:32.822537300Z"
+ },
+ "t_005_update": {
+ "hash": "1df24d7e0efd9d88ff9cc77fa5c951ac075acc22e1c3cd844adb37b87d805489",
+ "task": "t_005_update",
+ "lang": "rust",
+ "golden_published": true,
+ "model_name": "GPT-5",
+ "total_tests": 3,
+ "passed_tests": 0,
+ "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n if ctx.db.user().id().find(id).is_some() {\n ctx.db.user().id().update(User { id, name, age, active });\n }\n}",
+ "category": "basics",
+ "route_api_model": "gpt-5",
+ "golden_db": "basics-t-005-update-golden",
+ "llm_db": "basics-t-005-update-gpt-5-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_005_update\\rust\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_005_update\\rust\\server\\gpt-5\\llm",
+ "scorer_details": {
+ "schema_parity": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-005-update-golden`.\n",
+ "phase": "describe_golden"
+ }
+ },
+ "seed_users_row": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-005-update-golden`.\n",
+ "phase": "sql_golden",
+ "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)"
+ }
+ },
+ "data_parity_update_user": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-005-update-golden`.\n",
+ "phase": "call_reducer_golden"
+ }
+ }
+ },
+ "vendor": "openai",
+ "started_at": "2026-01-17T17:23:24.116749100Z",
+ "finished_at": "2026-01-17T17:24:33.107785500Z"
+ },
+ "t_006_delete": {
+ "hash": "1df24d7e0efd9d88ff9cc77fa5c951ac075acc22e1c3cd844adb37b87d805489",
+ "task": "t_006_delete",
+ "lang": "rust",
+ "golden_published": true,
+ "model_name": "GPT-5",
+ "total_tests": 3,
+ "passed_tests": 1,
+ "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn delete_user(ctx: &ReducerContext, id: i32) {\n ctx.db.user().id().delete(id);\n}",
+ "category": "basics",
+ "route_api_model": "gpt-5",
+ "golden_db": "basics-t-006-delete-golden",
+ "llm_db": "basics-t-006-delete-gpt-5-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_006_delete\\rust\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_006_delete\\rust\\server\\gpt-5\\llm",
+ "scorer_details": {
+ "schema_parity": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-006-delete-golden`.\n",
+ "phase": "describe_golden"
+ }
+ },
+ "seed_users_row": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-006-delete-golden`.\n",
+ "phase": "sql_golden",
+ "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)"
+ }
+ },
+ "delete_user_count_zero": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 0,
+ "expected": 0,
+ "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1"
+ }
+ }
+ },
+ "vendor": "openai",
+ "started_at": "2026-01-17T17:23:24.171770800Z",
+ "finished_at": "2026-01-17T17:24:32.728374300Z"
+ },
+ "t_007_crud": {
+ "hash": "1df24d7e0efd9d88ff9cc77fa5c951ac075acc22e1c3cd844adb37b87d805489",
+ "task": "t_007_crud",
+ "lang": "rust",
+ "golden_published": true,
+ "model_name": "GPT-5",
+ "total_tests": 4,
+ "passed_tests": 1,
+ "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn crud(ctx: &ReducerContext) {\n ctx.db.user().insert(User { id: 1, name: \"Alice\".into(), age: 30, active: true });\n ctx.db.user().insert(User { id: 2, name: \"Bob\".into(), age: 22, active: false });\n if let Some(mut u) = ctx.db.user().id().find(1) {\n u.name = \"Alice2\".into();\n u.age = 31;\n u.active = false;\n ctx.db.user().id().update(u);\n }\n ctx.db.user().id().delete(2);\n}",
+ "category": "basics",
+ "route_api_model": "gpt-5",
+ "golden_db": "basics-t-007-crud-golden",
+ "llm_db": "basics-t-007-crud-gpt-5-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_007_crud\\rust\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_007_crud\\rust\\server\\gpt-5\\llm",
+ "scorer_details": {
+ "crud_total_count_one": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "actual": 0,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM user"
+ }
+ },
+ "crud_row_id2_deleted": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 0,
+ "expected": 0,
+ "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2"
+ }
+ },
+ "crud_row_id1_parity": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-007-crud-golden`.\n",
+ "phase": "call_reducer_golden"
+ }
+ },
+ "schema_parity": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-007-crud-golden`.\n",
+ "phase": "describe_golden"
+ }
+ }
+ },
+ "vendor": "openai",
+ "started_at": "2026-01-17T17:23:24.224357900Z",
+ "finished_at": "2026-01-17T17:24:32.979731500Z"
+ },
+ "t_008_index_lookup": {
+ "hash": "1df24d7e0efd9d88ff9cc77fa5c951ac075acc22e1c3cd844adb37b87d805489",
+ "task": "t_008_index_lookup",
+ "lang": "rust",
+ "golden_published": true,
+ "model_name": "GPT-5",
+ "total_tests": 3,
+ "passed_tests": 0,
+ "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(user) = ctx.db.user().id().find(id) {\n ctx.db.result().insert(ResultRow { id: user.id, name: user.name });\n }\n}",
+ "category": "basics",
+ "route_api_model": "gpt-5",
+ "golden_db": "basics-t-008-index-lookup-golden",
+ "llm_db": "basics-t-008-index-lookup-gpt-5-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_008_index_lookup\\rust\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_008_index_lookup\\rust\\server\\gpt-5\\llm",
+ "scorer_details": {
+ "index_lookup_projection_parity": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-008-index-lookup-golden`.\n",
+ "phase": "call_reducer_golden"
+ }
+ },
+ "schema_parity": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-008-index-lookup-golden`.\n",
+ "phase": "describe_golden"
+ }
+ },
+ "seed_user_row": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-008-index-lookup-golden`.\n",
+ "phase": "sql_golden",
+ "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)"
+ }
+ }
+ },
+ "vendor": "openai",
+ "started_at": "2026-01-17T17:24:33.107958900Z",
+ "finished_at": "2026-01-17T17:25:30.626814500Z"
+ },
+ "t_009_init": {
+ "hash": "1df24d7e0efd9d88ff9cc77fa5c951ac075acc22e1c3cd844adb37b87d805489",
+ "task": "t_009_init",
+ "lang": "rust",
+ "golden_published": true,
+ "model_name": "GPT-5",
+ "total_tests": 4,
+ "passed_tests": 3,
+ "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) -> Result<(), String> {\n ctx.db.user().try_insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true })?;\n ctx.db.user().try_insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false })?;\n Ok(())\n}",
+ "category": "basics",
+ "route_api_model": "gpt-5",
+ "golden_db": "basics-t-009-init-golden",
+ "llm_db": "basics-t-009-init-gpt-5-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_009_init\\rust\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_009_init\\rust\\server\\gpt-5\\llm",
+ "scorer_details": {
+ "init_total_two": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 2,
+ "expected": 2,
+ "sql": "SELECT COUNT(*) AS n FROM user"
+ }
+ },
+ "schema_parity": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-009-init-golden`.\n",
+ "phase": "describe_golden"
+ }
+ },
+ "init_seed_alice": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true"
+ }
+ },
+ "init_seed_bob": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false"
+ }
+ }
+ },
+ "vendor": "openai",
+ "started_at": "2026-01-17T17:24:33.164389600Z",
+ "finished_at": "2026-01-17T17:25:31.181189800Z"
+ },
+ "t_010_connect": {
+ "hash": "1df24d7e0efd9d88ff9cc77fa5c951ac075acc22e1c3cd844adb37b87d805489",
+ "task": "t_010_connect",
+ "lang": "rust",
+ "golden_published": true,
+ "model_name": "GPT-5",
+ "total_tests": 1,
+ "passed_tests": 0,
+ "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = events)]\npub struct Event {\n #[primary_key]\n #[auto_inc]\n id: i32,\n kind: String,\n}\n\n#[spacetimedb::reducer(client_connected)]\npub fn client_connected(ctx: &ReducerContext) {\n ctx.db.events().insert(Event {\n id: 0,\n kind: \"connected\".into(),\n });\n}\n\n#[spacetimedb::reducer(client_disconnected)]\npub fn client_disconnected(ctx: &ReducerContext) {\n ctx.db.events().insert(Event {\n id: 0,\n kind: \"disconnected\".into(),\n });\n}",
+ "category": "basics",
+ "route_api_model": "gpt-5",
+ "golden_db": "basics-t-010-connect-golden",
+ "llm_db": "basics-t-010-connect-gpt-5-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_010_connect\\rust\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_010_connect\\rust\\server\\gpt-5\\llm",
+ "scorer_details": {
+ "schema_parity": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-010-connect-golden`.\n",
+ "phase": "describe_golden"
+ }
+ }
+ },
+ "vendor": "openai",
+ "started_at": "2026-01-17T17:24:33.220990900Z",
+ "finished_at": "2026-01-17T17:25:30.501080300Z"
+ },
+ "t_011_helper_function": {
+ "hash": "1df24d7e0efd9d88ff9cc77fa5c951ac075acc22e1c3cd844adb37b87d805489",
+ "task": "t_011_helper_function",
+ "lang": "rust",
+ "golden_published": true,
+ "model_name": "GPT-5",
+ "total_tests": 3,
+ "passed_tests": 0,
+ "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n sum: i32,\n}\n\nfn add(a: i32, b: i32) -> i32 {\n a + b\n}\n\n#[spacetimedb::reducer]\npub fn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) {\n ctx.db.result().insert(ResultRow { id, sum: add(a, b) });\n}",
+ "category": "basics",
+ "route_api_model": "gpt-5",
+ "golden_db": "basics-t-011-helper-function-golden",
+ "llm_db": "basics-t-011-helper-function-gpt-5-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_011_helper_function\\rust\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_011_helper_function\\rust\\server\\gpt-5\\llm",
+ "scorer_details": {
+ "helper_func_sum_abs": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "actual": 0,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1 AND sum=5"
+ }
+ },
+ "helper_func_sum_parity": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-011-helper-function-golden`.\n",
+ "phase": "call_reducer_golden"
+ }
+ },
+ "schema_parity": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-011-helper-function-golden`.\n",
+ "phase": "describe_golden"
+ }
+ }
+ },
+ "vendor": "openai",
+ "started_at": "2026-01-17T17:24:33.278357700Z",
+ "finished_at": "2026-01-17T17:25:18.585097100Z"
+ },
+ "t_012_spacetime_product_type": {
+ "hash": "1df24d7e0efd9d88ff9cc77fa5c951ac075acc22e1c3cd844adb37b87d805489",
+ "task": "t_012_spacetime_product_type",
+ "lang": "rust",
+ "golden_published": true,
+ "model_name": "GPT-5",
+ "total_tests": 3,
+ "passed_tests": 0,
+ "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Score {\n left: i32,\n right: i32,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Score,\n}\n\n#[spacetimedb::reducer]\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Score { left, right },\n });\n}",
+ "category": "schema",
+ "route_api_model": "gpt-5",
+ "golden_db": "schema-t-012-spacetime-product-type-golden",
+ "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_012_spacetime_product_type\\rust\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_012_spacetime_product_type\\rust\\server\\gpt-5\\llm",
+ "scorer_details": {
+ "product_type_row_count": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "actual": 0,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1"
+ }
+ },
+ "schema_parity": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-012-spacetime-product-type-golden`.\n",
+ "phase": "describe_golden"
+ }
+ },
+ "product_type_row_parity": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-012-spacetime-product-type-golden`.\n",
+ "phase": "call_reducer_golden"
+ }
+ }
+ },
+ "vendor": "openai",
+ "started_at": "2026-01-17T17:24:33.335528Z",
+ "finished_at": "2026-01-17T17:25:31.048649800Z"
+ },
+ "t_013_spacetime_sum_type": {
+ "hash": "1df24d7e0efd9d88ff9cc77fa5c951ac075acc22e1c3cd844adb37b87d805489",
+ "task": "t_013_spacetime_sum_type",
+ "lang": "rust",
+ "golden_published": true,
+ "model_name": "GPT-5",
+ "total_tests": 3,
+ "passed_tests": 0,
+ "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType, Clone, Debug)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Shape,\n}\n\n#[spacetimedb::reducer]\npub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Shape::Circle(radius),\n });\n}",
+ "category": "schema",
+ "route_api_model": "gpt-5",
+ "golden_db": "schema-t-013-spacetime-sum-type-golden",
+ "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_013_spacetime_sum_type\\rust\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_013_spacetime_sum_type\\rust\\server\\gpt-5\\llm",
+ "scorer_details": {
+ "sum_type_row_parity": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-013-spacetime-sum-type-golden`.\n",
+ "phase": "call_reducer_golden"
+ }
+ },
+ "sum_type_row_count": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "actual": 0,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1"
+ }
+ },
+ "schema_parity": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-013-spacetime-sum-type-golden`.\n",
+ "phase": "describe_golden"
+ }
+ }
+ },
+ "vendor": "openai",
+ "started_at": "2026-01-17T17:24:33.393640100Z",
+ "finished_at": "2026-01-17T17:25:30.952675100Z"
+ },
+ "t_014_elementary_columns": {
+ "hash": "1df24d7e0efd9d88ff9cc77fa5c951ac075acc22e1c3cd844adb37b87d805489",
+ "task": "t_014_elementary_columns",
+ "lang": "rust",
+ "golden_published": true,
+ "model_name": "GPT-5",
+ "total_tests": 3,
+ "passed_tests": 0,
+ "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = primitives, public)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitives().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000,\n price: 1.5f32,\n ratio: 2.25f64,\n active: true,\n name: \"Alice\".to_string(),\n });\n}",
+ "category": "schema",
+ "route_api_model": "gpt-5",
+ "golden_db": "schema-t-014-elementary-columns-golden",
+ "llm_db": "schema-t-014-elementary-columns-gpt-5-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_014_elementary_columns\\rust\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_014_elementary_columns\\rust\\server\\gpt-5\\llm",
+ "scorer_details": {
+ "elementary_columns_row_count": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "actual": 0,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM primitives WHERE id=1"
}
},
"elementary_columns_row_parity": {
- "pass": true,
- "partial": 1.0,
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "args": [],
- "golden_db": "schema-t-014-elementary-columns-golden",
- "golden_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"",
- "llm_db": "schema-t-014-elementary-columns-gpt-5-llm",
- "llm_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"",
- "query": "SELECT id, count, total, price, ratio, active, name FROM primitives WHERE id=1",
- "reducer": "seed",
- "server": "http://127.0.0.1:46587"
+ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-014-elementary-columns-golden`.\n",
+ "phase": "call_reducer_golden"
+ }
+ },
+ "schema_parity": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-014-elementary-columns-golden`.\n",
+ "phase": "describe_golden"
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:25:38.992406952Z",
- "finished_at": "2026-01-17T02:28:06.754249465Z"
+ "started_at": "2026-01-17T17:24:33.447981400Z",
+ "finished_at": "2026-01-17T17:25:30.756400Z"
},
"t_015_product_type_columns": {
- "hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a",
+ "hash": "1df24d7e0efd9d88ff9cc77fa5c951ac075acc22e1c3cd844adb37b87d805489",
"task": "t_015_product_type_columns",
"lang": "rust",
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 3,
- "passed_tests": 2,
- "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType, Clone)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType, Clone)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[table(name = profiles)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let home = Address { street: \"1 Main\".to_string(), zip: 11111 };\n let work = Address { street: \"2 Broad\".to_string(), zip: 22222 };\n let pos = Position { x: 7, y: 9 };\n ctx.db.profiles().insert(Profile { id: 1, home, work, pos });\n}",
+ "passed_tests": 0,
+ "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = profile)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.profile().insert(Profile {\n id: 1,\n home: Address {\n street: \"1 Main\".to_string(),\n zip: 11111,\n },\n work: Address {\n street: \"2 Broad\".to_string(),\n zip: 22222,\n },\n pos: Position { x: 7, y: 9 },\n });\n}",
"category": "schema",
"route_api_model": "gpt-5",
"golden_db": "schema-t-015-product-type-columns-golden",
"llm_db": "schema-t-015-product-type-columns-gpt-5-llm",
- "work_dir_golden": "target/llm-runs/schema/t_015_product_type_columns/rust/server/golden",
- "work_dir_llm": "target/llm-runs/schema/t_015_product_type_columns/rust/server/gpt-5/llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_015_product_type_columns\\rust\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_015_product_type_columns\\rust\\server\\gpt-5\\llm",
"scorer_details": {
- "product_type_columns_row_count": {
- "pass": true,
- "partial": 1.0,
+ "product_type_columns_row_parity": {
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM profiles WHERE id=1"
+ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-015-product-type-columns-golden`.\n",
+ "phase": "call_reducer_golden"
}
},
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
+ "product_type_columns_row_count": {
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "golden_db": "schema-t-015-product-type-columns-golden",
- "llm_db": "schema-t-015-product-type-columns-gpt-5-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "http://127.0.0.1:46587",
- "tables_diff": null,
- "tables_equal": true
+ "actual": 0,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM profile WHERE id=1"
}
},
- "product_type_columns_row_parity": {
+ "schema_parity": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `profiles`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46587/v1/database/c200f79f71d0434e7e09b201e09674fe9fc25e018d5ba8eae68ccedefeafa4b2/sql)\n",
- "phase": "sql_golden"
+ "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-015-product-type-columns-golden`.\n",
+ "phase": "describe_golden"
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:25:38.993007169Z",
- "finished_at": "2026-01-17T02:28:12.198565927Z"
+ "started_at": "2026-01-17T17:24:33.504015500Z",
+ "finished_at": "2026-01-17T17:25:30.857580700Z"
},
"t_016_sum_type_columns": {
- "hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a",
+ "hash": "1df24d7e0efd9d88ff9cc77fa5c951ac075acc22e1c3cd844adb37b87d805489",
"task": "t_016_sum_type_columns",
"lang": "rust",
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 3,
- "passed_tests": 3,
- "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = drawings)]\npub struct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let drawings = ctx.db.drawings();\n if drawings.count() == 0 {\n drawings.insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n }\n}",
+ "passed_tests": 0,
+ "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(name = drawings)]\npub struct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.drawings().insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n}",
"category": "schema",
"route_api_model": "gpt-5",
"golden_db": "schema-t-016-sum-type-columns-golden",
"llm_db": "schema-t-016-sum-type-columns-gpt-5-llm",
- "work_dir_golden": "target/llm-runs/schema/t_016_sum_type_columns/rust/server/golden",
- "work_dir_llm": "target/llm-runs/schema/t_016_sum_type_columns/rust/server/gpt-5/llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_016_sum_type_columns\\rust\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_016_sum_type_columns\\rust\\server\\gpt-5\\llm",
"scorer_details": {
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
+ "sum_type_columns_row_parity": {
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "golden_db": "schema-t-016-sum-type-columns-golden",
- "llm_db": "schema-t-016-sum-type-columns-gpt-5-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "http://127.0.0.1:46587",
- "tables_diff": null,
- "tables_equal": true
+ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-016-sum-type-columns-golden`.\n",
+ "phase": "call_reducer_golden"
}
},
- "sum_type_columns_row_parity": {
- "pass": true,
- "partial": 1.0,
+ "schema_parity": {
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "args": [],
- "golden_db": "schema-t-016-sum-type-columns-golden",
- "golden_out": "id | a | b ----+---------------+--------------------------------------- 1 | (Circle = 10) | (Rectangle = (width = 4, height = 6))",
- "llm_db": "schema-t-016-sum-type-columns-gpt-5-llm",
- "llm_out": "id | a | b ----+---------------+--------------------------------------- 1 | (Circle = 10) | (Rectangle = (width = 4, height = 6))",
- "query": "SELECT id, a, b FROM drawings WHERE id=1",
- "reducer": "seed",
- "server": "http://127.0.0.1:46587"
+ "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-016-sum-type-columns-golden`.\n",
+ "phase": "describe_golden"
}
},
"sum_type_columns_row_count": {
- "pass": true,
- "partial": 1.0,
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "actual": 1,
+ "actual": 0,
"expected": 1,
"sql": "SELECT COUNT(*) AS n FROM drawings WHERE id=1"
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:28:06.754312132Z",
- "finished_at": "2026-01-17T02:30:52.376483288Z"
+ "started_at": "2026-01-17T17:25:18.585391300Z",
+ "finished_at": "2026-01-17T17:25:50.255623200Z"
},
"t_017_scheduled_columns": {
- "hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a",
+ "hash": "1df24d7e0efd9d88ff9cc77fa5c951ac075acc22e1c3cd844adb37b87d805489",
"task": "t_017_scheduled_columns",
"lang": "rust",
- "golden_published": false,
+ "golden_published": true,
"model_name": "GPT-5",
"total_tests": 2,
- "passed_tests": 0,
- "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt};\n\n#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n if ctx.db.tick_timer().count() == 0 {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::RepeatMicros(50_000),\n });\n }\n}\n\n#[reducer]\npub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n // Scheduled reducer invoked with the scheduled row.\n}",
+ "passed_tests": 1,
+ "llm_output": "use std::time::Duration;\nuse spacetimedb::{ReducerContext, ScheduleAt, Table};\n\n#[spacetimedb::table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _row: TickTimer) -> Result<(), String> {\n Ok(())\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) -> Result<(), String> {\n ctx.db.tick_timer().try_insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::Interval(Duration::from_micros(50_000).into()),\n })?;\n Ok(())\n}",
"category": "schema",
"route_api_model": "gpt-5",
"golden_db": "schema-t-017-scheduled-columns-golden",
"llm_db": "schema-t-017-scheduled-columns-gpt-5-llm",
- "work_dir_golden": "target/llm-runs/schema/t_017_scheduled_columns/rust/server/golden",
- "work_dir_llm": "target/llm-runs/schema/t_017_scheduled_columns/rust/server/gpt-5/llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_017_scheduled_columns\\rust\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_017_scheduled_columns\\rust\\server\\gpt-5\\llm",
"scorer_details": {
- "publish_error": {
+ "schema_parity": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime publish failed (exit=1)\n--- stderr ---\n\u001b[1m\u001b[32m Updating\u001b[0m crates.io index\n\u001b[1m\u001b[32m Locking\u001b[0m 72 packages to latest compatible versions\n\u001b[1m\u001b[36m Adding\u001b[0m generic-array v0.14.7 \u001b[1m\u001b[33m(available: v0.14.9)\u001b[0m\n\u001b[1m\u001b[36m Adding\u001b[0m spacetimedb v1.11.1 \u001b[1m\u001b[33m(available: v1.11.3)\u001b[0m\n\u001b[1m\u001b[36m Adding\u001b[0m spacetimedb-bindings-macro v1.11.1 \u001b[1m\u001b[33m(available: v1.11.3)\u001b[0m\n\u001b[1m\u001b[36m Adding\u001b[0m spacetimedb-bindings-sys v1.11.1 \u001b[1m\u001b[33m(available: v1.11.3)\u001b[0m\n\u001b[1m\u001b[36m Adding\u001b[0m spacetimedb-lib v1.11.1 \u001b[1m\u001b[33m(available: v1.11.3)\u001b[0m\n\u001b[1m\u001b[36m Adding\u001b[0m spacetimedb-primitives v1.11.1 \u001b[1m\u001b[33m(available: v1.11.3)\u001b[0m\n\u001b[1m\u001b[36m Adding\u001b[0m spacetimedb-sats v1.11.1 \u001b[1m\u001b[33m(available: v1.11.3)\u001b[0m\n\u001b[1m\u001b[32m Compiling\u001b[0m proc-macro2 v1.0.105\n\u001b[1m\u001b[32m Compiling\u001b[0m quote v1.0.43\n\u001b[1m\u001b[32m Compiling\u001b[0m unicode-ident v1.0.22\n\u001b[1m\u001b[32m Compiling\u001b[0m version_check v0.9.5\n\u001b[1m\u001b[32m Compiling\u001b[0m typenum v1.19.0\n\u001b[1m\u001b[32m Compiling\u001b[0m autocfg v1.5.0\n\u001b[1m\u001b[32m Compiling\u001b[0m generic-array v0.14.7\n\u001b[1m\u001b[32m Compiling\u001b[0m heck v0.5.0\n\u001b[1m\u001b[32m Compiling\u001b[0m num-traits v0.2.19\n\u001b[1m\u001b[32m Compiling\u001b[0m serde_core v1.0.228\n\u001b[1m\u001b[32m Compiling\u001b[0m cfg-if v1.0.4\n\u001b[1m\u001b[32m Compiling\u001b[0m syn v2.0.114\n\u001b[1m\u001b[32m Compiling\u001b[0m shlex v1.3.0\n\u001b[1m\u001b[32m Compiling\u001b[0m serde v1.0.228\n\u001b[1m\u001b[32m Compiling\u001b[0m find-msvc-tools v0.1.8\n\u001b[1m\u001b[32m Compiling\u001b[0m zerocopy v0.8.33\n\u001b[1m\u001b[32m Compiling\u001b[0m either v1.15.0\n\u001b[1m\u001b[32m Compiling\u001b[0m itertools v0.12.1\n\u001b[1m\u001b[32m Compiling\u001b[0m cc v1.2.53\n\u001b[1m\u001b[32m Compiling\u001b[0m crypto-common v0.1.7\n\u001b[1m\u001b[32m Compiling\u001b[0m block-buffer v0.10.4\n\u001b[1m\u001b[32m Compiling\u001b[0m nohash-hasher v0.2.0\n\u001b[1m\u001b[32m Compiling\u001b[0m thiserror v1.0.69\n\u001b[1m\u001b[32m Compiling\u001b[0m bitflags v2.10.0\n\u001b[1m\u001b[32m Compiling\u001b[0m anyhow v1.0.100\n\u001b[1m\u001b[32m Compiling\u001b[0m digest v0.10.7\n\u001b[1m\u001b[32m Compiling\u001b[0m approx v0.3.2\n\u001b[1m\u001b[32m Compiling\u001b[0m getrandom v0.2.17\n\u001b[1m\u001b[32m Compiling\u001b[0m heck v0.4.1\n\u001b[1m\u001b[32m Compiling\u001b[0m humantime v2.3.0\n\u001b[1m\u001b[32m Compiling\u001b[0m blake3 v1.8.3\n\u001b[1m\u001b[32m Compiling\u001b[0m bytes v1.11.0\n\u001b[1m\u001b[32m Compiling\u001b[0m keccak v0.1.5\n\u001b[1m\u001b[32m Compiling\u001b[0m zmij v1.0.14\n\u001b[1m\u001b[32m Compiling\u001b[0m arrayvec v0.7.6\n\u001b[1m\u001b[32m Compiling\u001b[0m convert_case v0.4.0\n\u001b[1m\u001b[32m Compiling\u001b[0m sha3 v0.10.8\n\u001b[1m\u001b[32m Compiling\u001b[0m rand_core v0.6.4\n\u001b[1m\u001b[32m Compiling\u001b[0m enum-as-inner v0.6.1\n\u001b[1m\u001b[32m Compiling\u001b[0m thiserror-impl v1.0.69\n\u001b[1m\u001b[32m Compiling\u001b[0m ppv-lite86 v0.2.21\n\u001b[1m\u001b[32m Compiling\u001b[0m derive_more v0.99.20\n\u001b[1m\u001b[32m Compiling\u001b[0m spacetimedb-primitives v1.11.1\n\u001b[1m\u001b[32m Compiling\u001b[0m spacetimedb-bindings-macro v1.11.1\n\u001b[1m\u001b[32m Compiling\u001b[0m decorum v0.3.1\n\u001b[1m\u001b[32m Compiling\u001b[0m ethnum v1.5.2\n\u001b[1m\u001b[32m Compiling\u001b[0m chrono v0.4.43\n\u001b[1m\u001b[32m Compiling\u001b[0m bytemuck v1.24.0\n\u001b[1m\u001b[32m Compiling\u001b[0m arrayref v0.3.9\n\u001b[1m\u001b[32m Compiling\u001b[0m hex v0.4.3\n\u001b[1m\u001b[32m Compiling\u001b[0m smallvec v1.15.1\n\u001b[1m\u001b[32m Compiling\u001b[0m constant_time_eq v0.4.2\n\u001b[1m\u001b[32m Compiling\u001b[0m serde_json v1.0.149\n\u001b[1m\u001b[32m Compiling\u001b[0m second-stack v0.3.5\n\u001b[1m\u001b[32m Compiling\u001b[0m itoa v1.0.17\n\u001b[1m\u001b[32m Compiling\u001b[0m spacetimedb-lib v1.11.1\n\u001b[1m\u001b[32m Compiling\u001b[0m rand_chacha v0.3.1\n\u001b[1m\u001b[32m Compiling\u001b[0m log v0.4.29\n\u001b[1m\u001b[32m Compiling\u001b[0m spacetimedb-sats v1.11.1\n\u001b[1m\u001b[32m Compiling\u001b[0m memchr v2.7.6\n\u001b[1m\u001b[32m Compiling\u001b[0m rand v0.8.5\n\u001b[1m\u001b[32m Compiling\u001b[0m http v1.4.0\n\u001b[1m\u001b[32m Compiling\u001b[0m spacetimedb-bindings-sys v1.11.1\n\u001b[1m\u001b[32m Compiling\u001b[0m scoped-tls v1.0.1\n\u001b[1m\u001b[32m Compiling\u001b[0m spacetimedb v1.11.1\n\u001b[1m\u001b[32m Compiling\u001b[0m spacetime-module v0.1.0 (/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_017_scheduled_columns/rust/server/gpt-5/llm)\n\u001b[0m\u001b[1m\u001b[38;5;9merror\u001b[0m\u001b[0m\u001b[1m: expected one of: `public`, `private`, `name`, `index`, `scheduled`\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--> \u001b[0m\u001b[0msrc/lib.rs:4:28\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m4\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9m^^^^^^^^\u001b[0m\n\n\u001b[0m\u001b[1m\u001b[38;5;9merror[E0422]\u001b[0m\u001b[0m\u001b[1m: cannot find struct, variant or union type `TickTimer` in this scope\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--> \u001b[0m\u001b[0msrc/lib.rs:15:36\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m15\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m ctx.db.tick_timer().insert(TickTimer {\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9m^^^^^^^^^\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9mnot found in this scope\u001b[0m\n\n\u001b[0m\u001b[1m\u001b[38;5;9merror[E0412]\u001b[0m\u001b[0m\u001b[1m: cannot find type `TickTimer` in this scope\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--> \u001b[0m\u001b[0msrc/lib.rs:23:42\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m23\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0mpub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9m^^^^^^^^^\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9mnot found in this scope\u001b[0m\n\n\u001b[0m\u001b[1m\u001b[38;5;9merror[E0599]\u001b[0m\u001b[0m\u001b[1m: no method named `tick_timer` found for struct `Local` in the current scope\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--> \u001b[0m\u001b[0msrc/lib.rs:14:15\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m14\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m if ctx.db.tick_timer().count() == 0 {\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9m^^^^^^^^^^\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9mmethod not found in `Local`\u001b[0m\n\n\u001b[0m\u001b[1m\u001b[38;5;9merror[E0599]\u001b[0m\u001b[0m\u001b[1m: no method named `tick_timer` found for struct `Local` in the current scope\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--> \u001b[0m\u001b[0msrc/lib.rs:15:16\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m15\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m ctx.db.tick_timer().insert(TickTimer {\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9m^^^^^^^^^^\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9mmethod not found in `Local`\u001b[0m\n\n\u001b[0m\u001b[1m\u001b[38;5;9merror[E0599]\u001b[0m\u001b[0m\u001b[1m: no variant or associated item named `RepeatMicros` found for enum `ScheduleAt` in the current scope\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--> \u001b[0m\u001b[0msrc/lib.rs:17:39\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m17\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m scheduled_at: ScheduleAt::RepeatMicros(50_000),\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9m^^^^^^^^^^^^\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9mvariant or associated item not found in `ScheduleAt`\u001b[0m\n\n\u001b[0m\u001b[1m\u001b[38;5;9merror[E0277]\u001b[0m\u001b[0m\u001b[1m: invalid reducer signature\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--> \u001b[0m\u001b[0msrc/lib.rs:23:8\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m22\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m#[reducer]\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m----------\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12mrequired by a bound introduced by this call\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m23\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0mpub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9m^^^^\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9mthis reducer signature is not valid\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m= \u001b[0m\u001b[0m\u001b[1mhelp\u001b[0m\u001b[0m: the trait `Reducer<'_, _>` is not implemented for fn item `for<'a> fn(&'a ReducerContext, {type error}) {tick}`\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m= \u001b[0m\u001b[0m\u001b[1mnote\u001b[0m\u001b[0m: \u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m= \u001b[0m\u001b[0m\u001b[1mnote\u001b[0m\u001b[0m: reducer signatures must match the following pattern:\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m= \u001b[0m\u001b[0m\u001b[1mnote\u001b[0m\u001b[0m: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m= \u001b[0m\u001b[0m\u001b[1mnote\u001b[0m\u001b[0m: where each `Ti` type implements `SpacetimeType`.\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m= \u001b[0m\u001b[0m\u001b[1mnote\u001b[0m\u001b[0m: \u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;10mnote\u001b[0m\u001b[0m: required by a bound in `register_reducer`\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--> \u001b[0m\u001b[0m/home/runner/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/rt.rs:746:81\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m746\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0mpub fn register_reducer<'a, A: Args<'a>, I: FnInfo>(_: impl Reducer<'a, A>) {\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;10m^^^^^^^^^^^^^^\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;10mrequired by this bound in `register_reducer`\u001b[0m\n\n\u001b[0m\u001b[1m\u001b[38;5;9merror[E0277]\u001b[0m\u001b[0m\u001b[1m: invalid reducer signature\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--> \u001b[0m\u001b[0msrc/lib.rs:23:8\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m22\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m#[reducer]\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m----------\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12mrequired by a bound introduced by this call\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m23\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0mpub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9m^^^^\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;9mthis reducer signature is not valid\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m= \u001b[0m\u001b[0m\u001b[1mhelp\u001b[0m\u001b[0m: the trait `Reducer<'_, _>` is not implemented for fn item `for<'a> fn(&'a ReducerContext, {type error}) {tick}`\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m= \u001b[0m\u001b[0m\u001b[1mnote\u001b[0m\u001b[0m: \u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m= \u001b[0m\u001b[0m\u001b[1mnote\u001b[0m\u001b[0m: reducer signatures must match the following pattern:\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m= \u001b[0m\u001b[0m\u001b[1mnote\u001b[0m\u001b[0m: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m= \u001b[0m\u001b[0m\u001b[1mnote\u001b[0m\u001b[0m: where each `Ti` type implements `SpacetimeType`.\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m= \u001b[0m\u001b[0m\u001b[1mnote\u001b[0m\u001b[0m: \u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;10mnote\u001b[0m\u001b[0m: required by a bound in `invoke_reducer`\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--> \u001b[0m\u001b[0m/home/runner/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/rt.rs:45:19\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m44\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0mpub fn invoke_reducer<'a, A: Args<'a>>(\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m--------------\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12mrequired by a bound in this function\u001b[0m\n\u001b[0m\u001b[1m\u001b[38;5;12m45\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m reducer: impl Reducer<'a, A>,\u001b[0m\n\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;12m|\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;10m^^^^^^^^^^^^^^\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[1m\u001b[38;5;10mrequired by this bound in `invoke_reducer`\u001b[0m\n\n\u001b[0m\u001b[1mSome errors have detailed explanations: E0277, E0412, E0422, E0599.\u001b[0m\n\u001b[0m\u001b[1mFor more information about an error, try `rustc --explain E0277`.\u001b[0m\n\u001b[1m\u001b[31merror\u001b[0m\u001b[1m:\u001b[0m could not compile `spacetime-module` (lib) due to 8 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n",
- "phase": "build_or_publish"
+ "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-017-scheduled-columns-golden`.\n",
+ "phase": "describe_golden"
+ }
+ },
+ "scheduled_seeded_one_row": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM tick_timer WHERE scheduled_id>=0"
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:25:38.991765459Z",
- "finished_at": "2026-01-17T02:28:17.929994467Z"
+ "started_at": "2026-01-17T17:25:31.181359600Z",
+ "finished_at": "2026-01-17T17:26:19.960477500Z"
},
"t_018_constraints": {
- "hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a",
+ "hash": "1df24d7e0efd9d88ff9cc77fa5c951ac075acc22e1c3cd844adb37b87d805489",
"task": "t_018_constraints",
"lang": "rust",
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 3,
- "passed_tests": 2,
- "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = accounts, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let _ = ctx.db.accounts().try_insert(Account {\n id: 1,\n email: \"a@example.com\".to_string(),\n name: \"Alice\".to_string(),\n });\n let _ = ctx.db.accounts().try_insert(Account {\n id: 2,\n email: \"b@example.com\".to_string(),\n name: \"Bob\".to_string(),\n });\n}",
+ "passed_tests": 0,
+ "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = account, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.account().insert(Account {\n id: 1,\n email: \"a@example.com\".to_string(),\n name: \"Alice\".to_string(),\n });\n ctx.db.account().insert(Account {\n id: 2,\n email: \"b@example.com\".to_string(),\n name: \"Bob\".to_string(),\n });\n}",
"category": "schema",
"route_api_model": "gpt-5",
"golden_db": "schema-t-018-constraints-golden",
"llm_db": "schema-t-018-constraints-gpt-5-llm",
- "work_dir_golden": "target/llm-runs/schema/t_018_constraints/rust/server/golden",
- "work_dir_llm": "target/llm-runs/schema/t_018_constraints/rust/server/gpt-5/llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_018_constraints\\rust\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_018_constraints\\rust\\server\\gpt-5\\llm",
"scorer_details": {
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
+ "constraints_row_parity_after_seed": {
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "golden_db": "schema-t-018-constraints-golden",
- "llm_db": "schema-t-018-constraints-gpt-5-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "http://127.0.0.1:46587",
- "tables_diff": null,
- "tables_equal": true
+ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-018-constraints-golden`.\n",
+ "phase": "call_reducer_golden"
}
},
"constraints_seed_two_rows": {
- "pass": true,
- "partial": 1.0,
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "actual": 1,
+ "actual": 0,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM accounts WHERE id=2"
+ "sql": "SELECT COUNT(*) AS n FROM account WHERE id=2"
}
},
- "constraints_row_parity_after_seed": {
+ "schema_parity": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `accounts`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46587/v1/database/c2007d48b169e0048c4dd9c53c8d47e40c7761a8814064fe65c573b1ce844864/sql)\n",
- "phase": "sql_golden"
+ "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-018-constraints-golden`.\n",
+ "phase": "describe_golden"
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:25:38.991438731Z",
- "finished_at": "2026-01-17T02:28:15.400501818Z"
+ "started_at": "2026-01-17T17:25:31.241996400Z",
+ "finished_at": "2026-01-17T17:26:19.302522600Z"
},
"t_019_many_to_many": {
- "hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a",
+ "hash": "1df24d7e0efd9d88ff9cc77fa5c951ac075acc22e1c3cd844adb37b87d805489",
"task": "t_019_many_to_many",
"lang": "rust",
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 5,
- "passed_tests": 5,
- "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = users)]\npub struct User {\n #[primary_key]\n user_id: i32,\n name: String,\n}\n\n#[table(name = groups)]\npub struct Group {\n #[primary_key]\n group_id: i32,\n title: String,\n}\n\n#[table(\n name = memberships,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id]))\n)]\npub struct Membership {\n #[primary_key]\n id: i32,\n user_id: i32,\n group_id: i32,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let users = ctx.db.users();\n users.insert(User { user_id: 1, name: \"Alice\".to_string() });\n users.insert(User { user_id: 2, name: \"Bob\".to_string() });\n\n let groups = ctx.db.groups();\n groups.insert(Group { group_id: 10, title: \"Admin\".to_string() });\n groups.insert(Group { group_id: 20, title: \"Dev\".to_string() });\n\n let memberships = ctx.db.memberships();\n memberships.insert(Membership { id: 1, user_id: 1, group_id: 10 });\n memberships.insert(Membership { id: 2, user_id: 1, group_id: 20 });\n memberships.insert(Membership { id: 3, user_id: 2, group_id: 20 });\n}",
+ "passed_tests": 4,
+ "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n user_id: i32,\n name: String,\n}\n\n#[spacetimedb::table(name = group)]\npub struct Group {\n #[primary_key]\n group_id: i32,\n title: String,\n}\n\n#[spacetimedb::table(\n name = membership,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id]))\n)]\npub struct Membership {\n #[primary_key]\n id: i32,\n user_id: i32,\n group_id: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n // Clear existing data to ensure tables contain exactly the specified rows\n for m in ctx.db.membership().iter() {\n ctx.db.membership().id().delete(&m.id);\n }\n for g in ctx.db.group().iter() {\n ctx.db.group().group_id().delete(&g.group_id);\n }\n for u in ctx.db.user().iter() {\n ctx.db.user().user_id().delete(&u.user_id);\n }\n\n // Insert users\n ctx.db.user().insert(User { user_id: 1, name: \"Alice\".into() });\n ctx.db.user().insert(User { user_id: 2, name: \"Bob\".into() });\n\n // Insert groups\n ctx.db.group().insert(Group { group_id: 10, title: \"Admin\".into() });\n ctx.db.group().insert(Group { group_id: 20, title: \"Dev\".into() });\n\n // Insert memberships\n ctx.db.membership().insert(Membership { id: 1, user_id: 1, group_id: 10 });\n ctx.db.membership().insert(Membership { id: 2, user_id: 1, group_id: 20 });\n ctx.db.membership().insert(Membership { id: 3, user_id: 2, group_id: 20 });\n}",
"category": "schema",
"route_api_model": "gpt-5",
"golden_db": "schema-t-019-many-to-many-golden",
"llm_db": "schema-t-019-many-to-many-gpt-5-llm",
- "work_dir_golden": "target/llm-runs/schema/t_019_many_to_many/rust/server/golden",
- "work_dir_llm": "target/llm-runs/schema/t_019_many_to_many/rust/server/gpt-5/llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_019_many_to_many\\rust\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_019_many_to_many\\rust\\server\\gpt-5\\llm",
"scorer_details": {
- "m2m_has_1_20": {
+ "m2m_has_2_20": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM memberships WHERE user_id=1 AND group_id=20"
+ "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=2 AND group_id=20"
}
},
"memberships_three_rows": {
@@ -976,20 +1922,16 @@
"notes": {
"actual": 3,
"expected": 3,
- "sql": "SELECT COUNT(*) AS n FROM memberships"
+ "sql": "SELECT COUNT(*) AS n FROM membership"
}
},
- "schema_parity": {
+ "m2m_has_1_20": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-019-many-to-many-golden",
- "llm_db": "schema-t-019-many-to-many-gpt-5-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "http://127.0.0.1:46587",
- "tables_diff": null,
- "tables_equal": true
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=20"
}
},
"m2m_has_1_10": {
@@ -998,129 +1940,106 @@
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM memberships WHERE user_id=1 AND group_id=10"
+ "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=10"
}
},
- "m2m_has_2_20": {
- "pass": true,
- "partial": 1.0,
+ "schema_parity": {
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM memberships WHERE user_id=2 AND group_id=20"
+ "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-019-many-to-many-golden`.\n",
+ "phase": "describe_golden"
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:25:38.990785156Z",
- "finished_at": "2026-01-17T02:28:17.183262809Z"
+ "started_at": "2026-01-17T17:25:31.299904500Z",
+ "finished_at": "2026-01-17T17:26:19.858299600Z"
},
"t_020_ecs": {
- "hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a",
+ "hash": "1df24d7e0efd9d88ff9cc77fa5c951ac075acc22e1c3cd844adb37b87d805489",
"task": "t_020_ecs",
"lang": "rust",
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 5,
- "passed_tests": 5,
- "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = entities)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[table(name = positions)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[table(name = velocities)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[table(name = next_positions)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n if ctx.db.entities().id().find(1).is_none() {\n let _ = ctx.db.entities().insert(Entity { id: 1 });\n }\n match ctx.db.positions().entity_id().find(1) {\n Some(mut p) => {\n p.x = 0;\n p.y = 0;\n let _ = ctx.db.positions().entity_id().update(p);\n }\n None => {\n let _ = ctx.db.positions().insert(Position { entity_id: 1, x: 0, y: 0 });\n }\n }\n match ctx.db.velocities().entity_id().find(1) {\n Some(mut v) => {\n v.vx = 1;\n v.vy = 0;\n let _ = ctx.db.velocities().entity_id().update(v);\n }\n None => {\n let _ = ctx.db.velocities().insert(Velocity { entity_id: 1, vx: 1, vy: 0 });\n }\n }\n\n if ctx.db.entities().id().find(2).is_none() {\n let _ = ctx.db.entities().insert(Entity { id: 2 });\n }\n match ctx.db.positions().entity_id().find(2) {\n Some(mut p) => {\n p.x = 10;\n p.y = 0;\n let _ = ctx.db.positions().entity_id().update(p);\n }\n None => {\n let _ = ctx.db.positions().insert(Position { entity_id: 2, x: 10, y: 0 });\n }\n }\n match ctx.db.velocities().entity_id().find(2) {\n Some(mut v) => {\n v.vx = -2;\n v.vy = 3;\n let _ = ctx.db.velocities().entity_id().update(v);\n }\n None => {\n let _ = ctx.db.velocities().insert(Velocity { entity_id: 2, vx: -2, vy: 3 });\n }\n }\n}\n\n#[reducer]\npub fn step(ctx: &ReducerContext) {\n for pos in ctx.db.positions().iter() {\n if let Some(vel) = ctx.db.velocities().entity_id().find(pos.entity_id) {\n let nx = pos.x + vel.vx;\n let ny = pos.y + vel.vy;\n match ctx.db.next_positions().entity_id().find(pos.entity_id) {\n Some(mut np) => {\n np.x = nx;\n np.y = ny;\n let _ = ctx.db.next_positions().entity_id().update(np);\n }\n None => {\n let _ = ctx.db.next_positions().insert(NextPosition {\n entity_id: pos.entity_id,\n x: nx,\n y: ny,\n });\n }\n }\n }\n }\n}",
+ "passed_tests": 0,
+ "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[spacetimedb::table(name = position)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[spacetimedb::table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n // Entities\n if ctx.db.entity().id().find(1).is_none() {\n ctx.db.entity().insert(Entity { id: 1 });\n }\n if ctx.db.entity().id().find(2).is_none() {\n ctx.db.entity().insert(Entity { id: 2 });\n }\n\n // Positions\n if let Some(mut p) = ctx.db.position().entity_id().find(1) {\n p.x = 0;\n p.y = 0;\n ctx.db.position().entity_id().update(p);\n } else {\n ctx.db.position().insert(Position { entity_id: 1, x: 0, y: 0 });\n }\n\n if let Some(mut p) = ctx.db.position().entity_id().find(2) {\n p.x = 10;\n p.y = 0;\n ctx.db.position().entity_id().update(p);\n } else {\n ctx.db.position().insert(Position { entity_id: 2, x: 10, y: 0 });\n }\n\n // Velocities\n if let Some(mut v) = ctx.db.velocity().entity_id().find(1) {\n v.vx = 1;\n v.vy = 0;\n ctx.db.velocity().entity_id().update(v);\n } else {\n ctx.db.velocity().insert(Velocity { entity_id: 1, vx: 1, vy: 0 });\n }\n\n if let Some(mut v) = ctx.db.velocity().entity_id().find(2) {\n v.vx = -2;\n v.vy = 3;\n ctx.db.velocity().entity_id().update(v);\n } else {\n ctx.db.velocity().insert(Velocity { entity_id: 2, vx: -2, vy: 3 });\n }\n}\n\n#[spacetimedb::reducer]\npub fn step(ctx: &ReducerContext) {\n for pos in ctx.db.position().iter() {\n if let Some(vel) = ctx.db.velocity().entity_id().find(pos.entity_id) {\n let nx = pos.x + vel.vx;\n let ny = pos.y + vel.vy;\n if let Some(mut np) = ctx.db.next_position().entity_id().find(pos.entity_id) {\n np.x = nx;\n np.y = ny;\n ctx.db.next_position().entity_id().update(np);\n } else {\n ctx.db.next_position().insert(NextPosition {\n entity_id: pos.entity_id,\n x: nx,\n y: ny,\n });\n }\n }\n }\n}",
"category": "schema",
"route_api_model": "gpt-5",
"golden_db": "schema-t-020-ecs-golden",
"llm_db": "schema-t-020-ecs-gpt-5-llm",
- "work_dir_golden": "target/llm-runs/schema/t_020_ecs/rust/server/golden",
- "work_dir_llm": "target/llm-runs/schema/t_020_ecs/rust/server/gpt-5/llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_020_ecs\\rust\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_020_ecs\\rust\\server\\gpt-5\\llm",
"scorer_details": {
- "ecs_seed_positions_count": {
- "pass": true,
- "partial": 1.0,
+ "schema_parity": {
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "actual": 2,
- "expected": 2,
- "sql": "SELECT COUNT(*) AS n FROM positions"
+ "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-020-ecs-golden`.\n",
+ "phase": "describe_golden"
}
},
- "ecs_next_pos_entity1": {
- "pass": true,
- "partial": 1.0,
+ "ecs_step_next_positions_count": {
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM next_positions WHERE entity_id=1 AND x=1 AND y=0"
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: `next_positions` is not a valid table\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:51400/v1/database/c200bb4ac01e78338f7ddb7824a9f27a9545dc453e2eecd493a968b0d45d0829/sql)\n",
+ "phase": "sql"
}
},
"ecs_next_pos_entity2": {
- "pass": true,
- "partial": 1.0,
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM next_positions WHERE entity_id=2 AND x=8 AND y=3"
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: `next_positions` is not a valid table\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:51400/v1/database/c200bb4ac01e78338f7ddb7824a9f27a9545dc453e2eecd493a968b0d45d0829/sql)\n",
+ "phase": "sql"
}
},
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
+ "ecs_seed_positions_count": {
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "golden_db": "schema-t-020-ecs-golden",
- "llm_db": "schema-t-020-ecs-gpt-5-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "http://127.0.0.1:46587",
- "tables_diff": null,
- "tables_equal": true
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: `positions` is not a valid table\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:51400/v1/database/c200bb4ac01e78338f7ddb7824a9f27a9545dc453e2eecd493a968b0d45d0829/sql)\n",
+ "phase": "sql"
}
},
- "ecs_step_next_positions_count": {
- "pass": true,
- "partial": 1.0,
+ "ecs_next_pos_entity1": {
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "actual": 2,
- "expected": 2,
- "sql": "SELECT COUNT(*) AS n FROM next_positions"
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: `next_positions` is not a valid table\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:51400/v1/database/c200bb4ac01e78338f7ddb7824a9f27a9545dc453e2eecd493a968b0d45d0829/sql)\n",
+ "phase": "sql"
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:25:38.991130819Z",
- "finished_at": "2026-01-17T02:28:20.539029094Z"
+ "started_at": "2026-01-17T17:25:31.357064500Z",
+ "finished_at": "2026-01-17T17:26:22.067567200Z"
},
"t_021_multi_column_index": {
- "hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a",
+ "hash": "1df24d7e0efd9d88ff9cc77fa5c951ac075acc22e1c3cd844adb37b87d805489",
"task": "t_021_multi_column_index",
"lang": "rust",
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 4,
- "passed_tests": 4,
- "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = logs, index(name = by_user_day, btree(columns = [user_id, day])))]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let tbl = ctx.db.logs();\n let ids: Vec = tbl.iter().map(|r| r.id).collect();\n for id in ids {\n tbl.id().delete(&id);\n }\n\n tbl.insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".to_string() });\n tbl.insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".to_string() });\n tbl.insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".to_string() });\n}",
+ "passed_tests": 3,
+ "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = log, index(name = by_user_day, btree(columns = [user_id, day])))]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.log().insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".to_string() });\n ctx.db.log().insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".to_string() });\n ctx.db.log().insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".to_string() });\n}",
"category": "schema",
"route_api_model": "gpt-5",
"golden_db": "schema-t-021-multi-column-index-golden",
"llm_db": "schema-t-021-multi-column-index-gpt-5-llm",
- "work_dir_golden": "target/llm-runs/schema/t_021_multi_column_index/rust/server/golden",
- "work_dir_llm": "target/llm-runs/schema/t_021_multi_column_index/rust/server/gpt-5/llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_021_multi_column_index\\rust\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_021_multi_column_index\\rust\\server\\gpt-5\\llm",
"scorer_details": {
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "golden_db": "schema-t-021-multi-column-index-golden",
- "llm_db": "schema-t-021-multi-column-index-gpt-5-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "http://127.0.0.1:46587",
- "tables_diff": null,
- "tables_equal": true
- }
- },
"mcindex_lookup_u7_d1": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM logs WHERE user_id=7 AND day=1"
+ "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=1"
}
},
"mcindex_lookup_u7_d2": {
@@ -1129,7 +2048,7 @@
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM logs WHERE user_id=7 AND day=2"
+ "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=2"
}
},
"mcindex_seed_count": {
@@ -1138,13 +2057,21 @@
"notes": {
"actual": 3,
"expected": 3,
- "sql": "SELECT COUNT(*) AS n FROM logs"
+ "sql": "SELECT COUNT(*) AS n FROM log"
+ }
+ },
+ "schema_parity": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-021-multi-column-index-golden`.\n",
+ "phase": "describe_golden"
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:25:38.992084803Z",
- "finished_at": "2026-01-17T02:28:17.745571962Z"
+ "started_at": "2026-01-17T17:25:31.412817200Z",
+ "finished_at": "2026-01-17T17:26:19.605752700Z"
}
}
}
@@ -1245,7 +2172,7 @@
"syntax": "rust"
},
"t_001_basic_tables": {
- "answer": "use spacetimedb::table;\n\n#[table(name = users)]\npub struct Users {\n #[primary_key]\n pub id: i32,\n pub name: String,\n pub age: i32,\n pub active: bool,\n}\n\n#[table(name = products)]\npub struct Products {\n #[primary_key]\n pub id: i32,\n pub title: String,\n pub price: f32,\n pub in_stock: bool,\n}\n\n#[table(name = notes)]\npub struct Notes {\n #[primary_key]\n pub id: i32,\n pub body: String,\n pub rating: i64,\n pub pinned: bool,\n}\n",
+ "answer": "use spacetimedb::table;\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n pub id: i32,\n pub name: String,\n pub age: i32,\n pub active: bool,\n}\n\n#[table(name = product)]\npub struct Product {\n #[primary_key]\n pub id: i32,\n pub title: String,\n pub price: f32,\n pub in_stock: bool,\n}\n\n#[table(name = note)]\npub struct Note {\n #[primary_key]\n pub id: i32,\n pub body: String,\n pub rating: i64,\n pub pinned: bool,\n}\n",
"syntax": "rust"
},
"t_002_scheduled_table": {
@@ -1253,59 +2180,59 @@
"syntax": "rust"
},
"t_003_struct_in_table": {
- "answer": "use spacetimedb::{table, SpacetimeType};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Position {\n pub x: i32,\n pub y: i32,\n}\n\n#[table(name = entities)]\npub struct Entity {\n #[primary_key]\n pub id: i32,\n pub pos: Position,\n}\n\n",
+ "answer": "use spacetimedb::{table, SpacetimeType};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Position {\n pub x: i32,\n pub y: i32,\n}\n\n#[table(name = entity)]\npub struct Entity {\n #[primary_key]\n pub id: i32,\n pub pos: Position,\n}\n\n",
"syntax": "rust"
},
"t_004_insert": {
- "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(name = users)]\npub struct Users {\n #[primary_key]\n pub id: i32,\n pub name: String,\n pub age: i32,\n pub active: bool,\n}\n\n#[reducer]\npub fn insert_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) -> Result<(), String> {\n ctx.db.users().insert(Users { id, name, age, active });\n Ok(())\n}\n",
+ "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n pub id: i32,\n pub name: String,\n pub age: i32,\n pub active: bool,\n}\n\n#[reducer]\npub fn insert_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) -> Result<(), String> {\n ctx.db.user().insert(User { id, name, age, active });\n Ok(())\n}\n",
"syntax": "rust"
},
"t_005_update": {
- "answer": "use spacetimedb::{reducer, table, ReducerContext};\n\n#[table(name = users)]\npub struct User {\n #[primary_key]\n pub id: i32,\n pub name: String,\n pub age: i32,\n pub active: bool,\n}\n\n#[reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n ctx.db.users().id().update(User { id, name, age, active });\n}",
+ "answer": "use spacetimedb::{reducer, table, ReducerContext};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n pub id: i32,\n pub name: String,\n pub age: i32,\n pub active: bool,\n}\n\n#[reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n ctx.db.user().id().update(User { id, name, age, active });\n}",
"syntax": "rust"
},
"t_006_delete": {
- "answer": "use spacetimedb::{reducer, table, ReducerContext};\n\n#[table(name = users)]\npub struct User {\n #[primary_key]\n pub id: i32,\n pub name: String,\n pub age: i32,\n pub active: bool,\n}\n\n#[reducer]\npub fn delete_user(ctx: &ReducerContext, id: i32) {\n ctx.db.users().id().delete(id);\n}\n",
+ "answer": "use spacetimedb::{reducer, table, ReducerContext};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n pub id: i32,\n pub name: String,\n pub age: i32,\n pub active: bool,\n}\n\n#[reducer]\npub fn delete_user(ctx: &ReducerContext, id: i32) {\n ctx.db.user().id().delete(id);\n}\n",
"syntax": "rust"
},
"t_007_crud": {
- "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(name = users)]\npub struct User {\n #[primary_key]\n pub id: i32,\n pub name: String,\n pub age: i32,\n pub active: bool,\n}\n\n#[reducer]\npub fn crud(ctx: &ReducerContext) {\n ctx.db.users().insert(User { id: 1, name: \"Alice\".into(), age: 30, active: true });\n ctx.db.users().insert(User { id: 2, name: \"Bob\".into(), age: 22, active: false });\n ctx.db.users().id().update(User { id: 1, name: \"Alice2\".into(), age: 31, active: false });\n ctx.db.users().id().delete(2);\n}\n",
+ "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n pub id: i32,\n pub name: String,\n pub age: i32,\n pub active: bool,\n}\n\n#[reducer]\npub fn crud(ctx: &ReducerContext) {\n ctx.db.user().insert(User { id: 1, name: \"Alice\".into(), age: 30, active: true });\n ctx.db.user().insert(User { id: 2, name: \"Bob\".into(), age: 22, active: false });\n ctx.db.user().id().update(User { id: 1, name: \"Alice2\".into(), age: 31, active: false });\n ctx.db.user().id().delete(2);\n}\n",
"syntax": "rust"
},
"t_008_index_lookup": {
- "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(name = users)]\npub struct User {\n #[primary_key]\n pub id: i32,\n pub name: String,\n pub age: i32,\n pub active: bool,\n}\n\n#[table(name = results)]\npub struct ResultRow {\n #[primary_key]\n pub id: i32,\n pub name: String,\n}\n\n#[reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(u) = ctx.db.users().id().find(id) {\n ctx.db.results().insert(ResultRow { id: u.id, name: u.name });\n }\n}\n",
+ "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n pub id: i32,\n pub name: String,\n pub age: i32,\n pub active: bool,\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n pub id: i32,\n pub name: String,\n}\n\n#[reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(u) = ctx.db.user().id().find(id) {\n ctx.db.result().insert(ResultRow { id: u.id, name: u.name });\n }\n}\n",
"syntax": "rust"
},
"t_009_init": {
- "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(name = users)]\npub struct User {\n #[primary_key]\n pub id: i32,\n pub name: String,\n pub age: i32,\n pub active: bool,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.users().insert(User { id: 1, name: \"Alice\".into(), age: 30, active: true });\n ctx.db.users().insert(User { id: 2, name: \"Bob\".into(), age: 22, active: false });\n}\n",
+ "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n pub id: i32,\n pub name: String,\n pub age: i32,\n pub active: bool,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.user().insert(User { id: 1, name: \"Alice\".into(), age: 30, active: true });\n ctx.db.user().insert(User { id: 2, name: \"Bob\".into(), age: 22, active: false });\n}\n",
"syntax": "rust"
},
"t_010_connect": {
- "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(name = events)]\npub struct Event {\n #[primary_key]\n #[auto_inc]\n pub id: u64,\n pub kind: String,\n}\n\n#[reducer(client_connected)]\npub fn client_connected(ctx: &ReducerContext) {\n ctx.db.events().insert(Event { id: 0, kind: \"connected\".into() });\n}\n\n#[reducer(client_disconnected)]\npub fn client_disconnected(ctx: &ReducerContext) {\n ctx.db.events().insert(Event { id: 0, kind: \"disconnected\".into() });\n}\n",
+ "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(name = event)]\npub struct Event {\n #[primary_key]\n #[auto_inc]\n pub id: u64,\n pub kind: String,\n}\n\n#[reducer(client_connected)]\npub fn client_connected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event { id: 0, kind: \"connected\".into() });\n}\n\n#[reducer(client_disconnected)]\npub fn client_disconnected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event { id: 0, kind: \"disconnected\".into() });\n}\n",
"syntax": "rust"
},
"t_011_helper_function": {
- "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(name = results)]\npub struct ResultRow {\n #[primary_key]\n pub id: i32,\n pub sum: i32,\n}\n\nfn add(a: i32, b: i32) -> i32 { a + b }\n\n#[reducer]\npub fn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) {\n ctx.db.results().insert(ResultRow { id, sum: add(a, b) });\n}\n",
+ "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n pub id: i32,\n pub sum: i32,\n}\n\nfn add(a: i32, b: i32) -> i32 { a + b }\n\n#[reducer]\npub fn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) {\n ctx.db.result().insert(ResultRow { id, sum: add(a, b) });\n}\n",
"syntax": "rust"
},
"t_012_spacetime_product_type": {
- "answer": "use spacetimedb::{reducer, table, ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Score {\n pub left: i32,\n pub right: i32,\n}\n\n#[table(name = results)]\npub struct ResultRow {\n #[primary_key]\n pub id: i32,\n pub value: Score,\n}\n\n#[reducer]\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db.results().insert(ResultRow { id, value: Score { left, right } });\n}\n",
+ "answer": "use spacetimedb::{reducer, table, ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Score {\n pub left: i32,\n pub right: i32,\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n pub id: i32,\n pub value: Score,\n}\n\n#[reducer]\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db.result().insert(ResultRow { id, value: Score { left, right } });\n}\n",
"syntax": "rust"
},
"t_013_spacetime_sum_type": {
- "answer": "use spacetimedb::{reducer, table, ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Rect {\n pub width: i32,\n pub height: i32,\n}\n\n#[derive(SpacetimeType, Clone, Debug)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = results)]\npub struct ResultRow {\n #[primary_key]\n pub id: i32,\n pub value: Shape,\n}\n\n#[reducer]\npub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db.results().insert(ResultRow { id, value: Shape::Circle(radius) });\n}\n",
+ "answer": "use spacetimedb::{reducer, table, ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Rect {\n pub width: i32,\n pub height: i32,\n}\n\n#[derive(SpacetimeType, Clone, Debug)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n pub id: i32,\n pub value: Shape,\n}\n\n#[reducer]\npub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db.result().insert(ResultRow { id, value: Shape::Circle(radius) });\n}\n",
"syntax": "rust"
},
"t_014_elementary_columns": {
- "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(name = primitives)]\npub struct Primitive {\n #[primary_key]\n pub id: i32,\n pub count: i32,\n pub total: i64,\n pub price: f32,\n pub ratio: f64,\n pub active: bool,\n pub name: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitives().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000,\n price: 1.5,\n ratio: 2.25,\n active: true,\n name: \"Alice\".into(),\n });\n}\n",
+ "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(name = primitive)]\npub struct Primitive {\n #[primary_key]\n pub id: i32,\n pub count: i32,\n pub total: i64,\n pub price: f32,\n pub ratio: f64,\n pub active: bool,\n pub name: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000,\n price: 1.5,\n ratio: 2.25,\n active: true,\n name: \"Alice\".into(),\n });\n}\n",
"syntax": "rust"
},
"t_015_product_type_columns": {
- "answer": "use spacetimedb::{reducer, table, ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Address {\n pub street: String,\n pub zip: i32,\n}\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Position {\n pub x: i32,\n pub y: i32,\n}\n\n#[table(name = profiles)]\npub struct Profile {\n #[primary_key]\n pub id: i32,\n pub home: Address,\n pub work: Address,\n pub pos: Position,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.profiles().insert(Profile {\n id: 1,\n home: Address { street: \"1 Main\".into(), zip: 11111 },\n work: Address { street: \"2 Broad\".into(), zip: 22222 },\n pos: Position { x: 7, y: 9 },\n });\n}\n",
+ "answer": "use spacetimedb::{reducer, table, ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Address {\n pub street: String,\n pub zip: i32,\n}\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Position {\n pub x: i32,\n pub y: i32,\n}\n\n#[table(name = profile)]\npub struct Profile {\n #[primary_key]\n pub id: i32,\n pub home: Address,\n pub work: Address,\n pub pos: Position,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.profile().insert(Profile {\n id: 1,\n home: Address { street: \"1 Main\".into(), zip: 11111 },\n work: Address { street: \"2 Broad\".into(), zip: 22222 },\n pos: Position { x: 7, y: 9 },\n });\n}\n",
"syntax": "rust"
},
"t_016_sum_type_columns": {
- "answer": "use spacetimedb::{reducer, table, ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Rect {\n pub width: i32,\n pub height: i32,\n}\n\n#[derive(SpacetimeType, Clone, Debug)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = drawings)]\npub struct Drawing {\n #[primary_key]\n pub id: i32,\n pub a: Shape,\n pub b: Shape,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.drawings().insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n}\n",
+ "answer": "use spacetimedb::{reducer, table, ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Rect {\n pub width: i32,\n pub height: i32,\n}\n\n#[derive(SpacetimeType, Clone, Debug)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = drawing)]\npub struct Drawing {\n #[primary_key]\n pub id: i32,\n pub a: Shape,\n pub b: Shape,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.drawing().insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n}\n",
"syntax": "rust"
},
"t_017_scheduled_columns": {
@@ -1313,19 +2240,19 @@
"syntax": "rust"
},
"t_018_constraints": {
- "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(\n name = accounts,\n index(name = by_name, btree(columns = [name]))\n)]\npub struct Account {\n #[primary_key]\n pub id: i32,\n #[unique]\n pub email: String,\n pub name: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.accounts().insert(Account { id: 1, email: \"a@example.com\".into(), name: \"Alice\".into() });\n ctx.db.accounts().insert(Account { id: 2, email: \"b@example.com\".into(), name: \"Bob\".into() });\n}\n",
+ "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(\n name = account,\n index(name = by_name, btree(columns = [name]))\n)]\npub struct Account {\n #[primary_key]\n pub id: i32,\n #[unique]\n pub email: String,\n pub name: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.account().insert(Account { id: 1, email: \"a@example.com\".into(), name: \"Alice\".into() });\n ctx.db.account().insert(Account { id: 2, email: \"b@example.com\".into(), name: \"Bob\".into() });\n}\n",
"syntax": "rust"
},
"t_019_many_to_many": {
- "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(name = users)]\npub struct User {\n #[primary_key]\n pub user_id: i32,\n pub name: String,\n}\n\n#[table(name = groups)]\npub struct Group {\n #[primary_key]\n pub group_id: i32,\n pub title: String,\n}\n\n#[table(\n name = memberships,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id]))\n)]\npub struct Membership {\n #[primary_key]\n pub id: i32,\n pub user_id: i32,\n pub group_id: i32,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.users().insert(User { user_id: 1, name: \"Alice\".into() });\n ctx.db.users().insert(User { user_id: 2, name: \"Bob\".into() });\n\n ctx.db.groups().insert(Group { group_id: 10, title: \"Admin\".into() });\n ctx.db.groups().insert(Group { group_id: 20, title: \"Dev\".into() });\n\n ctx.db.memberships().insert(Membership { id: 1, user_id: 1, group_id: 10 });\n ctx.db.memberships().insert(Membership { id: 2, user_id: 1, group_id: 20 });\n ctx.db.memberships().insert(Membership { id: 3, user_id: 2, group_id: 20 });\n}\n",
+ "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n pub user_id: i32,\n pub name: String,\n}\n\n#[table(name = group)]\npub struct Group {\n #[primary_key]\n pub group_id: i32,\n pub title: String,\n}\n\n#[table(\n name = membership,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id]))\n)]\npub struct Membership {\n #[primary_key]\n pub id: i32,\n pub user_id: i32,\n pub group_id: i32,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.user().insert(User { user_id: 1, name: \"Alice\".into() });\n ctx.db.user().insert(User { user_id: 2, name: \"Bob\".into() });\n\n ctx.db.group().insert(Group { group_id: 10, title: \"Admin\".into() });\n ctx.db.group().insert(Group { group_id: 20, title: \"Dev\".into() });\n\n ctx.db.membership().insert(Membership { id: 1, user_id: 1, group_id: 10 });\n ctx.db.membership().insert(Membership { id: 2, user_id: 1, group_id: 20 });\n ctx.db.membership().insert(Membership { id: 3, user_id: 2, group_id: 20 });\n}\n",
"syntax": "rust"
},
"t_020_ecs": {
- "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(name = entities)]\npub struct Entity {\n #[primary_key]\n pub id: i32,\n}\n\n#[table(name = positions)]\npub struct Position {\n #[primary_key]\n pub entity_id: i32,\n pub x: i32,\n pub y: i32,\n}\n\n#[table(name = velocities)]\npub struct Velocity {\n #[primary_key]\n pub entity_id: i32,\n pub vx: i32,\n pub vy: i32,\n}\n\n#[table(name = next_positions)]\npub struct NextPosition {\n #[primary_key]\n pub entity_id: i32,\n pub x: i32,\n pub y: i32,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.entities().insert(Entity { id: 1 });\n ctx.db.entities().insert(Entity { id: 2 });\n\n ctx.db.positions().insert(Position {\n entity_id: 1,\n x: 1,\n y: 0,\n });\n ctx.db.positions().insert(Position {\n entity_id: 2,\n x: 10,\n y: 0,\n });\n\n ctx.db.velocities().insert(Velocity {\n entity_id: 1,\n vx: 1,\n vy: 0,\n });\n ctx.db.velocities().insert(Velocity {\n entity_id: 2,\n vx: -2,\n vy: 3,\n });\n}\n\n#[spacetimedb::reducer]\npub fn step(ctx: &ReducerContext) {\n for p in ctx.db.positions().iter() {\n if let Some(v) = ctx.db.velocities().entity_id().find(p.entity_id) {\n let np = NextPosition {\n entity_id: p.entity_id,\n x: p.x + v.vx,\n y: p.y + v.vy,\n };\n\n if ctx.db.next_positions().entity_id().find(p.entity_id).is_some() {\n ctx.db.next_positions().entity_id().update(np);\n } else {\n ctx.db.next_positions().insert(np);\n }\n }\n }\n}\n",
+ "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(name = entity)]\npub struct Entity {\n #[primary_key]\n pub id: i32,\n}\n\n#[table(name = position)]\npub struct Position {\n #[primary_key]\n pub entity_id: i32,\n pub x: i32,\n pub y: i32,\n}\n\n#[table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n pub entity_id: i32,\n pub vx: i32,\n pub vy: i32,\n}\n\n#[table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n pub entity_id: i32,\n pub x: i32,\n pub y: i32,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.entity().insert(Entity { id: 1 });\n ctx.db.entity().insert(Entity { id: 2 });\n\n ctx.db.position().insert(Position {\n entity_id: 1,\n x: 1,\n y: 0,\n });\n ctx.db.position().insert(Position {\n entity_id: 2,\n x: 10,\n y: 0,\n });\n\n ctx.db.velocity().insert(Velocity {\n entity_id: 1,\n vx: 1,\n vy: 0,\n });\n ctx.db.velocity().insert(Velocity {\n entity_id: 2,\n vx: -2,\n vy: 3,\n });\n}\n\n#[spacetimedb::reducer]\npub fn step(ctx: &ReducerContext) {\n for p in ctx.db.position().iter() {\n if let Some(v) = ctx.db.velocity().entity_id().find(p.entity_id) {\n let np = NextPosition {\n entity_id: p.entity_id,\n x: p.x + v.vx,\n y: p.y + v.vy,\n };\n\n if ctx.db.next_position().entity_id().find(p.entity_id).is_some() {\n ctx.db.next_position().entity_id().update(np);\n } else {\n ctx.db.next_position().insert(np);\n }\n }\n }\n}\n",
"syntax": "rust"
},
"t_021_multi_column_index": {
- "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(\n name = logs,\n index(name = by_user_day, btree(columns = [user_id, day]))\n)]\npub struct Log {\n #[primary_key]\n pub id: i32,\n pub user_id: i32,\n pub day: i32,\n pub message: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.logs().insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".into() });\n ctx.db.logs().insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".into() });\n ctx.db.logs().insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".into() });\n}\n",
+ "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(\n name = log,\n index(name = by_user_day, btree(columns = [user_id, day]))\n)]\npub struct Log {\n #[primary_key]\n pub id: i32,\n pub user_id: i32,\n pub day: i32,\n pub message: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.log().insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".into() });\n ctx.db.log().insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".into() });\n ctx.db.log().insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".into() });\n}\n",
"syntax": "rust"
}
}
@@ -1335,14 +2262,14 @@
"modes": [
{
"mode": "docs",
- "hash": "7bd1056c239ec41df56d3be0edd5b7aac1c433acc3efbac0fe6f3b2a79ca2f1f",
+ "hash": "c9afa66dc818f616ea70c65180c235ac58b63ddcb485be6e6f173bb206a122d5",
"models": [
{
"name": "GPT-5",
"route_api_model": "gpt-5",
"tasks": {
"t_000_empty_reducers": {
- "hash": "7bd1056c239ec41df56d3be0edd5b7aac1c433acc3efbac0fe6f3b2a79ca2f1f",
+ "hash": "c9afa66dc818f616ea70c65180c235ac58b63ddcb485be6e6f173bb206a122d5",
"task": "t_000_empty_reducers",
"lang": "csharp",
"golden_published": true,
@@ -1365,25 +2292,25 @@
"llm_db": "basics-t-000-empty-reducers-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:35677",
+ "server": "http://127.0.0.1:37375",
"tables_diff": null,
"tables_equal": true
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:40:24.282055723Z",
- "finished_at": "2026-01-17T02:40:56.269475914Z"
+ "started_at": "2026-01-20T01:42:27.068364290Z",
+ "finished_at": "2026-01-20T01:43:19.406794050Z"
},
"t_001_basic_tables": {
- "hash": "7bd1056c239ec41df56d3be0edd5b7aac1c433acc3efbac0fe6f3b2a79ca2f1f",
+ "hash": "c9afa66dc818f616ea70c65180c235ac58b63ddcb485be6e6f173bb206a122d5",
"task": "t_001_basic_tables",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 1,
"passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table]\n public partial struct Product\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Title;\n public float Price;\n public bool InStock;\n }\n\n [SpacetimeDB.Table]\n public partial struct Note\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Body;\n public long Rating;\n public bool Pinned;\n }\n}",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"Product\", Public = true)]\n public partial struct Product\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Title;\n public float Price;\n public bool InStock;\n }\n\n [SpacetimeDB.Table(Name = \"Note\", Public = true)]\n public partial struct Note\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Body;\n public long Rating;\n public bool Pinned;\n }\n}",
"category": "basics",
"route_api_model": "gpt-5",
"golden_db": "basics-t-001-basic-tables-golden",
@@ -1399,25 +2326,25 @@
"llm_db": "basics-t-001-basic-tables-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:35677",
+ "server": "http://127.0.0.1:37375",
"tables_diff": null,
"tables_equal": true
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:43:28.640515207Z",
- "finished_at": "2026-01-17T02:43:58.187104574Z"
+ "started_at": "2026-01-20T01:41:53.640855693Z",
+ "finished_at": "2026-01-20T01:43:15.400054129Z"
},
"t_002_scheduled_table": {
- "hash": "7bd1056c239ec41df56d3be0edd5b7aac1c433acc3efbac0fe6f3b2a79ca2f1f",
+ "hash": "c9afa66dc818f616ea70c65180c235ac58b63ddcb485be6e6f173bb206a122d5",
"task": "t_002_scheduled_table",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 1,
"passed_tests": 1,
- "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [PrimaryKey, AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMilliseconds(50))\n });\n }\n\n [Reducer]\n public static void Tick(ReducerContext ctx, TickTimer timer)\n {\n }\n}",
+ "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [PrimaryKey, AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [Reducer]\n public static void Tick(ReducerContext ctx, TickTimer _timer)\n {\n }\n\n [Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMilliseconds(50))\n });\n }\n}",
"category": "basics",
"route_api_model": "gpt-5",
"golden_db": "basics-t-002-scheduled-table-golden",
@@ -1433,25 +2360,25 @@
"llm_db": "basics-t-002-scheduled-table-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:35677",
+ "server": "http://127.0.0.1:37375",
"tables_diff": null,
"tables_equal": true
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:42:25.159472748Z",
- "finished_at": "2026-01-17T02:43:07.893721504Z"
+ "started_at": "2026-01-20T01:43:07.332489593Z",
+ "finished_at": "2026-01-20T01:43:41.631485656Z"
},
"t_003_struct_in_table": {
- "hash": "7bd1056c239ec41df56d3be0edd5b7aac1c433acc3efbac0fe6f3b2a79ca2f1f",
+ "hash": "c9afa66dc818f616ea70c65180c235ac58b63ddcb485be6e6f173bb206a122d5",
"task": "t_003_struct_in_table",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 1,
"passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Entity\", Public = true)]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Position Pos;\n }\n}",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Entity\")]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Position Pos;\n }\n}",
"category": "basics",
"route_api_model": "gpt-5",
"golden_db": "basics-t-003-struct-in-table-golden",
@@ -1467,18 +2394,18 @@
"llm_db": "basics-t-003-struct-in-table-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:35677",
+ "server": "http://127.0.0.1:37375",
"tables_diff": null,
"tables_equal": true
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:43:07.893755608Z",
- "finished_at": "2026-01-17T02:43:39.643821282Z"
+ "started_at": "2026-01-20T01:41:53.643438245Z",
+ "finished_at": "2026-01-20T01:43:11.935014768Z"
},
"t_004_insert": {
- "hash": "7bd1056c239ec41df56d3be0edd5b7aac1c433acc3efbac0fe6f3b2a79ca2f1f",
+ "hash": "c9afa66dc818f616ea70c65180c235ac58b63ddcb485be6e6f173bb206a122d5",
"task": "t_004_insert",
"lang": "csharp",
"golden_published": true,
@@ -1497,7 +2424,7 @@
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c200933266559154a4d1e8adf441af2b283882c3206ed34f5c4ef907d29de6a7/sql)\n",
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37375/v1/database/c200c94740d8f1b4685d4497d0a77f8d7ee1d2ef5dc4aad6273dd49d723ef216/sql)\n",
"phase": "sql_golden"
}
},
@@ -1509,25 +2436,25 @@
"llm_db": "basics-t-004-insert-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:35677",
+ "server": "http://127.0.0.1:37375",
"tables_diff": null,
"tables_equal": true
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:43:58.187152683Z",
- "finished_at": "2026-01-17T02:44:26.839511728Z"
+ "started_at": "2026-01-20T01:41:53.649527015Z",
+ "finished_at": "2026-01-20T01:42:27.068291893Z"
},
"t_005_update": {
- "hash": "7bd1056c239ec41df56d3be0edd5b7aac1c433acc3efbac0fe6f3b2a79ca2f1f",
+ "hash": "c9afa66dc818f616ea70c65180c235ac58b63ddcb485be6e6f173bb206a122d5",
"task": "t_005_update",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 3,
"passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n if (ctx.Db.User.Id.Find(id) is User user)\n {\n user.Id = id;\n user.Name = name;\n user.Age = age;\n user.Active = active;\n ctx.Db.User.Id.Update(user);\n }\n }\n}",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n var existing = ctx.Db.User.Id.Find(id);\n if (existing == null)\n {\n throw new System.Exception(\"User not found\");\n }\n\n var updated = new User\n {\n Id = id,\n Name = name,\n Age = age,\n Active = active\n };\n\n ctx.Db.User.Id.Update(updated);\n }\n}",
"category": "basics",
"route_api_model": "gpt-5",
"golden_db": "basics-t-005-update-golden",
@@ -1535,14 +2462,6 @@
"work_dir_golden": "target/llm-runs/basics/t_005_update/csharp/server/golden",
"work_dir_llm": "target/llm-runs/basics/t_005_update/csharp/server/gpt-5/llm",
"scorer_details": {
- "data_parity_update_user": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: SpacetimeDB.NoSuchRowException: The row was not found, e.g., in an update call\n at SpacetimeDB.Internal.FFI.CheckedStatus.Marshaller.ConvertToManaged(Errno )\n at SpacetimeDB.Internal.FFI.datastore_update_bsatn(TableId , IndexId , Span`1 , UInt32& )\n at SpacetimeDB.Internal.UniqueIndex`4[[SpacetimeDB.Internal.TableHandles.User, StdbModule, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null],[Module.User, StdbModule, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null],[System.Int32, System.Private.CoreLib, Version=8.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e],[SpacetimeDB.BSATN.I32, SpacetimeDB.BSATN.Runtime, Version=1.6.0.0, Culture=neutral, PublicKeyToken=null]].DoUpdate(User )\n at SpacetimeDB.Internal.TableHandles.User.IdUniqueIndex.Update(User )\n at Module.UpdateUser(ReducerContext , Int32 , String , Int32 , Boolean )\n at ModuleRegistration.UpdateUser.Invoke(BinaryReader , IReducerContext )\n at SpacetimeDB.Internal.Module.__call_reducer__(UInt32 id, UInt64 sender_0, UInt64 sender_1, UInt64 sender_2, UInt64 sender_3, UInt64 conn_id_0, UInt64 conn_id_1, Timestamp timestamp, BytesSource args, BytesSink error)\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:35677/v1/database/c200523e08ccb74ff8d13ab66f492b457bb8aeee5e4fdafc5d1fec7431a9b4e9/call/UpdateUser)\n",
- "phase": "call_reducer_golden"
- }
- },
"schema_parity": {
"pass": true,
"partial": 1.0,
@@ -1551,7 +2470,7 @@
"llm_db": "basics-t-005-update-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:35677",
+ "server": "http://127.0.0.1:37375",
"tables_diff": null,
"tables_equal": true
}
@@ -1560,25 +2479,33 @@
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c200523e08ccb74ff8d13ab66f492b457bb8aeee5e4fdafc5d1fec7431a9b4e9/sql)\n",
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37375/v1/database/c20033139bef872cfe2bfd2109e764bc794b357c86a50753533c9c532161df94/sql)\n",
"phase": "sql_golden",
"sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
}
+ },
+ "data_parity_update_user": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: SpacetimeDB.NoSuchRowException: The row was not found, e.g., in an update call\n at SpacetimeDB.Internal.FFI.CheckedStatus.Marshaller.ConvertToManaged(Errno )\n at SpacetimeDB.Internal.FFI.datastore_update_bsatn(TableId , IndexId , Span`1 , UInt32& )\n at SpacetimeDB.Internal.UniqueIndex`4[[SpacetimeDB.Internal.TableHandles.User, StdbModule, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null],[Module.User, StdbModule, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null],[System.Int32, System.Private.CoreLib, Version=8.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e],[SpacetimeDB.BSATN.I32, SpacetimeDB.BSATN.Runtime, Version=1.6.0.0, Culture=neutral, PublicKeyToken=null]].DoUpdate(User )\n at SpacetimeDB.Internal.TableHandles.User.IdUniqueIndex.Update(User )\n at Module.UpdateUser(ReducerContext , Int32 , String , Int32 , Boolean )\n at ModuleRegistration.UpdateUser.Invoke(BinaryReader , IReducerContext )\n at SpacetimeDB.Internal.Module.__call_reducer__(UInt32 id, UInt64 sender_0, UInt64 sender_1, UInt64 sender_2, UInt64 sender_3, UInt64 conn_id_0, UInt64 conn_id_1, Timestamp timestamp, BytesSource args, BytesSink error)\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:37375/v1/database/c20033139bef872cfe2bfd2109e764bc794b357c86a50753533c9c532161df94/call/UpdateUser)\n",
+ "phase": "call_reducer_golden"
+ }
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:42:19.102421878Z",
- "finished_at": "2026-01-17T02:42:52.237316726Z"
+ "started_at": "2026-01-20T01:43:19.406818832Z",
+ "finished_at": "2026-01-20T01:43:51.938246362Z"
},
"t_006_delete": {
- "hash": "7bd1056c239ec41df56d3be0edd5b7aac1c433acc3efbac0fe6f3b2a79ca2f1f",
+ "hash": "c9afa66dc818f616ea70c65180c235ac58b63ddcb485be6e6f173bb206a122d5",
"task": "t_006_delete",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 3,
"passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void DeleteUser(ReducerContext ctx, int id)\n {\n ctx.Db.User.Id.Delete(id);\n }\n}",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void DeleteUser(ReducerContext ctx, int id)\n {\n ctx.Db.User.Id.Delete(id);\n }\n}",
"category": "basics",
"route_api_model": "gpt-5",
"golden_db": "basics-t-006-delete-golden",
@@ -1586,6 +2513,15 @@
"work_dir_golden": "target/llm-runs/basics/t_006_delete/csharp/server/golden",
"work_dir_llm": "target/llm-runs/basics/t_006_delete/csharp/server/gpt-5/llm",
"scorer_details": {
+ "seed_users_row": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37375/v1/database/c200461e3b2c21fbedbfd5ca3a6de128dcd3f24a539449d3530eca7bbfbc5939/sql)\n",
+ "phase": "sql_golden",
+ "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
+ }
+ },
"schema_parity": {
"pass": true,
"partial": 1.0,
@@ -1594,7 +2530,7 @@
"llm_db": "basics-t-006-delete-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:35677",
+ "server": "http://127.0.0.1:37375",
"tables_diff": null,
"tables_equal": true
}
@@ -1603,33 +2539,24 @@
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c2004b74d3be00407c2d02282ee08b25e5e8242063995dea613d8fc1467ef09e/sql)\n",
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37375/v1/database/c200d7ccf962b6202351545969e5fdac00030c3173aa5f82861c44a87560d507/sql)\n",
"phase": "sql"
}
- },
- "seed_users_row": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c200052ec3b2af2555b2766443d690ff70a4d07ffd410be917b433381552b936/sql)\n",
- "phase": "sql_golden",
- "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
- }
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:40:56.269507302Z",
- "finished_at": "2026-01-17T02:41:28.991775660Z"
+ "started_at": "2026-01-20T01:43:52.165830877Z",
+ "finished_at": "2026-01-20T01:44:21.031010432Z"
},
"t_007_crud": {
- "hash": "7bd1056c239ec41df56d3be0edd5b7aac1c433acc3efbac0fe6f3b2a79ca2f1f",
+ "hash": "c9afa66dc818f616ea70c65180c235ac58b63ddcb485be6e6f173bb206a122d5",
"task": "t_007_crud",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 4,
"passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void Crud(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n\n if (ctx.Db.User.Id.Find(1) is User user)\n {\n user.Name = \"Alice2\";\n user.Age = 31;\n user.Active = false;\n ctx.Db.User.Id.Update(user);\n }\n\n ctx.Db.User.Id.Delete(2);\n }\n}",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void Crud(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n\n if (ctx.Db.User.Id.Find(1) is User u)\n {\n u.Name = \"Alice2\";\n u.Age = 31;\n u.Active = false;\n ctx.Db.User.Id.Update(u);\n }\n\n ctx.Db.User.Id.Delete(2);\n }\n}",
"category": "basics",
"route_api_model": "gpt-5",
"golden_db": "basics-t-007-crud-golden",
@@ -1637,22 +2564,6 @@
"work_dir_golden": "target/llm-runs/basics/t_007_crud/csharp/server/golden",
"work_dir_llm": "target/llm-runs/basics/t_007_crud/csharp/server/gpt-5/llm",
"scorer_details": {
- "crud_row_id1_parity": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c2006b22042e3fba4f36b1b44b80a07da826122ccc12419c53c511830a0c78f9/sql)\n",
- "phase": "sql_golden"
- }
- },
- "crud_row_id2_deleted": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c2003c5aaa4b9c829e64c505f35cb7241f299a733d0f5a89da64241c5e052096/sql)\n",
- "phase": "sql"
- }
- },
"schema_parity": {
"pass": true,
"partial": 1.0,
@@ -1661,7 +2572,7 @@
"llm_db": "basics-t-007-crud-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:35677",
+ "server": "http://127.0.0.1:37375",
"tables_diff": null,
"tables_equal": true
}
@@ -1670,24 +2581,40 @@
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c2003c5aaa4b9c829e64c505f35cb7241f299a733d0f5a89da64241c5e052096/sql)\n",
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37375/v1/database/c2002e202a038508a32a9d443dc1c63dd15314e7a4ae42f8b18a563c5b7dea56/sql)\n",
+ "phase": "sql"
+ }
+ },
+ "crud_row_id1_parity": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37375/v1/database/c2001be58a6d663b1425393e27a72272ddd75b0fdf528881894768b052629853/sql)\n",
+ "phase": "sql_golden"
+ }
+ },
+ "crud_row_id2_deleted": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37375/v1/database/c2002e202a038508a32a9d443dc1c63dd15314e7a4ae42f8b18a563c5b7dea56/sql)\n",
"phase": "sql"
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:40:51.462439194Z",
- "finished_at": "2026-01-17T02:41:35.805469514Z"
+ "started_at": "2026-01-20T01:43:15.400084536Z",
+ "finished_at": "2026-01-20T01:43:52.102649565Z"
},
"t_008_index_lookup": {
- "hash": "7bd1056c239ec41df56d3be0edd5b7aac1c433acc3efbac0fe6f3b2a79ca2f1f",
+ "hash": "c9afa66dc818f616ea70c65180c235ac58b63ddcb485be6e6f173bb206a122d5",
"task": "t_008_index_lookup",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 3,
"passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"Result\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void LookupUserName(ReducerContext ctx, int id)\n {\n if (ctx.Db.User.Id.Find(id) is User user)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = user.Id,\n Name = user.Name\n });\n }\n }\n}",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void LookupUserName(ReducerContext ctx, int id)\n {\n if (ctx.Db.User.Id.Find(id) is User user)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = user.Id,\n Name = user.Name\n });\n }\n }\n}",
"category": "basics",
"route_api_model": "gpt-5",
"golden_db": "basics-t-008-index-lookup-golden",
@@ -1695,43 +2622,43 @@
"work_dir_golden": "target/llm-runs/basics/t_008_index_lookup/csharp/server/golden",
"work_dir_llm": "target/llm-runs/basics/t_008_index_lookup/csharp/server/gpt-5/llm",
"scorer_details": {
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
+ "index_lookup_projection_parity": {
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "golden_db": "basics-t-008-index-lookup-golden",
- "llm_db": "basics-t-008-index-lookup-gpt-5-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "http://127.0.0.1:35677",
- "tables_diff": null,
- "tables_equal": true
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `results`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37375/v1/database/c2003774ae931536ebc6ad7f9274cfb87aeb96d8d764ac2ae7109babebdecc65/sql)\n",
+ "phase": "sql_golden"
}
},
"seed_user_row": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c2009fb018392f6be9608948e77d6c1cbdd60e04fc795fc595009cb76a175394/sql)\n",
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37375/v1/database/c2003774ae931536ebc6ad7f9274cfb87aeb96d8d764ac2ae7109babebdecc65/sql)\n",
"phase": "sql_golden",
"sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
}
},
- "index_lookup_projection_parity": {
- "pass": false,
- "partial": 0.0,
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `results`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c2009fb018392f6be9608948e77d6c1cbdd60e04fc795fc595009cb76a175394/sql)\n",
- "phase": "sql_golden"
+ "golden_db": "basics-t-008-index-lookup-golden",
+ "llm_db": "basics-t-008-index-lookup-gpt-5-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "http://127.0.0.1:37375",
+ "tables_diff": null,
+ "tables_equal": true
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:41:28.991836824Z",
- "finished_at": "2026-01-17T02:42:19.102358640Z"
+ "started_at": "2026-01-20T01:43:52.163281010Z",
+ "finished_at": "2026-01-20T01:44:23.181394773Z"
},
"t_009_init": {
- "hash": "7bd1056c239ec41df56d3be0edd5b7aac1c433acc3efbac0fe6f3b2a79ca2f1f",
+ "hash": "c9afa66dc818f616ea70c65180c235ac58b63ddcb485be6e6f173bb206a122d5",
"task": "t_009_init",
"lang": "csharp",
"golden_published": true,
@@ -1746,11 +2673,19 @@
"work_dir_golden": "target/llm-runs/basics/t_009_init/csharp/server/golden",
"work_dir_llm": "target/llm-runs/basics/t_009_init/csharp/server/gpt-5/llm",
"scorer_details": {
- "init_seed_bob": {
+ "init_seed_alice": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37375/v1/database/c2006ba9d8287211c1bf2ec502e8fb2a4ee853e75cb38784fd1ecf7f27ae05f6/sql)\n",
+ "phase": "sql"
+ }
+ },
+ "init_total_two": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c20058e57001dda166048aa4be9fd4471b9b37f134fd7481e1ada013429df130/sql)\n",
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37375/v1/database/c2006ba9d8287211c1bf2ec502e8fb2a4ee853e75cb38784fd1ecf7f27ae05f6/sql)\n",
"phase": "sql"
}
},
@@ -1762,41 +2697,33 @@
"llm_db": "basics-t-009-init-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:35677",
+ "server": "http://127.0.0.1:37375",
"tables_diff": null,
"tables_equal": true
}
},
- "init_seed_alice": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c20058e57001dda166048aa4be9fd4471b9b37f134fd7481e1ada013429df130/sql)\n",
- "phase": "sql"
- }
- },
- "init_total_two": {
+ "init_seed_bob": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c20058e57001dda166048aa4be9fd4471b9b37f134fd7481e1ada013429df130/sql)\n",
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `users`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37375/v1/database/c2006ba9d8287211c1bf2ec502e8fb2a4ee853e75cb38784fd1ecf7f27ae05f6/sql)\n",
"phase": "sql"
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:41:35.805507675Z",
- "finished_at": "2026-01-17T02:42:25.159425259Z"
+ "started_at": "2026-01-20T01:43:41.631516940Z",
+ "finished_at": "2026-01-20T01:44:09.136426429Z"
},
"t_010_connect": {
- "hash": "7bd1056c239ec41df56d3be0edd5b7aac1c433acc3efbac0fe6f3b2a79ca2f1f",
+ "hash": "c9afa66dc818f616ea70c65180c235ac58b63ddcb485be6e6f173bb206a122d5",
"task": "t_010_connect",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 1,
"passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Event\")]\n public partial struct Event\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public int Id;\n public string Kind;\n }\n\n [Reducer(ReducerKind.ClientConnected)]\n public static void ClientConnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event\n {\n Id = 0,\n Kind = \"connected\",\n });\n }\n\n [Reducer(ReducerKind.ClientDisconnected)]\n public static void ClientDisconnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event\n {\n Id = 0,\n Kind = \"disconnected\",\n });\n }\n}",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Event\")]\n public partial struct Event\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public int Id;\n public string Kind;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientConnected)]\n public static void ClientConnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event\n {\n Id = 0,\n Kind = \"connected\"\n });\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientDisconnected)]\n public static void ClientDisconnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event\n {\n Id = 0,\n Kind = \"disconnected\"\n });\n }\n}",
"category": "basics",
"route_api_model": "gpt-5",
"golden_db": "basics-t-010-connect-golden",
@@ -1812,25 +2739,25 @@
"llm_db": "basics-t-010-connect-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:35677",
+ "server": "http://127.0.0.1:37375",
"tables_diff": null,
"tables_equal": true
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:43:39.643859323Z",
- "finished_at": "2026-01-17T02:44:14.452325815Z"
+ "started_at": "2026-01-20T01:43:11.935046077Z",
+ "finished_at": "2026-01-20T01:43:52.163256943Z"
},
"t_011_helper_function": {
- "hash": "7bd1056c239ec41df56d3be0edd5b7aac1c433acc3efbac0fe6f3b2a79ca2f1f",
+ "hash": "c9afa66dc818f616ea70c65180c235ac58b63ddcb485be6e6f173bb206a122d5",
"task": "t_011_helper_function",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 3,
"passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Result\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Sum;\n }\n\n public static int Add(int a, int b)\n {\n return a + b;\n }\n\n [SpacetimeDB.Reducer]\n public static void ComputeSum(ReducerContext ctx, int id, int a, int b)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Sum = Add(a, b)\n });\n }\n}",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Result\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Sum;\n }\n\n public static int Add(int a, int b) => a + b;\n\n [SpacetimeDB.Reducer]\n public static void ComputeSum(ReducerContext ctx, int id, int a, int b)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Sum = Add(a, b)\n });\n }\n}",
"category": "basics",
"route_api_model": "gpt-5",
"golden_db": "basics-t-011-helper-function-golden",
@@ -1838,14 +2765,6 @@
"work_dir_golden": "target/llm-runs/basics/t_011_helper_function/csharp/server/golden",
"work_dir_llm": "target/llm-runs/basics/t_011_helper_function/csharp/server/gpt-5/llm",
"scorer_details": {
- "helper_func_sum_abs": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `results`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c200bacfa9e399c2f18f42a36dcfe8479e933d44e6e749cbcc699afdf2cd855d/sql)\n",
- "phase": "sql"
- }
- },
"schema_parity": {
"pass": true,
"partial": 1.0,
@@ -1854,26 +2773,34 @@
"llm_db": "basics-t-011-helper-function-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:35677",
+ "server": "http://127.0.0.1:37375",
"tables_diff": null,
"tables_equal": true
}
},
+ "helper_func_sum_abs": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `results`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37375/v1/database/c200c7993c42ef0874c1289908b3ff4d4ca7073fba3d0d0b66230934be15edab/sql)\n",
+ "phase": "sql"
+ }
+ },
"helper_func_sum_parity": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `results`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c200b67e605283bc59b71366b2f40ab582a2bbbfc946e2c8510ce84f55424a14/sql)\n",
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `results`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37375/v1/database/c200f4c4de99f0f613e0b22095e0584ab477f6718813d93f412702fe70c17cb8/sql)\n",
"phase": "sql_golden"
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:42:52.237342945Z",
- "finished_at": "2026-01-17T02:43:28.640296252Z"
+ "started_at": "2026-01-20T01:41:53.646335902Z",
+ "finished_at": "2026-01-20T01:43:07.332403303Z"
},
"t_012_spacetime_product_type": {
- "hash": "7bd1056c239ec41df56d3be0edd5b7aac1c433acc3efbac0fe6f3b2a79ca2f1f",
+ "hash": "c9afa66dc818f616ea70c65180c235ac58b63ddcb485be6e6f173bb206a122d5",
"task": "t_012_spacetime_product_type",
"lang": "csharp",
"golden_published": true,
@@ -1888,24 +2815,11 @@
"work_dir_golden": "target/llm-runs/schema/t_012_spacetime_product_type/csharp/server/golden",
"work_dir_llm": "target/llm-runs/schema/t_012_spacetime_product_type/csharp/server/gpt-5/llm",
"scorer_details": {
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "golden_db": "schema-t-012-spacetime-product-type-golden",
- "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "http://127.0.0.1:35677",
- "tables_diff": null,
- "tables_equal": true
- }
- },
"product_type_row_parity": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `results`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c200000593a1c1f41d9a7ca9ff85b3f10b7eb54ba57159f084fd8b1d4196e453/sql)\n",
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `results`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37375/v1/database/c20067fe1e213b2b6055a2265abacde90f33a54d9ca62894ef018009332c524d/sql)\n",
"phase": "sql_golden"
}
},
@@ -1913,24 +2827,37 @@
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `results`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c200773151fae81e0c0863ae9d0a4534b2934c3b4023363ffb2b7c1930309b46/sql)\n",
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `results`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37375/v1/database/c20091a026a6e24cbedfe348785ee1a0dfc5c34a7ba9189ffa505d8c187b1ac9/sql)\n",
"phase": "sql"
}
+ },
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "golden_db": "schema-t-012-spacetime-product-type-golden",
+ "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "http://127.0.0.1:37375",
+ "tables_diff": null,
+ "tables_equal": true
+ }
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:38:56.019269384Z",
- "finished_at": "2026-01-17T02:39:27.502611490Z"
+ "started_at": "2026-01-20T01:44:59.289433942Z",
+ "finished_at": "2026-01-20T01:45:25.879019400Z"
},
"t_013_spacetime_sum_type": {
- "hash": "7bd1056c239ec41df56d3be0edd5b7aac1c433acc3efbac0fe6f3b2a79ca2f1f",
+ "hash": "c9afa66dc818f616ea70c65180c235ac58b63ddcb485be6e6f173bb206a122d5",
"task": "t_013_spacetime_sum_type",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 3,
"passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetCircle(SpacetimeDB.ReducerContext ctx, int id, int radius)\n {\n var circle = new Circle { Radius = radius };\n var shape = new Shape.Circle(circle);\n ctx.Db.Result.Insert(new Result { Id = id, Value = shape });\n }\n}",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetCircle(ReducerContext ctx, int id, int radius)\n {\n var shape = new Shape.Circle(new Circle { Radius = radius });\n ctx.Db.Result.Insert(new Result { Id = id, Value = shape });\n }\n}",
"category": "schema",
"route_api_model": "gpt-5",
"golden_db": "schema-t-013-spacetime-sum-type-golden",
@@ -1942,10 +2869,18 @@
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `results`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c2009a86be1f0f78642d2ade147db198738ef88aa551d11bb76e42c1c8900e9a/sql)\n",
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `results`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37375/v1/database/c2001ba8a258a2548168b63dc5df5a0d8c5b5a039ddccee735c7d894475d29f5/sql)\n",
"phase": "sql"
}
},
+ "sum_type_row_parity": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `results`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37375/v1/database/c2001fa6fa22143c93406d60c7c8d72b3aba7fad66ca6789161989cae8535bdb/sql)\n",
+ "phase": "sql_golden"
+ }
+ },
"schema_parity": {
"pass": true,
"partial": 1.0,
@@ -1954,26 +2889,18 @@
"llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:35677",
+ "server": "http://127.0.0.1:37375",
"tables_diff": null,
"tables_equal": true
}
- },
- "sum_type_row_parity": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `results`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c200d82761d2c72dc9a864fd786af31b4c9ef43bb79b633450e8e561c3eea2ce/sql)\n",
- "phase": "sql_golden"
- }
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:40:02.303793183Z",
- "finished_at": "2026-01-17T02:40:51.462059047Z"
+ "started_at": "2026-01-20T01:44:59.294545896Z",
+ "finished_at": "2026-01-20T01:45:44.814268440Z"
},
"t_014_elementary_columns": {
- "hash": "7bd1056c239ec41df56d3be0edd5b7aac1c433acc3efbac0fe6f3b2a79ca2f1f",
+ "hash": "c9afa66dc818f616ea70c65180c235ac58b63ddcb485be6e6f173bb206a122d5",
"task": "t_014_elementary_columns",
"lang": "csharp",
"golden_published": true,
@@ -1992,7 +2919,7 @@
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitives`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c2003f9c20afc908b6f3fae929b8555ec0e90cf407b107f224de84971189e7b3/sql)\n",
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitives`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37375/v1/database/c2006e548a448e4a448582cfd67f2eaab13207dea03c90534493c91c319a2b33/sql)\n",
"phase": "sql_golden"
}
},
@@ -2000,7 +2927,7 @@
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitives`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c2009b868c8501f40d75f3046c80b4fa8bc7018d8baf65ff12c991bbab6d605a/sql)\n",
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitives`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37375/v1/database/c2004448c4bb2fdac69d279b105bbc78e720492071a8f12b30eba24659470387/sql)\n",
"phase": "sql"
}
},
@@ -2012,25 +2939,25 @@
"llm_db": "schema-t-014-elementary-columns-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:35677",
+ "server": "http://127.0.0.1:37375",
"tables_diff": null,
"tables_equal": true
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:38:36.670677504Z",
- "finished_at": "2026-01-17T02:39:15.092747325Z"
+ "started_at": "2026-01-20T01:44:59.292139426Z",
+ "finished_at": "2026-01-20T01:45:30.094932382Z"
},
"t_015_product_type_columns": {
- "hash": "7bd1056c239ec41df56d3be0edd5b7aac1c433acc3efbac0fe6f3b2a79ca2f1f",
+ "hash": "c9afa66dc818f616ea70c65180c235ac58b63ddcb485be6e6f173bb206a122d5",
"task": "t_015_product_type_columns",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 3,
"passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Address\n {\n public string Street;\n public int Zip;\n }\n\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Profile\", Public = true)]\n public partial struct Profile\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Address Home;\n public Address Work;\n public Position Pos;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Profile.Insert(new Profile\n {\n Id = 1,\n Home = new Address { Street = \"1 Main\", Zip = 11111 },\n Work = new Address { Street = \"2 Broad\", Zip = 22222 },\n Pos = new Position { X = 7, Y = 9 }\n });\n }\n}",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Address\n {\n public string Street;\n public int Zip;\n }\n\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Profile\", Public = true)]\n public partial struct Profile\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Address Home;\n public Address Work;\n public Position Pos;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n foreach (var row in ctx.Db.Profile.Iter())\n {\n ctx.Db.Profile.Id.Delete(row.Id);\n }\n\n ctx.Db.Profile.Insert(new Profile\n {\n Id = 1,\n Home = new Address { Street = \"1 Main\", Zip = 11111 },\n Work = new Address { Street = \"2 Broad\", Zip = 22222 },\n Pos = new Position { X = 7, Y = 9 }\n });\n }\n}",
"category": "schema",
"route_api_model": "gpt-5",
"golden_db": "schema-t-015-product-type-columns-golden",
@@ -2046,41 +2973,41 @@
"llm_db": "schema-t-015-product-type-columns-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:35677",
+ "server": "http://127.0.0.1:37375",
"tables_diff": null,
"tables_equal": true
}
},
- "product_type_columns_row_count": {
+ "product_type_columns_row_parity": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `profiles`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c2007bfa13561ce0c8cb477e4e11171bc3733d8ba01683aecd94ed2d3f231319/sql)\n",
- "phase": "sql"
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `profiles`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37375/v1/database/c200f4aef87c4d4177cc2dee268475a2712df95903187036756bb7679ea8d71d/sql)\n",
+ "phase": "sql_golden"
}
},
- "product_type_columns_row_parity": {
+ "product_type_columns_row_count": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `profiles`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c200e5ea78db1095b274afeafb8041e139a0d7f11834b194b863882d4a7c3328/sql)\n",
- "phase": "sql_golden"
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `profiles`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37375/v1/database/c200934c2b281e6bfdd45af65a2f7b3729112714ae5e763507eba1b3bf1281f7/sql)\n",
+ "phase": "sql"
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:39:15.092804541Z",
- "finished_at": "2026-01-17T02:40:02.303741426Z"
+ "started_at": "2026-01-20T01:45:25.879059692Z",
+ "finished_at": "2026-01-20T01:45:57.350652091Z"
},
"t_016_sum_type_columns": {
- "hash": "7bd1056c239ec41df56d3be0edd5b7aac1c433acc3efbac0fe6f3b2a79ca2f1f",
+ "hash": "c9afa66dc818f616ea70c65180c235ac58b63ddcb485be6e6f173bb206a122d5",
"task": "t_016_sum_type_columns",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 3,
"passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"Drawing\")]\n public partial struct Drawing\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape A;\n public Shape B;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Drawing.Insert(new Drawing\n {\n Id = 1,\n A = new Shape.Circle(new Circle { Radius = 10 }),\n B = new Shape.Rectangle(new Rectangle { Width = 4, Height = 6 })\n });\n }\n}",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"Drawing\", Public = true)]\n public partial struct Drawing\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape A;\n public Shape B;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n var a = new Shape.Circle(new Circle { Radius = 10 });\n var b = new Shape.Rectangle(new Rectangle { Width = 4, Height = 6 });\n ctx.Db.Drawing.Insert(new Drawing\n {\n Id = 1,\n A = a,\n B = b\n });\n }\n}",
"category": "schema",
"route_api_model": "gpt-5",
"golden_db": "schema-t-016-sum-type-columns-golden",
@@ -2088,49 +3015,49 @@
"work_dir_golden": "target/llm-runs/schema/t_016_sum_type_columns/csharp/server/golden",
"work_dir_llm": "target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm",
"scorer_details": {
- "sum_type_columns_row_parity": {
- "pass": false,
- "partial": 0.0,
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c2004da484f449e95616d5eb1c6832e7e06faf68e5ebbe454e63475d86740e6c/sql)\n",
- "phase": "sql_golden"
+ "golden_db": "schema-t-016-sum-type-columns-golden",
+ "llm_db": "schema-t-016-sum-type-columns-gpt-5-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "http://127.0.0.1:37375",
+ "tables_diff": null,
+ "tables_equal": true
}
},
"sum_type_columns_row_count": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c2008ad55b1dba50ea8bb36552e0bc9480b27d8060b57d8ed5e9e4d86cb6b5f8/sql)\n",
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37375/v1/database/c20012883db5294aeb48cc58b0bf586d7bceee6b9f5f6d8bbfb890dda90bfac5/sql)\n",
"phase": "sql"
}
},
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
+ "sum_type_columns_row_parity": {
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "golden_db": "schema-t-016-sum-type-columns-golden",
- "llm_db": "schema-t-016-sum-type-columns-gpt-5-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "http://127.0.0.1:35677",
- "tables_diff": null,
- "tables_equal": true
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37375/v1/database/c200a0d2cc9dc8b14351840e33df1960da4daa53109ac78d761949a1baaaf201/sql)\n",
+ "phase": "sql_golden"
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:39:27.502655041Z",
- "finished_at": "2026-01-17T02:40:24.282007063Z"
+ "started_at": "2026-01-20T01:45:30.174966802Z",
+ "finished_at": "2026-01-20T01:46:16.833371759Z"
},
"t_017_scheduled_columns": {
- "hash": "7bd1056c239ec41df56d3be0edd5b7aac1c433acc3efbac0fe6f3b2a79ca2f1f",
+ "hash": "c9afa66dc818f616ea70c65180c235ac58b63ddcb485be6e6f173bb206a122d5",
"task": "t_017_scheduled_columns",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 2,
"passed_tests": 1,
- "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [SpacetimeDB.Reducer]\n public static void Tick(ReducerContext ctx, TickTimer _timer)\n {\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMilliseconds(50))\n });\n }\n}",
+ "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [SpacetimeDB.Reducer]\n public static void Tick(ReducerContext ctx, TickTimer timer)\n {\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMicroseconds(50_000))\n });\n }\n}",
"category": "schema",
"route_api_model": "gpt-5",
"golden_db": "schema-t-017-scheduled-columns-golden",
@@ -2138,14 +3065,6 @@
"work_dir_golden": "target/llm-runs/schema/t_017_scheduled_columns/csharp/server/golden",
"work_dir_llm": "target/llm-runs/schema/t_017_scheduled_columns/csharp/server/gpt-5/llm",
"scorer_details": {
- "scheduled_seeded_one_row": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `tick_timer`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c20024b2c639ed44059f68c872d4ae8686e43def045ff88fb49f452645a17338/sql)\n",
- "phase": "sql"
- }
- },
"schema_parity": {
"pass": true,
"partial": 1.0,
@@ -2154,25 +3073,33 @@
"llm_db": "schema-t-017-scheduled-columns-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:35677",
+ "server": "http://127.0.0.1:37375",
"tables_diff": null,
"tables_equal": true
}
+ },
+ "scheduled_seeded_one_row": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `tick_timer`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37375/v1/database/c200725096303f79bcc46c94e18e69914684c51594a0f19278b5d50444f1a231/sql)\n",
+ "phase": "sql"
+ }
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:38:01.497464655Z",
- "finished_at": "2026-01-17T02:38:36.670468145Z"
+ "started_at": "2026-01-20T01:44:50.125868865Z",
+ "finished_at": "2026-01-20T01:45:30.174936935Z"
},
"t_018_constraints": {
- "hash": "7bd1056c239ec41df56d3be0edd5b7aac1c433acc3efbac0fe6f3b2a79ca2f1f",
+ "hash": "c9afa66dc818f616ea70c65180c235ac58b63ddcb485be6e6f173bb206a122d5",
"task": "t_018_constraints",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 3,
"passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Account\")]\n [SpacetimeDB.Index.BTree(Name = \"by_name\", Columns = [\"Name\"])]\n public partial struct Account\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n [SpacetimeDB.Unique]\n public string Email;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Account.Insert(new Account { Id = 1, Email = \"a@example.com\", Name = \"Alice\" });\n ctx.Db.Account.Insert(new Account { Id = 2, Email = \"b@example.com\", Name = \"Bob\" });\n }\n}",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Account\")]\n [SpacetimeDB.Index.BTree(Name = \"by_name\", Columns = [\"Name\"])]\n public partial struct Account\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n [SpacetimeDB.Unique]\n public string Email;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(SpacetimeDB.ReducerContext ctx)\n {\n ctx.Db.Account.Insert(new Account { Id = 1, Email = \"a@example.com\", Name = \"Alice\" });\n ctx.Db.Account.Insert(new Account { Id = 2, Email = \"b@example.com\", Name = \"Bob\" });\n }\n}",
"category": "schema",
"route_api_model": "gpt-5",
"golden_db": "schema-t-018-constraints-golden",
@@ -2180,6 +3107,14 @@
"work_dir_golden": "target/llm-runs/schema/t_018_constraints/csharp/server/golden",
"work_dir_llm": "target/llm-runs/schema/t_018_constraints/csharp/server/gpt-5/llm",
"scorer_details": {
+ "constraints_seed_two_rows": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `accounts`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37375/v1/database/c20041d01c6e91da51fa154658cf9d37c8b3baaac06f6e2aec892bea4a0ed66e/sql)\n",
+ "phase": "sql"
+ }
+ },
"schema_parity": {
"pass": true,
"partial": 1.0,
@@ -2188,7 +3123,7 @@
"llm_db": "schema-t-018-constraints-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:35677",
+ "server": "http://127.0.0.1:37375",
"tables_diff": null,
"tables_equal": true
}
@@ -2197,32 +3132,24 @@
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `accounts`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c200db49191274d7a3a7263567507f9945c4e802cb3173928c835966c8331803/sql)\n",
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `accounts`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37375/v1/database/c2003ec499a4de2e8d980575345f0ea33142838d7b40496b4ee7bdffca1602e0/sql)\n",
"phase": "sql_golden"
}
- },
- "constraints_seed_two_rows": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `accounts`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c20025629920e94d036ee70c22a88bf38265ab06fbb5cfc2c505e147018ad40c/sql)\n",
- "phase": "sql"
- }
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:37:43.947065043Z",
- "finished_at": "2026-01-17T02:38:24.847760366Z"
+ "started_at": "2026-01-20T01:44:21.031043080Z",
+ "finished_at": "2026-01-20T01:44:59.063845425Z"
},
"t_019_many_to_many": {
- "hash": "7bd1056c239ec41df56d3be0edd5b7aac1c433acc3efbac0fe6f3b2a79ca2f1f",
+ "hash": "c9afa66dc818f616ea70c65180c235ac58b63ddcb485be6e6f173bb206a122d5",
"task": "t_019_many_to_many",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 5,
"passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int UserId;\n public string Name;\n }\n\n [SpacetimeDB.Table(Name = \"Group\")]\n public partial struct Group\n {\n [SpacetimeDB.PrimaryKey]\n public int GroupId;\n public string Title;\n }\n\n [SpacetimeDB.Table(Name = \"Membership\")]\n [SpacetimeDB.Index.BTree(Name = \"by_user\", Columns = new[] { \"UserId\" })]\n [SpacetimeDB.Index.BTree(Name = \"by_group\", Columns = new[] { \"GroupId\" })]\n public partial struct Membership\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int GroupId;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { UserId = 1, Name = \"Alice\" });\n ctx.Db.User.Insert(new User { UserId = 2, Name = \"Bob\" });\n\n ctx.Db.Group.Insert(new Group { GroupId = 10, Title = \"Admin\" });\n ctx.Db.Group.Insert(new Group { GroupId = 20, Title = \"Dev\" });\n\n ctx.Db.Membership.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\n ctx.Db.Membership.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\n ctx.Db.Membership.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\n }\n}",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int UserId;\n public string Name;\n }\n\n [SpacetimeDB.Table(Name = \"Group\", Public = true)]\n public partial struct Group\n {\n [SpacetimeDB.PrimaryKey]\n public int GroupId;\n public string Title;\n }\n\n [SpacetimeDB.Table(Name = \"Membership\", Public = true)]\n [SpacetimeDB.Index.BTree(Name = \"by_user\", Columns = new[] { nameof(Membership.UserId) })]\n [SpacetimeDB.Index.BTree(Name = \"by_group\", Columns = new[] { nameof(Membership.GroupId) })]\n public partial struct Membership\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int GroupId;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n // Clear existing rows to ensure exactly these rows exist\n foreach (var m in ctx.Db.Membership.Iter())\n {\n ctx.Db.Membership.Id.Delete(m.Id);\n }\n foreach (var u in ctx.Db.User.Iter())\n {\n ctx.Db.User.UserId.Delete(u.UserId);\n }\n foreach (var g in ctx.Db.Group.Iter())\n {\n ctx.Db.Group.GroupId.Delete(g.GroupId);\n }\n\n // Insert Users\n ctx.Db.User.Insert(new User { UserId = 1, Name = \"Alice\" });\n ctx.Db.User.Insert(new User { UserId = 2, Name = \"Bob\" });\n\n // Insert Groups\n ctx.Db.Group.Insert(new Group { GroupId = 10, Title = \"Admin\" });\n ctx.Db.Group.Insert(new Group { GroupId = 20, Title = \"Dev\" });\n\n // Insert Memberships\n ctx.Db.Membership.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\n ctx.Db.Membership.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\n ctx.Db.Membership.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\n }\n}",
"category": "schema",
"route_api_model": "gpt-5",
"golden_db": "schema-t-019-many-to-many-golden",
@@ -2230,11 +3157,11 @@
"work_dir_golden": "target/llm-runs/schema/t_019_many_to_many/csharp/server/golden",
"work_dir_llm": "target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm",
"scorer_details": {
- "m2m_has_1_20": {
+ "memberships_three_rows": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `memberships`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c2008ea9cb58625a6494cb7b2f3998b558db44fc3308f07f5e48fe0a03dee66e/sql)\n",
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `memberships`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37375/v1/database/c2002f5232cb589d1d6e52337e5c7a657a3bc7e65d337593f4e1d573c32c8c0d/sql)\n",
"phase": "sql"
}
},
@@ -2242,53 +3169,53 @@
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `memberships`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c2008ea9cb58625a6494cb7b2f3998b558db44fc3308f07f5e48fe0a03dee66e/sql)\n",
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `memberships`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37375/v1/database/c2002f5232cb589d1d6e52337e5c7a657a3bc7e65d337593f4e1d573c32c8c0d/sql)\n",
"phase": "sql"
}
},
- "memberships_three_rows": {
- "pass": false,
- "partial": 0.0,
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `memberships`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c2008ea9cb58625a6494cb7b2f3998b558db44fc3308f07f5e48fe0a03dee66e/sql)\n",
- "phase": "sql"
+ "golden_db": "schema-t-019-many-to-many-golden",
+ "llm_db": "schema-t-019-many-to-many-gpt-5-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "http://127.0.0.1:37375",
+ "tables_diff": null,
+ "tables_equal": true
}
},
"m2m_has_1_10": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `memberships`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c2008ea9cb58625a6494cb7b2f3998b558db44fc3308f07f5e48fe0a03dee66e/sql)\n",
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `memberships`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37375/v1/database/c2002f5232cb589d1d6e52337e5c7a657a3bc7e65d337593f4e1d573c32c8c0d/sql)\n",
"phase": "sql"
}
},
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
+ "m2m_has_1_20": {
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "golden_db": "schema-t-019-many-to-many-golden",
- "llm_db": "schema-t-019-many-to-many-gpt-5-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "http://127.0.0.1:35677",
- "tables_diff": null,
- "tables_equal": true
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `memberships`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37375/v1/database/c2002f5232cb589d1d6e52337e5c7a657a3bc7e65d337593f4e1d573c32c8c0d/sql)\n",
+ "phase": "sql"
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:36:42.262573385Z",
- "finished_at": "2026-01-17T02:37:43.946992007Z"
+ "started_at": "2026-01-20T01:44:23.181428365Z",
+ "finished_at": "2026-01-20T01:44:58.917328881Z"
},
"t_020_ecs": {
- "hash": "7bd1056c239ec41df56d3be0edd5b7aac1c433acc3efbac0fe6f3b2a79ca2f1f",
+ "hash": "c9afa66dc818f616ea70c65180c235ac58b63ddcb485be6e6f173bb206a122d5",
"task": "t_020_ecs",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 5,
"passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Entity\")]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n }\n\n [SpacetimeDB.Table(Name = \"Position\")]\n public partial struct Position\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Velocity\")]\n public partial struct Velocity\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int VX;\n public int VY;\n }\n\n [SpacetimeDB.Table(Name = \"NextPosition\")]\n public partial struct NextPosition\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n // Entity 1\n ctx.Db.Entity.Insert(new Entity { Id = 1 });\n ctx.Db.Position.Insert(new Position { EntityId = 1, X = 0, Y = 0 });\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 1, VX = 1, VY = 0 });\n\n // Entity 2\n ctx.Db.Entity.Insert(new Entity { Id = 2 });\n ctx.Db.Position.Insert(new Position { EntityId = 2, X = 10, Y = 0 });\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 2, VX = -2, VY = 3 });\n }\n\n [SpacetimeDB.Reducer]\n public static void Step(ReducerContext ctx)\n {\n foreach (var pos in ctx.Db.Position.Iter())\n {\n var vel = ctx.Db.Velocity.EntityId.Find(pos.EntityId);\n if (vel == null)\n {\n continue;\n }\n\n var nx = pos.X + vel.Value.VX;\n var ny = pos.Y + vel.Value.VY;\n\n var existing = ctx.Db.NextPosition.EntityId.Find(pos.EntityId);\n if (existing != null)\n {\n var row = existing.Value;\n row.X = nx;\n row.Y = ny;\n ctx.Db.NextPosition.EntityId.Update(row);\n }\n else\n {\n ctx.Db.NextPosition.Insert(new NextPosition\n {\n EntityId = pos.EntityId,\n X = nx,\n Y = ny\n });\n }\n }\n }\n}",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Entity\")]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n }\n\n [SpacetimeDB.Table(Name = \"Position\")]\n public partial struct Position\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Velocity\")]\n public partial struct Velocity\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int VX;\n public int VY;\n }\n\n [SpacetimeDB.Table(Name = \"NextPosition\")]\n public partial struct NextPosition\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n // Entity 1\n ctx.Db.Entity.Insert(new Entity { Id = 1 });\n ctx.Db.Position.Insert(new Position { EntityId = 1, X = 0, Y = 0 });\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 1, VX = 1, VY = 0 });\n\n // Entity 2\n ctx.Db.Entity.Insert(new Entity { Id = 2 });\n ctx.Db.Position.Insert(new Position { EntityId = 2, X = 10, Y = 0 });\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 2, VX = -2, VY = 3 });\n }\n\n [SpacetimeDB.Reducer]\n public static void Step(ReducerContext ctx)\n {\n foreach (var pos in ctx.Db.Position.Iter())\n {\n var velOpt = ctx.Db.Velocity.EntityId.Find(pos.EntityId);\n int vx = 0, vy = 0;\n if (velOpt.HasValue)\n {\n var v = velOpt.Value;\n vx = v.VX;\n vy = v.VY;\n }\n\n int nx = pos.X + vx;\n int ny = pos.Y + vy;\n\n var nextOpt = ctx.Db.NextPosition.EntityId.Find(pos.EntityId);\n if (nextOpt.HasValue)\n {\n var np = nextOpt.Value;\n np.X = nx;\n np.Y = ny;\n ctx.Db.NextPosition.EntityId.Update(np);\n }\n else\n {\n ctx.Db.NextPosition.Insert(new NextPosition\n {\n EntityId = pos.EntityId,\n X = nx,\n Y = ny\n });\n }\n }\n }\n}",
"category": "schema",
"route_api_model": "gpt-5",
"golden_db": "schema-t-020-ecs-golden",
@@ -2300,23 +3227,7 @@
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c20091a11c03f51be06cd18ee4b36dc6b1e87d93f9897c479cb76452564e3605/sql)\n",
- "phase": "sql"
- }
- },
- "ecs_seed_positions_count": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c20091a11c03f51be06cd18ee4b36dc6b1e87d93f9897c479cb76452564e3605/sql)\n",
- "phase": "sql"
- }
- },
- "ecs_next_pos_entity2": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c20091a11c03f51be06cd18ee4b36dc6b1e87d93f9897c479cb76452564e3605/sql)\n",
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37375/v1/database/c20087d836f66db15364b6185169e3c5915af99aec3092cc7c379115765be4fe/sql)\n",
"phase": "sql"
}
},
@@ -2324,7 +3235,7 @@
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c20091a11c03f51be06cd18ee4b36dc6b1e87d93f9897c479cb76452564e3605/sql)\n",
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37375/v1/database/c20087d836f66db15364b6185169e3c5915af99aec3092cc7c379115765be4fe/sql)\n",
"phase": "sql"
}
},
@@ -2336,18 +3247,34 @@
"llm_db": "schema-t-020-ecs-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:35677",
+ "server": "http://127.0.0.1:37375",
"tables_diff": null,
"tables_equal": true
}
+ },
+ "ecs_next_pos_entity2": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37375/v1/database/c20087d836f66db15364b6185169e3c5915af99aec3092cc7c379115765be4fe/sql)\n",
+ "phase": "sql"
+ }
+ },
+ "ecs_seed_positions_count": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37375/v1/database/c20087d836f66db15364b6185169e3c5915af99aec3092cc7c379115765be4fe/sql)\n",
+ "phase": "sql"
+ }
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:36:42.266606487Z",
- "finished_at": "2026-01-17T02:38:01.497277347Z"
+ "started_at": "2026-01-20T01:43:52.168519242Z",
+ "finished_at": "2026-01-20T01:44:59.289370726Z"
},
"t_021_multi_column_index": {
- "hash": "7bd1056c239ec41df56d3be0edd5b7aac1c433acc3efbac0fe6f3b2a79ca2f1f",
+ "hash": "c9afa66dc818f616ea70c65180c235ac58b63ddcb485be6e6f173bb206a122d5",
"task": "t_021_multi_column_index",
"lang": "csharp",
"golden_published": true,
@@ -2362,11 +3289,11 @@
"work_dir_golden": "target/llm-runs/schema/t_021_multi_column_index/csharp/server/golden",
"work_dir_llm": "target/llm-runs/schema/t_021_multi_column_index/csharp/server/gpt-5/llm",
"scorer_details": {
- "mcindex_lookup_u7_d1": {
+ "mcindex_lookup_u7_d2": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `logs`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c200d69219860461311e299808a41ebae9617218c9ae2e796bc344b6340cc257/sql)\n",
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `logs`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37375/v1/database/c20091e3ea03d149a0d10d2cb512cbbc0eed1364fc061f95a3457992179c58d4/sql)\n",
"phase": "sql"
}
},
@@ -2374,7 +3301,7 @@
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `logs`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c200d69219860461311e299808a41ebae9617218c9ae2e796bc344b6340cc257/sql)\n",
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `logs`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37375/v1/database/c20091e3ea03d149a0d10d2cb512cbbc0eed1364fc061f95a3457992179c58d4/sql)\n",
"phase": "sql"
}
},
@@ -2386,23 +3313,23 @@
"llm_db": "schema-t-021-multi-column-index-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
- "server": "http://127.0.0.1:35677",
+ "server": "http://127.0.0.1:37375",
"tables_diff": null,
"tables_equal": true
}
},
- "mcindex_lookup_u7_d2": {
+ "mcindex_lookup_u7_d1": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `logs`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35677/v1/database/c200d69219860461311e299808a41ebae9617218c9ae2e796bc344b6340cc257/sql)\n",
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `logs`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37375/v1/database/c20091e3ea03d149a0d10d2cb512cbbc0eed1364fc061f95a3457992179c58d4/sql)\n",
"phase": "sql"
}
}
},
"vendor": "openai",
- "started_at": "2026-01-17T02:38:24.847804258Z",
- "finished_at": "2026-01-17T02:38:56.019226354Z"
+ "started_at": "2026-01-20T01:44:09.136460600Z",
+ "finished_at": "2026-01-20T01:44:50.125826908Z"
}
}
}
diff --git a/docs/llms/docs-benchmark-summary.json b/docs/llms/docs-benchmark-summary.json
index 30e341ff193..63665cb705a 100644
--- a/docs/llms/docs-benchmark-summary.json
+++ b/docs/llms/docs-benchmark-summary.json
@@ -1,11 +1,11 @@
{
"version": 1,
- "generated_at": "2026-01-17T02:44:26.905Z",
+ "generated_at": "2026-01-20T01:46:16.880Z",
"by_language": {
"csharp": {
"modes": {
"docs": {
- "hash": "7bd1056c239ec41df56d3be0edd5b7aac1c433acc3efbac0fe6f3b2a79ca2f1f",
+ "hash": "c9afa66dc818f616ea70c65180c235ac58b63ddcb485be6e6f173bb206a122d5",
"models": {
"GPT-5": {
"categories": {
@@ -41,6 +41,39 @@
},
"rust": {
"modes": {
+ "docs": {
+ "hash": "1df24d7e0efd9d88ff9cc77fa5c951ac075acc22e1c3cd844adb37b87d805489",
+ "models": {
+ "GPT-5": {
+ "categories": {
+ "basics": {
+ "tasks": 12,
+ "total_tests": 27,
+ "passed_tests": 5,
+ "pass_pct": 18.518518,
+ "task_pass_equiv": 1.3333334,
+ "task_pass_pct": 11.111112
+ },
+ "schema": {
+ "tasks": 10,
+ "total_tests": 34,
+ "passed_tests": 8,
+ "pass_pct": 23.529411,
+ "task_pass_equiv": 2.05,
+ "task_pass_pct": 20.5
+ }
+ },
+ "totals": {
+ "tasks": 22,
+ "total_tests": 61,
+ "passed_tests": 13,
+ "pass_pct": 21.311476,
+ "task_pass_equiv": 3.3833334,
+ "task_pass_pct": 15.378788
+ }
+ }
+ }
+ },
"rustdoc_json": {
"hash": "40baf0d53a9de9eab21250fad1d96f530213fd7dd09cb9a3d97f40ff548f0d8a",
"models": {
@@ -49,27 +82,27 @@
"basics": {
"tasks": 12,
"total_tests": 27,
- "passed_tests": 25,
- "pass_pct": 92.59259,
- "task_pass_equiv": 10.0,
- "task_pass_pct": 83.33333
+ "passed_tests": 9,
+ "pass_pct": 33.333332,
+ "task_pass_equiv": 4.333333,
+ "task_pass_pct": 36.111107
},
"schema": {
"tasks": 10,
"total_tests": 34,
- "passed_tests": 29,
- "pass_pct": 85.29412,
- "task_pass_equiv": 8.0,
- "task_pass_pct": 80.0
+ "passed_tests": 9,
+ "pass_pct": 26.470589,
+ "task_pass_equiv": 2.65,
+ "task_pass_pct": 26.500002
}
},
"totals": {
"tasks": 22,
"total_tests": 61,
- "passed_tests": 54,
- "pass_pct": 88.52459,
- "task_pass_equiv": 18.0,
- "task_pass_pct": 81.818184
+ "passed_tests": 18,
+ "pass_pct": 29.508196,
+ "task_pass_equiv": 6.9833336,
+ "task_pass_pct": 31.742426
}
}
}
diff --git a/docs/llms/llm-comparison-details.json b/docs/llms/llm-comparison-details.json
index 0c1fa6abf4e..560bdb80ca7 100644
--- a/docs/llms/llm-comparison-details.json
+++ b/docs/llms/llm-comparison-details.json
@@ -11,375 +11,328 @@
"name": "GPT-5",
"route_api_model": "gpt-5",
"tasks": {
- "t_019_many_to_many": {
+ "t_000_empty_reducers": {
"hash": "6f75b0c555fd7577df52872b447d9237496b7f004485c56464181b8bfb7834de",
- "task": "t_019_many_to_many",
+ "task": "t_000_empty_reducers",
"lang": "csharp",
- "golden_published": false,
+ "golden_published": true,
"model_name": "GPT-5",
- "total_tests": 5,
+ "total_tests": 1,
"passed_tests": 0,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"users\", Public = true)]\n public partial struct User\n {\n [PrimaryKey]\n public int UserId;\n public string Name;\n }\n\n [Table(Name = \"groups\", Public = true)]\n public partial struct Group\n {\n [PrimaryKey]\n public int GroupId;\n public string Title;\n }\n\n [Table(Name = \"memberships\", Public = true)]\n public partial struct Membership\n {\n [PrimaryKey]\n public int Id;\n\n [SpacetimeDB.Index.BTree(Name = \"by_user\")]\n public int UserId;\n\n [SpacetimeDB.Index.BTree(Name = \"by_group\")]\n public int GroupId;\n }\n\n [Reducer]\n public static void Seed(ReducerContext ctx)\n {\n // Clear existing rows to ensure the tables contain exactly the specified rows\n foreach (var m in ctx.Db.Membership.Iter())\n {\n ctx.Db.Membership.Id.Delete(m.Id);\n }\n foreach (var g in ctx.Db.Group.Iter())\n {\n ctx.Db.Group.GroupId.Delete(g.GroupId);\n }\n foreach (var u in ctx.Db.User.Iter())\n {\n ctx.Db.User.UserId.Delete(u.UserId);\n }\n\n // users\n ctx.Db.User.Insert(new User { UserId = 1, Name = \"Alice\" });\n ctx.Db.User.Insert(new User { UserId = 2, Name = \"Bob\" });\n\n // groups\n ctx.Db.Group.Insert(new Group { GroupId = 10, Title = \"Admin\" });\n ctx.Db.Group.Insert(new Group { GroupId = 20, Title = \"Dev\" });\n\n // memberships\n ctx.Db.Membership.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\n ctx.Db.Membership.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\n ctx.Db.Membership.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\n }\n}",
- "category": "schema",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_NoArgs(ReducerContext ctx)\n {\n }\n\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_WithInt(ReducerContext ctx, int count)\n {\n }\n\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_WithString(ReducerContext ctx, string name)\n {\n }\n\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_WithTwoArgs(ReducerContext ctx, int count, string name)\n {\n }\n\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_WithThreeArgs(ReducerContext ctx, bool active, float ratio, string label)\n {\n }\n}",
+ "category": "basics",
"route_api_model": "gpt-5",
- "golden_db": "schema-t-019-many-to-many-golden",
- "llm_db": "schema-t-019-many-to-many-gpt-5-llm",
- "work_dir_golden": "target/llm-runs/schema/t_019_many_to_many/csharp/server/golden",
- "work_dir_llm": "target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm",
+ "golden_db": "basics-t-000-empty-reducers-golden",
+ "llm_db": "basics-t-000-empty-reducers-gpt-5-llm",
+ "work_dir_golden": "target/llm-runs/basics/t_000_empty_reducers/csharp/server/golden",
+ "work_dir_llm": "target/llm-runs/basics/t_000_empty_reducers/csharp/server/gpt-5/llm",
"scorer_details": {
- "publish_error": {
+ "schema_parity": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/obj/Release/net8.0/wasi-wasm/SpacetimeDB.Codegen/SpacetimeDB.Codegen.Module/FFI.cs(155,24): warning CS8981: The type name 'users' only contains lower-cased ascii characters. Such names may become reserved for the language. [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/obj/Release/net8.0/wasi-wasm/SpacetimeDB.Codegen/SpacetimeDB.Codegen.Module/FFI.cs(70,24): warning CS8981: The type name 'memberships' only contains lower-cased ascii characters. Such names may become reserved for the language. [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/obj/Release/net8.0/wasi-wasm/SpacetimeDB.Codegen/SpacetimeDB.Codegen.Module/FFI.cs(27,32): warning CS8981: The type name 'groups' only contains lower-cased ascii characters. Such names may become reserved for the language. [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/Lib.cs(39,34): error CS1061: 'Local' does not contain a definition for 'Membership' and no accessible extension method 'Membership' accepting a first argument of type 'Local' could be found (are you missing a using directive or an assembly reference?) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/Lib.cs(41,20): error CS1061: 'Local' does not contain a definition for 'Membership' and no accessible extension method 'Membership' accepting a first argument of type 'Local' could be found (are you missing a using directive or an assembly reference?) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/Lib.cs(43,34): error CS1061: 'Local' does not contain a definition for 'Group' and no accessible extension method 'Group' accepting a first argument of type 'Local' could be found (are you missing a using directive or an assembly reference?) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/Lib.cs(45,20): error CS1061: 'Local' does not contain a definition for 'Group' and no accessible extension method 'Group' accepting a first argument of type 'Local' could be found (are you missing a using directive or an assembly reference?) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/Lib.cs(47,34): error CS1061: 'Local' does not contain a definition for 'User' and no accessible extension method 'User' accepting a first argument of type 'Local' could be found (are you missing a using directive or an assembly reference?) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/Lib.cs(49,20): error CS1061: 'Local' does not contain a definition for 'User' and no accessible extension method 'User' accepting a first argument of type 'Local' could be found (are you missing a using directive or an assembly reference?) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/Lib.cs(53,16): error CS1061: 'Local' does not contain a definition for 'User' and no accessible extension method 'User' accepting a first argument of type 'Local' could be found (are you missing a using directive or an assembly reference?) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/Lib.cs(54,16): error CS1061: 'Local' does not contain a definition for 'User' and no accessible extension method 'User' accepting a first argument of type 'Local' could be found (are you missing a using directive or an assembly reference?) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/Lib.cs(57,16): error CS1061: 'Local' does not contain a definition for 'Group' and no accessible extension method 'Group' accepting a first argument of type 'Local' could be found (are you missing a using directive or an assembly reference?) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/Lib.cs(58,16): error CS1061: 'Local' does not contain a definition for 'Group' and no accessible extension method 'Group' accepting a first argument of type 'Local' could be found (are you missing a using directive or an assembly reference?) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/Lib.cs(61,16): error CS1061: 'Local' does not contain a definition for 'Membership' and no accessible extension method 'Membership' accepting a first argument of type 'Local' could be found (are you missing a using directive or an assembly reference?) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/Lib.cs(62,16): error CS1061: 'Local' does not contain a definition for 'Membership' and no accessible extension method 'Membership' accepting a first argument of type 'Local' could be found (are you missing a using directive or an assembly reference?) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/Lib.cs(63,16): error CS1061: 'Local' does not contain a definition for 'Membership' and no accessible extension method 'Membership' accepting a first argument of type 'Local' could be found (are you missing a using directive or an assembly reference?) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/StdbModule.csproj]\n",
- "phase": "build_or_publish"
+ "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-000-empty-reducers-golden`.\n",
+ "phase": "describe_golden"
}
}
},
"vendor": "openai",
- "started_at": "2026-01-04T18:02:06.130072420Z",
- "finished_at": "2026-01-04T18:03:19.445236666Z"
+ "started_at": "2026-01-04T18:03:02.314835122Z",
+ "finished_at": "2026-01-04T18:03:49.955356015Z"
},
- "t_005_update": {
+ "t_001_basic_tables": {
"hash": "6f75b0c555fd7577df52872b447d9237496b7f004485c56464181b8bfb7834de",
- "task": "t_005_update",
+ "task": "t_001_basic_tables",
"lang": "csharp",
- "golden_published": true,
+ "golden_published": false,
"model_name": "GPT-5",
- "total_tests": 3,
+ "total_tests": 1,
"passed_tests": 0,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n if (ctx.Db.users.Id.Find(id) is User u)\n {\n u.Id = id;\n u.Name = name;\n u.Age = age;\n u.Active = active;\n ctx.Db.users.Id.Update(u);\n }\n }\n}",
+ "llm_output": null,
"category": "basics",
"route_api_model": "gpt-5",
- "golden_db": "basics-t-005-update-golden",
- "llm_db": "basics-t-005-update-gpt-5-llm",
- "work_dir_golden": "target/llm-runs/basics/t_005_update/csharp/server/golden",
- "work_dir_llm": "target/llm-runs/basics/t_005_update/csharp/server/gpt-5/llm",
+ "golden_db": null,
+ "llm_db": null,
+ "work_dir_golden": null,
+ "work_dir_llm": null,
"scorer_details": {
- "seed_users_row": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-005-update-golden`.\n",
- "phase": "sql_golden",
- "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
- }
- },
- "schema_parity": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-005-update-golden`.\n",
- "phase": "describe_golden"
- }
- },
- "data_parity_update_user": {
+ "publish_error": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-005-update-golden`.\n",
- "phase": "call_reducer_golden"
+ "error": "LLM call timed out",
+ "phase": "build_or_publish"
}
}
},
"vendor": "openai",
- "started_at": "2026-01-04T18:03:32.117137873Z",
- "finished_at": "2026-01-04T18:04:51.774335555Z"
+ "started_at": "2026-01-04T18:07:17.772342092Z",
+ "finished_at": "2026-01-04T18:07:17.772342092Z"
},
- "t_015_product_type_columns": {
+ "t_002_scheduled_table": {
"hash": "6f75b0c555fd7577df52872b447d9237496b7f004485c56464181b8bfb7834de",
- "task": "t_015_product_type_columns",
+ "task": "t_002_scheduled_table",
"lang": "csharp",
- "golden_published": false,
+ "golden_published": true,
"model_name": "GPT-5",
- "total_tests": 3,
+ "total_tests": 1,
"passed_tests": 0,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Address\n {\n public string Street;\n public int Zip;\n }\n\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"profiles\", Public = true)]\n public partial struct Profile\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Address Home;\n public Address Work;\n public Position Pos;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.profiles.Insert(new Profile\n {\n Id = 1,\n Home = new Address { Street = \"1 Main\", Zip = 11111 },\n Work = new Address { Street = \"2 Broad\", Zip = 22222 },\n Pos = new Position { X = 7, Y = 9 }\n });\n }\n}",
- "category": "schema",
+ "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"tick_timer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [PrimaryKey, AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.tick_timer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMilliseconds(50))\n });\n }\n\n [Reducer]\n public static void Tick(ReducerContext ctx, TickTimer _timer)\n {\n }\n}",
+ "category": "basics",
"route_api_model": "gpt-5",
- "golden_db": "schema-t-015-product-type-columns-golden",
- "llm_db": "schema-t-015-product-type-columns-gpt-5-llm",
- "work_dir_golden": "target/llm-runs/schema/t_015_product_type_columns/csharp/server/golden",
- "work_dir_llm": "target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm",
+ "golden_db": "basics-t-002-scheduled-table-golden",
+ "llm_db": "basics-t-002-scheduled-table-gpt-5-llm",
+ "work_dir_golden": "target/llm-runs/basics/t_002_scheduled_table/csharp/server/golden",
+ "work_dir_llm": "target/llm-runs/basics/t_002_scheduled_table/csharp/server/gpt-5/llm",
"scorer_details": {
- "publish_error": {
+ "schema_parity": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\n\nInstalling pack Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk version 8.0.22...\nSkipping NuGet package signature verification.\nWriting workload pack installation record for Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk version 8.0.22...\nInstalling pack Microsoft.NETCore.App.Runtime.Mono.wasi-wasm version 8.0.22...\nWriting workload pack installation record for Microsoft.NETCore.App.Runtime.Mono.wasi-wasm version 8.0.22...\nInstalling pack Microsoft.NET.Runtime.WebAssembly.Templates version 8.0.22...\nPack Microsoft.NET.Runtime.WebAssembly.Templates version 8.0.22 is already installed.\nWriting workload pack installation record for Microsoft.NET.Runtime.WebAssembly.Templates version 8.0.22...\nInstalling pack Microsoft.NET.Runtime.MonoAOTCompiler.Task version 8.0.22...\nPack Microsoft.NET.Runtime.MonoAOTCompiler.Task version 8.0.22 is already installed.\nWriting workload pack installation record for Microsoft.NET.Runtime.MonoAOTCompiler.Task version 8.0.22...\nInstalling pack Microsoft.NET.Runtime.MonoTargets.Sdk version 8.0.22...\nPack Microsoft.NET.Runtime.MonoTargets.Sdk version 8.0.22 is already installed.\nWriting workload pack installation record for Microsoft.NET.Runtime.MonoTargets.Sdk version 8.0.22...\nGarbage collecting for SDK feature band(s) 8.0.100 8.0.200 8.0.300 8.0.400 9.0.100 9.0.200 9.0.300 10.0.100...\n\nSuccessfully installed workload(s) .\n\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/obj/Release/net8.0/wasi-wasm/SpacetimeDB.Codegen/SpacetimeDB.Codegen.Module/FFI.cs(27,32): warning CS8981: The type name 'profiles' only contains lower-cased ascii characters. Such names may become reserved for the language. [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: The \"EmitBundleObjectFiles\" task failed unexpectedly. [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: System.AggregateException: One or more errors occurred. (Pipe is broken.) (Pipe is broken.) (Pipe is broken.) (Pipe is broken.) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: ---> System.IO.IOException: Pipe is broken. [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.Pipes.PipeStream.CheckWriteOperations() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.StreamWriter.Flush(Boolean flushStream, Boolean flushEncoder) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.StreamWriter.Dispose(Boolean disposing) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.TextWriter.Dispose() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.<>c__DisplayClass27_1.b__2(Stream codeStream) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at Utils.TryRunProcess(TaskLoggingHelper logger, String path, String args, IDictionary`2 envVars, String workingDir, Boolean silent, Boolean logStdErrAsMessage, MessageImportance debugMessageImportance, String label, Action`1 inputProvider) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleObjectFiles.EmitBundleFile(String destinationFile, Action`1 EmitBundleFile) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.<>c__DisplayClass27_0.b__1(Int32 i, ParallelLoopState state) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.<>c__DisplayClass19_0`2.b__1(RangeWorker& currentWorker, Int64 timeout, Boolean& replicationDelegateYieldedBeforeCompletion) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: --- End of stack trace from previous location --- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.<>c__DisplayClass19_0`2.b__1(RangeWorker& currentWorker, Int64 timeout, Boolean& replicationDelegateYieldedBeforeCompletion) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.TaskReplicator.Replica.Execute() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: --- End of inner exception stack trace --- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.TaskReplicator.Run[TState](ReplicatableUserAction`1 action, ParallelOptions options, Boolean stopOnFirstFailure) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.ForWorker[TLocal,TInt](TInt fromInclusive, TInt toExclusive, ParallelOptions parallelOptions, Action`1 body, Action`2 bodyWithState, Func`4 bodyWithLocal, Func`1 localInit, Action`1 localFinally) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: --- End of stack trace from previous location --- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.ForWorker[TLocal,TInt](TInt fromInclusive, TInt toExclusive, ParallelOptions parallelOptions, Action`1 body, Action`2 bodyWithState, Func`4 bodyWithLocal, Func`1 localInit, Action`1 localFinally) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.For(Int32 fromInclusive, Int32 toExclusive, ParallelOptions parallelOptions, Action`2 body) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.Execute() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleObjectFiles.Execute() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at Microsoft.Build.BackEnd.TaskExecutionHost.Execute() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at Microsoft.Build.BackEnd.TaskBuilder.ExecuteInstantiatedTask(TaskExecutionHost taskExecutionHost, TaskLoggingContext taskLoggingContext, TaskHost taskHost, ItemBucket bucket, TaskExecutionMode howToExecuteTask) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: ---> (Inner Exception #1) System.IO.IOException: Pipe is broken. [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.Pipes.PipeStream.CheckWriteOperations() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.StreamWriter.Flush(Boolean flushStream, Boolean flushEncoder) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.StreamWriter.Dispose(Boolean disposing) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.TextWriter.Dispose() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.<>c__DisplayClass27_1.b__2(Stream codeStream) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at Utils.TryRunProcess(TaskLoggingHelper logger, String path, String args, IDictionary`2 envVars, String workingDir, Boolean silent, Boolean logStdErrAsMessage, MessageImportance debugMessageImportance, String label, Action`1 inputProvider) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleObjectFiles.EmitBundleFile(String destinationFile, Action`1 EmitBundleFile) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.<>c__DisplayClass27_0.b__1(Int32 i, ParallelLoopState state) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.<>c__DisplayClass19_0`2.b__1(RangeWorker& currentWorker, Int64 timeout, Boolean& replicationDelegateYieldedBeforeCompletion) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: --- End of stack trace from previous location --- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.<>c__DisplayClass19_0`2.b__1(RangeWorker& currentWorker, Int64 timeout, Boolean& replicationDelegateYieldedBeforeCompletion) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.TaskReplicator.Replica.Execute()<--- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: ---> (Inner Exception #2) System.IO.IOException: Pipe is broken. [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.Pipes.PipeStream.CheckWriteOperations() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.StreamWriter.Flush(Boolean flushStream, Boolean flushEncoder) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.StreamWriter.Dispose(Boolean disposing) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.TextWriter.Dispose() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.<>c__DisplayClass27_1.b__2(Stream codeStream) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at Utils.TryRunProcess(TaskLoggingHelper logger, String path, String args, IDictionary`2 envVars, String workingDir, Boolean silent, Boolean logStdErrAsMessage, MessageImportance debugMessageImportance, String label, Action`1 inputProvider) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleObjectFiles.EmitBundleFile(String destinationFile, Action`1 EmitBundleFile) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.<>c__DisplayClass27_0.b__1(Int32 i, ParallelLoopState state) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.<>c__DisplayClass19_0`2.b__1(RangeWorker& currentWorker, Int64 timeout, Boolean& replicationDelegateYieldedBeforeCompletion) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: --- End of stack trace from previous location --- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.<>c__DisplayClass19_0`2.b__1(RangeWorker& currentWorker, Int64 timeout, Boolean& replicationDelegateYieldedBeforeCompletion) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.TaskReplicator.Replica.Execute()<--- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: ---> (Inner Exception #3) System.IO.IOException: Pipe is broken. [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.Pipes.PipeStream.CheckWriteOperations() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.StreamWriter.Flush(Boolean flushStream, Boolean flushEncoder) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.StreamWriter.Dispose(Boolean disposing) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.TextWriter.Dispose() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.<>c__DisplayClass27_1.b__2(Stream codeStream) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at Utils.TryRunProcess(TaskLoggingHelper logger, String path, String args, IDictionary`2 envVars, String workingDir, Boolean silent, Boolean logStdErrAsMessage, MessageImportance debugMessageImportance, String label, Action`1 inputProvider) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleObjectFiles.EmitBundleFile(String destinationFile, Action`1 EmitBundleFile) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.<>c__DisplayClass27_0.b__1(Int32 i, ParallelLoopState state) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.<>c__DisplayClass19_0`2.b__1(RangeWorker& currentWorker, Int64 timeout, Boolean& replicationDelegateYieldedBeforeCompletion) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: --- End of stack trace from previous location --- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.<>c__DisplayClass19_0`2.b__1(RangeWorker& currentWorker, Int64 timeout, Boolean& replicationDelegateYieldedBeforeCompletion) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.TaskReplicator.Replica.Execute()<--- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n",
- "phase": "build_or_publish"
+ "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-002-scheduled-table-golden`.\n",
+ "phase": "describe_golden"
}
}
},
"vendor": "openai",
- "started_at": "2026-01-04T18:02:06.160910108Z",
- "finished_at": "2026-01-04T18:03:02.247345187Z"
+ "started_at": "2026-01-04T18:03:32.125218903Z",
+ "finished_at": "2026-01-04T18:04:51.921266018Z"
},
- "t_008_index_lookup": {
+ "t_003_struct_in_table": {
"hash": "6f75b0c555fd7577df52872b447d9237496b7f004485c56464181b8bfb7834de",
- "task": "t_008_index_lookup",
+ "task": "t_003_struct_in_table",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-5",
- "total_tests": 3,
+ "total_tests": 1,
"passed_tests": 0,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"results\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void LookupUserName(ReducerContext ctx, int id)\n {\n if (ctx.Db.users.Id.Find(id) is User user)\n {\n ctx.Db.results.Insert(new Result\n {\n Id = user.Id,\n Name = user.Name\n });\n }\n }\n}",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n\n public Position(int X, int Y)\n {\n this.X = X;\n this.Y = Y;\n }\n }\n\n [SpacetimeDB.Table(Name = \"entities\", Public = true)]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Position Pos;\n }\n}",
"category": "basics",
"route_api_model": "gpt-5",
- "golden_db": "basics-t-008-index-lookup-golden",
- "llm_db": "basics-t-008-index-lookup-gpt-5-llm",
- "work_dir_golden": "target/llm-runs/basics/t_008_index_lookup/csharp/server/golden",
- "work_dir_llm": "target/llm-runs/basics/t_008_index_lookup/csharp/server/gpt-5/llm",
+ "golden_db": "basics-t-003-struct-in-table-golden",
+ "llm_db": "basics-t-003-struct-in-table-gpt-5-llm",
+ "work_dir_golden": "target/llm-runs/basics/t_003_struct_in_table/csharp/server/golden",
+ "work_dir_llm": "target/llm-runs/basics/t_003_struct_in_table/csharp/server/gpt-5/llm",
"scorer_details": {
"schema_parity": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-008-index-lookup-golden`.\n",
+ "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-003-struct-in-table-golden`.\n",
"phase": "describe_golden"
}
- },
- "index_lookup_projection_parity": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-008-index-lookup-golden`.\n",
- "phase": "call_reducer_golden"
- }
- },
- "seed_user_row": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-008-index-lookup-golden`.\n",
- "phase": "sql_golden",
- "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
- }
}
},
"vendor": "openai",
- "started_at": "2026-01-04T18:03:19.445262915Z",
- "finished_at": "2026-01-04T18:04:40.032489470Z"
+ "started_at": "2026-01-04T18:03:49.955467773Z",
+ "finished_at": "2026-01-04T18:05:04.199024592Z"
},
- "t_020_ecs": {
+ "t_004_insert": {
"hash": "6f75b0c555fd7577df52872b447d9237496b7f004485c56464181b8bfb7834de",
- "task": "t_020_ecs",
+ "task": "t_004_insert",
"lang": "csharp",
- "golden_published": false,
+ "golden_published": true,
"model_name": "GPT-5",
- "total_tests": 5,
+ "total_tests": 2,
"passed_tests": 0,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"entities\", Public = true)]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n }\n\n [SpacetimeDB.Table(Name = \"positions\", Public = true)]\n public partial struct Position\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"velocities\", Public = true)]\n public partial struct Velocity\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int VX;\n public int VY;\n }\n\n [SpacetimeDB.Table(Name = \"next_positions\", Public = true)]\n public partial struct NextPosition\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n // Entity 1\n ctx.Db.entities.Insert(new Entity\n {\n Id = 1\n });\n ctx.Db.positions.Insert(new Position\n {\n EntityId = 1,\n X = 0,\n Y = 0\n });\n ctx.Db.velocities.Insert(new Velocity\n {\n EntityId = 1,\n VX = 1,\n VY = 0\n });\n\n // Entity 2\n ctx.Db.entities.Insert(new Entity\n {\n Id = 2\n });\n ctx.Db.positions.Insert(new Position\n {\n EntityId = 2,\n X = 10,\n Y = 0\n });\n ctx.Db.velocities.Insert(new Velocity\n {\n EntityId = 2,\n VX = -2,\n VY = 3\n });\n }\n\n [SpacetimeDB.Reducer]\n public static void Step(ReducerContext ctx)\n {\n foreach (var pos in ctx.Db.positions.Iter())\n {\n if (ctx.Db.velocities.EntityId.Find(pos.EntityId) is Velocity vel)\n {\n var nx = pos.X + vel.VX;\n var ny = pos.Y + vel.VY;\n\n if (ctx.Db.next_positions.EntityId.Find(pos.EntityId) is NextPosition existing)\n {\n existing.X = nx;\n existing.Y = ny;\n ctx.Db.next_positions.EntityId.Update(existing);\n }\n else\n {\n ctx.Db.next_positions.Insert(new NextPosition\n {\n EntityId = pos.EntityId,\n X = nx,\n Y = ny\n });\n }\n }\n }\n }\n}",
- "category": "schema",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void InsertUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n ctx.Db.users.Insert(new User\n {\n Id = id,\n Name = name,\n Age = age,\n Active = active\n });\n }\n}",
+ "category": "basics",
"route_api_model": "gpt-5",
- "golden_db": "schema-t-020-ecs-golden",
- "llm_db": "schema-t-020-ecs-gpt-5-llm",
- "work_dir_golden": "target/llm-runs/schema/t_020_ecs/csharp/server/golden",
- "work_dir_llm": "target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm",
+ "golden_db": "basics-t-004-insert-golden",
+ "llm_db": "basics-t-004-insert-gpt-5-llm",
+ "work_dir_golden": "target/llm-runs/basics/t_004_insert/csharp/server/golden",
+ "work_dir_llm": "target/llm-runs/basics/t_004_insert/csharp/server/gpt-5/llm",
"scorer_details": {
- "publish_error": {
+ "data_parity_insert_user": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/obj/Release/net8.0/wasi-wasm/SpacetimeDB.Codegen/SpacetimeDB.Codegen.Module/FFI.cs(27,32): warning CS8981: The type name 'entities' only contains lower-cased ascii characters. Such names may become reserved for the language. [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/obj/Release/net8.0/wasi-wasm/SpacetimeDB.Codegen/SpacetimeDB.Codegen.Module/FFI.cs(113,24): warning CS8981: The type name 'positions' only contains lower-cased ascii characters. Such names may become reserved for the language. [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/obj/Release/net8.0/wasi-wasm/SpacetimeDB.Codegen/SpacetimeDB.Codegen.Module/FFI.cs(156,24): warning CS8981: The type name 'velocities' only contains lower-cased ascii characters. Such names may become reserved for the language. [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: The \"EmitBundleObjectFiles\" task failed unexpectedly. [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: System.AggregateException: One or more errors occurred. (Pipe is broken.) (Pipe is broken.) (Pipe is broken.) (Pipe is broken.) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: ---> System.IO.IOException: Pipe is broken. [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.Pipes.PipeStream.CheckWriteOperations() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.StreamWriter.Flush(Boolean flushStream, Boolean flushEncoder) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.StreamWriter.Dispose(Boolean disposing) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.TextWriter.Dispose() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.<>c__DisplayClass27_1.b__2(Stream codeStream) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at Utils.TryRunProcess(TaskLoggingHelper logger, String path, String args, IDictionary`2 envVars, String workingDir, Boolean silent, Boolean logStdErrAsMessage, MessageImportance debugMessageImportance, String label, Action`1 inputProvider) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleObjectFiles.EmitBundleFile(String destinationFile, Action`1 EmitBundleFile) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.<>c__DisplayClass27_0.b__1(Int32 i, ParallelLoopState state) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.<>c__DisplayClass19_0`2.b__1(RangeWorker& currentWorker, Int64 timeout, Boolean& replicationDelegateYieldedBeforeCompletion) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: --- End of stack trace from previous location --- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.<>c__DisplayClass19_0`2.b__1(RangeWorker& currentWorker, Int64 timeout, Boolean& replicationDelegateYieldedBeforeCompletion) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.TaskReplicator.Replica.Execute() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: --- End of inner exception stack trace --- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.TaskReplicator.Run[TState](ReplicatableUserAction`1 action, ParallelOptions options, Boolean stopOnFirstFailure) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.ForWorker[TLocal,TInt](TInt fromInclusive, TInt toExclusive, ParallelOptions parallelOptions, Action`1 body, Action`2 bodyWithState, Func`4 bodyWithLocal, Func`1 localInit, Action`1 localFinally) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: --- End of stack trace from previous location --- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.ForWorker[TLocal,TInt](TInt fromInclusive, TInt toExclusive, ParallelOptions parallelOptions, Action`1 body, Action`2 bodyWithState, Func`4 bodyWithLocal, Func`1 localInit, Action`1 localFinally) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.For(Int32 fromInclusive, Int32 toExclusive, ParallelOptions parallelOptions, Action`2 body) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.Execute() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleObjectFiles.Execute() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at Microsoft.Build.BackEnd.TaskExecutionHost.Execute() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at Microsoft.Build.BackEnd.TaskBuilder.ExecuteInstantiatedTask(TaskExecutionHost taskExecutionHost, TaskLoggingContext taskLoggingContext, TaskHost taskHost, ItemBucket bucket, TaskExecutionMode howToExecuteTask) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: ---> (Inner Exception #1) System.IO.IOException: Pipe is broken. [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.Pipes.PipeStream.CheckWriteOperations() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.StreamWriter.Flush(Boolean flushStream, Boolean flushEncoder) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.StreamWriter.Dispose(Boolean disposing) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.TextWriter.Dispose() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.<>c__DisplayClass27_1.b__2(Stream codeStream) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at Utils.TryRunProcess(TaskLoggingHelper logger, String path, String args, IDictionary`2 envVars, String workingDir, Boolean silent, Boolean logStdErrAsMessage, MessageImportance debugMessageImportance, String label, Action`1 inputProvider) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleObjectFiles.EmitBundleFile(String destinationFile, Action`1 EmitBundleFile) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.<>c__DisplayClass27_0.b__1(Int32 i, ParallelLoopState state) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.<>c__DisplayClass19_0`2.b__1(RangeWorker& currentWorker, Int64 timeout, Boolean& replicationDelegateYieldedBeforeCompletion) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: --- End of stack trace from previous location --- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.<>c__DisplayClass19_0`2.b__1(RangeWorker& currentWorker, Int64 timeout, Boolean& replicationDelegateYieldedBeforeCompletion) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.TaskReplicator.Replica.Execute()<--- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: ---> (Inner Exception #2) System.IO.IOException: Pipe is broken. [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.Pipes.PipeStream.CheckWriteOperations() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.StreamWriter.Flush(Boolean flushStream, Boolean flushEncoder) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.StreamWriter.Dispose(Boolean disposing) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.TextWriter.Dispose() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.<>c__DisplayClass27_1.b__2(Stream codeStream) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at Utils.TryRunProcess(TaskLoggingHelper logger, String path, String args, IDictionary`2 envVars, String workingDir, Boolean silent, Boolean logStdErrAsMessage, MessageImportance debugMessageImportance, String label, Action`1 inputProvider) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleObjectFiles.EmitBundleFile(String destinationFile, Action`1 EmitBundleFile) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.<>c__DisplayClass27_0.b__1(Int32 i, ParallelLoopState state) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.<>c__DisplayClass19_0`2.b__1(RangeWorker& currentWorker, Int64 timeout, Boolean& replicationDelegateYieldedBeforeCompletion) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: --- End of stack trace from previous location --- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.<>c__DisplayClass19_0`2.b__1(RangeWorker& currentWorker, Int64 timeout, Boolean& replicationDelegateYieldedBeforeCompletion) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.TaskReplicator.Replica.Execute()<--- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: ---> (Inner Exception #3) System.IO.IOException: Pipe is broken. [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.Pipes.PipeStream.CheckWriteOperations() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.StreamWriter.Flush(Boolean flushStream, Boolean flushEncoder) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.StreamWriter.Dispose(Boolean disposing) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.TextWriter.Dispose() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.<>c__DisplayClass27_1.b__2(Stream codeStream) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at Utils.TryRunProcess(TaskLoggingHelper logger, String path, String args, IDictionary`2 envVars, String workingDir, Boolean silent, Boolean logStdErrAsMessage, MessageImportance debugMessageImportance, String label, Action`1 inputProvider) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleObjectFiles.EmitBundleFile(String destinationFile, Action`1 EmitBundleFile) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.<>c__DisplayClass27_0.b__1(Int32 i, ParallelLoopState state) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.<>c__DisplayClass19_0`2.b__1(RangeWorker& currentWorker, Int64 timeout, Boolean& replicationDelegateYieldedBeforeCompletion) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: --- End of stack trace from previous location --- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.<>c__DisplayClass19_0`2.b__1(RangeWorker& currentWorker, Int64 timeout, Boolean& replicationDelegateYieldedBeforeCompletion) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.TaskReplicator.Replica.Execute()<--- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n",
- "phase": "build_or_publish"
+ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-004-insert-golden`.\n",
+ "phase": "call_reducer_golden"
+ }
+ },
+ "schema_parity": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-004-insert-golden`.\n",
+ "phase": "describe_golden"
}
}
},
"vendor": "openai",
- "started_at": "2026-01-04T18:02:06.133458136Z",
- "finished_at": "2026-01-04T18:03:02.314808993Z"
+ "started_at": "2026-01-04T18:04:40.032531267Z",
+ "finished_at": "2026-01-04T18:05:19.104533291Z"
},
- "t_018_constraints": {
+ "t_005_update": {
"hash": "6f75b0c555fd7577df52872b447d9237496b7f004485c56464181b8bfb7834de",
- "task": "t_018_constraints",
+ "task": "t_005_update",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 3,
"passed_tests": 0,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"accounts\")]\n [SpacetimeDB.Index.BTree(Name = \"by_name\", Columns = new[] { \"Name\" })]\n public partial struct Account\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n [SpacetimeDB.Unique]\n public string Email;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.accounts.Insert(new Account { Id = 1, Email = \"a@example.com\", Name = \"Alice\" });\n ctx.Db.accounts.Insert(new Account { Id = 2, Email = \"b@example.com\", Name = \"Bob\" });\n }\n}",
- "category": "schema",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n if (ctx.Db.users.Id.Find(id) is User u)\n {\n u.Id = id;\n u.Name = name;\n u.Age = age;\n u.Active = active;\n ctx.Db.users.Id.Update(u);\n }\n }\n}",
+ "category": "basics",
"route_api_model": "gpt-5",
- "golden_db": "schema-t-018-constraints-golden",
- "llm_db": "schema-t-018-constraints-gpt-5-llm",
- "work_dir_golden": "target/llm-runs/schema/t_018_constraints/csharp/server/golden",
- "work_dir_llm": "target/llm-runs/schema/t_018_constraints/csharp/server/gpt-5/llm",
+ "golden_db": "basics-t-005-update-golden",
+ "llm_db": "basics-t-005-update-gpt-5-llm",
+ "work_dir_golden": "target/llm-runs/basics/t_005_update/csharp/server/golden",
+ "work_dir_llm": "target/llm-runs/basics/t_005_update/csharp/server/gpt-5/llm",
"scorer_details": {
- "constraints_seed_two_rows": {
+ "schema_parity": {
"pass": false,
"partial": 0.0,
"notes": {
- "actual": 0,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM accounts WHERE Id=2"
+ "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-005-update-golden`.\n",
+ "phase": "describe_golden"
}
},
- "schema_parity": {
+ "seed_users_row": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-018-constraints-golden`.\n",
- "phase": "describe_golden"
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-005-update-golden`.\n",
+ "phase": "sql_golden",
+ "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
}
},
- "constraints_row_parity_after_seed": {
+ "data_parity_update_user": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-018-constraints-golden`.\n",
+ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-005-update-golden`.\n",
"phase": "call_reducer_golden"
}
}
},
"vendor": "openai",
- "started_at": "2026-01-04T18:02:06.137382585Z",
- "finished_at": "2026-01-04T18:03:31.415276583Z"
+ "started_at": "2026-01-04T18:03:32.117137873Z",
+ "finished_at": "2026-01-04T18:04:51.774335555Z"
},
- "t_021_multi_column_index": {
+ "t_006_delete": {
"hash": "6f75b0c555fd7577df52872b447d9237496b7f004485c56464181b8bfb7834de",
- "task": "t_021_multi_column_index",
+ "task": "t_006_delete",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-5",
- "total_tests": 4,
- "passed_tests": 3,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"logs\", Public = true)]\n [SpacetimeDB.Index.BTree(Name = \"by_user_day\", Columns = new[] { \"UserId\", \"Day\" })]\n public partial struct Log\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int Day;\n public string Message;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.logs.Insert(new Log { Id = 1, UserId = 7, Day = 1, Message = \"a\" });\n ctx.Db.logs.Insert(new Log { Id = 2, UserId = 7, Day = 2, Message = \"b\" });\n ctx.Db.logs.Insert(new Log { Id = 3, UserId = 9, Day = 1, Message = \"c\" });\n }\n}",
- "category": "schema",
+ "total_tests": 3,
+ "passed_tests": 1,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void DeleteUser(ReducerContext ctx, int id)\n {\n ctx.Db.users.Id.Delete(id);\n }\n}",
+ "category": "basics",
"route_api_model": "gpt-5",
- "golden_db": "schema-t-021-multi-column-index-golden",
- "llm_db": "schema-t-021-multi-column-index-gpt-5-llm",
- "work_dir_golden": "target/llm-runs/schema/t_021_multi_column_index/csharp/server/golden",
- "work_dir_llm": "target/llm-runs/schema/t_021_multi_column_index/csharp/server/gpt-5/llm",
+ "golden_db": "basics-t-006-delete-golden",
+ "llm_db": "basics-t-006-delete-gpt-5-llm",
+ "work_dir_golden": "target/llm-runs/basics/t_006_delete/csharp/server/golden",
+ "work_dir_llm": "target/llm-runs/basics/t_006_delete/csharp/server/gpt-5/llm",
"scorer_details": {
- "mcindex_seed_count": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 3,
- "expected": 3,
- "sql": "SELECT COUNT(*) AS n FROM logs"
- }
- },
- "mcindex_lookup_u7_d1": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM logs WHERE UserId=7 AND Day=1"
- }
- },
"schema_parity": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-021-multi-column-index-golden`.\n",
+ "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-006-delete-golden`.\n",
"phase": "describe_golden"
}
},
- "mcindex_lookup_u7_d2": {
+ "delete_user_count_zero": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM logs WHERE UserId=7 AND Day=2"
+ "actual": 0,
+ "expected": 0,
+ "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=1"
}
- }
- },
- "vendor": "openai",
- "started_at": "2026-01-04T18:02:06.146746037Z",
- "finished_at": "2026-01-04T18:03:32.116692104Z"
- },
- "t_012_spacetime_product_type": {
- "hash": "6f75b0c555fd7577df52872b447d9237496b7f004485c56464181b8bfb7834de",
- "task": "t_012_spacetime_product_type",
- "lang": "csharp",
- "golden_published": false,
- "model_name": "GPT-5",
- "total_tests": 3,
- "passed_tests": 0,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Type]\n public partial struct Score\n {\n public int Left;\n public int Right;\n }\n\n [Table(Name = \"results\", Public = true)]\n public partial struct Result\n {\n [PrimaryKey]\n public int Id;\n public Score Value;\n }\n\n [Reducer]\n public static void SetScore(ReducerContext ctx, int id, int left, int right)\n {\n ctx.Db.results.Insert(new Result\n {\n Id = id,\n Value = new Score\n {\n Left = left,\n Right = right\n }\n });\n }\n}",
- "category": "schema",
- "route_api_model": "gpt-5",
- "golden_db": "schema-t-012-spacetime-product-type-golden",
- "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm",
- "work_dir_golden": "target/llm-runs/schema/t_012_spacetime_product_type/csharp/server/golden",
- "work_dir_llm": "target/llm-runs/schema/t_012_spacetime_product_type/csharp/server/gpt-5/llm",
- "scorer_details": {
- "publish_error": {
+ },
+ "seed_users_row": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nSystem.IO.IOException: The system cannot open the device or file specified. : 'NuGet-Migrations'\n at System.Threading.Mutex.CreateMutexCore(Boolean initiallyOwned, String name, Boolean& createdNew)\n at System.Threading.Mutex..ctor(Boolean initiallyOwned, String name)\n at NuGet.Common.Migrations.MigrationRunner.Run(String migrationsDirectory)\n at Microsoft.DotNet.Configurer.DotnetFirstTimeUseConfigurer.Configure()\n at Microsoft.DotNet.Cli.Program.ConfigureDotNetForFirstTimeUse(IFirstTimeUseNoticeSentinel firstTimeUseNoticeSentinel, IAspNetCertificateSentinel aspNetCertificateSentinel, IFileSentinel toolPathSentinel, Boolean isDotnetBeingInvokedFromNativeInstaller, DotnetFirstRunConfiguration dotnetFirstRunConfiguration, IEnvironmentProvider environmentProvider, Dictionary`2 performanceMeasurements)\n at Microsoft.DotNet.Cli.Program.ProcessArgs(String[] args, TimeSpan startupTime, ITelemetry telemetryClient)\n at Microsoft.DotNet.Cli.Program.Main(String[] args)\nError: command [\"dotnet\", \"workload\", \"list\"] exited with code 1\n\n--- stdout ---\n",
- "phase": "build_or_publish"
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-006-delete-golden`.\n",
+ "phase": "sql_golden",
+ "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
}
}
},
"vendor": "openai",
- "started_at": "2026-01-04T18:02:06.156122014Z",
- "finished_at": "2026-01-04T18:02:28.230519129Z"
+ "started_at": "2026-01-04T18:03:16.646286587Z",
+ "finished_at": "2026-01-04T18:04:28.771873798Z"
},
- "t_010_connect": {
+ "t_007_crud": {
"hash": "6f75b0c555fd7577df52872b447d9237496b7f004485c56464181b8bfb7834de",
- "task": "t_010_connect",
+ "task": "t_007_crud",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-5",
- "total_tests": 1,
- "passed_tests": 0,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"events\")]\n public partial struct Event\n {\n [SpacetimeDB.PrimaryKey, SpacetimeDB.AutoInc]\n public int Id;\n public string Kind;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientConnected)]\n public static void ClientConnected(ReducerContext ctx)\n {\n ctx.Db.events.Insert(new Event { Kind = \"connected\" });\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientDisconnected)]\n public static void ClientDisconnected(ReducerContext ctx)\n {\n ctx.Db.events.Insert(new Event { Kind = \"disconnected\" });\n }\n}",
+ "total_tests": 4,
+ "passed_tests": 1,
+ "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void Crud(ReducerContext ctx)\n {\n // insert (Id=1, Name=\"Alice\", Age=30, Active=true)\n ctx.Db.users.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n\n // insert (Id=2, Name=\"Bob\", Age=22, Active=false)\n ctx.Db.users.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n\n // update (Id=1, Name=\"Alice2\", Age=31, Active=false)\n var u1 = ctx.Db.users.Id.Find(1) ?? throw new Exception(\"User with Id=1 not found\");\n u1.Name = \"Alice2\";\n u1.Age = 31;\n u1.Active = false;\n ctx.Db.users.Id.Update(u1);\n\n // delete Id=2\n ctx.Db.users.Id.Delete(2);\n }\n}",
"category": "basics",
"route_api_model": "gpt-5",
- "golden_db": "basics-t-010-connect-golden",
- "llm_db": "basics-t-010-connect-gpt-5-llm",
- "work_dir_golden": "target/llm-runs/basics/t_010_connect/csharp/server/golden",
- "work_dir_llm": "target/llm-runs/basics/t_010_connect/csharp/server/gpt-5/llm",
+ "golden_db": "basics-t-007-crud-golden",
+ "llm_db": "basics-t-007-crud-gpt-5-llm",
+ "work_dir_golden": "target/llm-runs/basics/t_007_crud/csharp/server/golden",
+ "work_dir_llm": "target/llm-runs/basics/t_007_crud/csharp/server/gpt-5/llm",
"scorer_details": {
"schema_parity": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-010-connect-golden`.\n",
+ "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-007-crud-golden`.\n",
"phase": "describe_golden"
}
+ },
+ "crud_row_id1_parity": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-007-crud-golden`.\n",
+ "phase": "call_reducer_golden"
+ }
+ },
+ "crud_row_id2_deleted": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 0,
+ "expected": 0,
+ "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=2"
+ }
+ },
+ "crud_total_count_one": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "actual": 0,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM users"
+ }
}
},
"vendor": "openai",
- "started_at": "2026-01-04T18:04:28.771931334Z",
- "finished_at": "2026-01-04T18:05:16.349401352Z"
+ "started_at": "2026-01-04T18:03:02.557397448Z",
+ "finished_at": "2026-01-04T18:03:57.769578396Z"
},
- "t_006_delete": {
+ "t_008_index_lookup": {
"hash": "6f75b0c555fd7577df52872b447d9237496b7f004485c56464181b8bfb7834de",
- "task": "t_006_delete",
+ "task": "t_008_index_lookup",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 3,
- "passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void DeleteUser(ReducerContext ctx, int id)\n {\n ctx.Db.users.Id.Delete(id);\n }\n}",
+ "passed_tests": 0,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"results\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void LookupUserName(ReducerContext ctx, int id)\n {\n if (ctx.Db.users.Id.Find(id) is User user)\n {\n ctx.Db.results.Insert(new Result\n {\n Id = user.Id,\n Name = user.Name\n });\n }\n }\n}",
"category": "basics",
"route_api_model": "gpt-5",
- "golden_db": "basics-t-006-delete-golden",
- "llm_db": "basics-t-006-delete-gpt-5-llm",
- "work_dir_golden": "target/llm-runs/basics/t_006_delete/csharp/server/golden",
- "work_dir_llm": "target/llm-runs/basics/t_006_delete/csharp/server/gpt-5/llm",
+ "golden_db": "basics-t-008-index-lookup-golden",
+ "llm_db": "basics-t-008-index-lookup-gpt-5-llm",
+ "work_dir_golden": "target/llm-runs/basics/t_008_index_lookup/csharp/server/golden",
+ "work_dir_llm": "target/llm-runs/basics/t_008_index_lookup/csharp/server/gpt-5/llm",
"scorer_details": {
- "delete_user_count_zero": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 0,
- "expected": 0,
- "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=1"
- }
- },
- "seed_users_row": {
+ "seed_user_row": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-006-delete-golden`.\n",
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-008-index-lookup-golden`.\n",
"phase": "sql_golden",
"sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
}
@@ -388,101 +341,107 @@
"pass": false,
"partial": 0.0,
"notes": {
- "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-006-delete-golden`.\n",
+ "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-008-index-lookup-golden`.\n",
"phase": "describe_golden"
}
- }
- },
- "vendor": "openai",
- "started_at": "2026-01-04T18:03:16.646286587Z",
- "finished_at": "2026-01-04T18:04:28.771873798Z"
- },
- "t_001_basic_tables": {
- "hash": "6f75b0c555fd7577df52872b447d9237496b7f004485c56464181b8bfb7834de",
- "task": "t_001_basic_tables",
- "lang": "csharp",
- "golden_published": false,
- "model_name": "GPT-5",
- "total_tests": 1,
- "passed_tests": 0,
- "llm_output": null,
- "category": "basics",
- "route_api_model": "gpt-5",
- "golden_db": null,
- "llm_db": null,
- "work_dir_golden": null,
- "work_dir_llm": null,
- "scorer_details": {
- "publish_error": {
+ },
+ "index_lookup_projection_parity": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "LLM call timed out",
- "phase": "build_or_publish"
+ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-008-index-lookup-golden`.\n",
+ "phase": "call_reducer_golden"
}
}
},
"vendor": "openai",
- "started_at": "2026-01-04T18:07:17.772342092Z",
- "finished_at": "2026-01-04T18:07:17.772342092Z"
+ "started_at": "2026-01-04T18:03:19.445262915Z",
+ "finished_at": "2026-01-04T18:04:40.032489470Z"
},
- "t_002_scheduled_table": {
+ "t_009_init": {
"hash": "6f75b0c555fd7577df52872b447d9237496b7f004485c56464181b8bfb7834de",
- "task": "t_002_scheduled_table",
+ "task": "t_009_init",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-5",
- "total_tests": 1,
- "passed_tests": 0,
- "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"tick_timer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [PrimaryKey, AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.tick_timer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMilliseconds(50))\n });\n }\n\n [Reducer]\n public static void Tick(ReducerContext ctx, TickTimer _timer)\n {\n }\n}",
+ "total_tests": 4,
+ "passed_tests": 3,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer(SpacetimeDB.ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.users.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.users.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n }\n}",
"category": "basics",
"route_api_model": "gpt-5",
- "golden_db": "basics-t-002-scheduled-table-golden",
- "llm_db": "basics-t-002-scheduled-table-gpt-5-llm",
- "work_dir_golden": "target/llm-runs/basics/t_002_scheduled_table/csharp/server/golden",
- "work_dir_llm": "target/llm-runs/basics/t_002_scheduled_table/csharp/server/gpt-5/llm",
+ "golden_db": "basics-t-009-init-golden",
+ "llm_db": "basics-t-009-init-gpt-5-llm",
+ "work_dir_golden": "target/llm-runs/basics/t_009_init/csharp/server/golden",
+ "work_dir_llm": "target/llm-runs/basics/t_009_init/csharp/server/gpt-5/llm",
"scorer_details": {
"schema_parity": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-002-scheduled-table-golden`.\n",
+ "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-009-init-golden`.\n",
"phase": "describe_golden"
}
+ },
+ "init_seed_alice": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true"
+ }
+ },
+ "init_total_two": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 2,
+ "expected": 2,
+ "sql": "SELECT COUNT(*) AS n FROM users"
+ }
+ },
+ "init_seed_bob": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false"
+ }
}
},
"vendor": "openai",
- "started_at": "2026-01-04T18:03:32.125218903Z",
- "finished_at": "2026-01-04T18:04:51.921266018Z"
+ "started_at": "2026-01-04T18:03:31.415313652Z",
+ "finished_at": "2026-01-04T18:04:59.072525790Z"
},
- "t_003_struct_in_table": {
+ "t_010_connect": {
"hash": "6f75b0c555fd7577df52872b447d9237496b7f004485c56464181b8bfb7834de",
- "task": "t_003_struct_in_table",
+ "task": "t_010_connect",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 1,
"passed_tests": 0,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n\n public Position(int X, int Y)\n {\n this.X = X;\n this.Y = Y;\n }\n }\n\n [SpacetimeDB.Table(Name = \"entities\", Public = true)]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Position Pos;\n }\n}",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"events\")]\n public partial struct Event\n {\n [SpacetimeDB.PrimaryKey, SpacetimeDB.AutoInc]\n public int Id;\n public string Kind;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientConnected)]\n public static void ClientConnected(ReducerContext ctx)\n {\n ctx.Db.events.Insert(new Event { Kind = \"connected\" });\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientDisconnected)]\n public static void ClientDisconnected(ReducerContext ctx)\n {\n ctx.Db.events.Insert(new Event { Kind = \"disconnected\" });\n }\n}",
"category": "basics",
"route_api_model": "gpt-5",
- "golden_db": "basics-t-003-struct-in-table-golden",
- "llm_db": "basics-t-003-struct-in-table-gpt-5-llm",
- "work_dir_golden": "target/llm-runs/basics/t_003_struct_in_table/csharp/server/golden",
- "work_dir_llm": "target/llm-runs/basics/t_003_struct_in_table/csharp/server/gpt-5/llm",
+ "golden_db": "basics-t-010-connect-golden",
+ "llm_db": "basics-t-010-connect-gpt-5-llm",
+ "work_dir_golden": "target/llm-runs/basics/t_010_connect/csharp/server/golden",
+ "work_dir_llm": "target/llm-runs/basics/t_010_connect/csharp/server/gpt-5/llm",
"scorer_details": {
"schema_parity": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-003-struct-in-table-golden`.\n",
+ "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-010-connect-golden`.\n",
"phase": "describe_golden"
}
}
},
"vendor": "openai",
- "started_at": "2026-01-04T18:03:49.955467773Z",
- "finished_at": "2026-01-04T18:05:04.199024592Z"
+ "started_at": "2026-01-04T18:04:28.771931334Z",
+ "finished_at": "2026-01-04T18:05:16.349401352Z"
},
"t_011_helper_function": {
"hash": "6f75b0c555fd7577df52872b447d9237496b7f004485c56464181b8bfb7834de",
@@ -508,21 +467,21 @@
"phase": "describe_golden"
}
},
- "helper_func_sum_abs": {
+ "helper_func_sum_parity": {
"pass": false,
"partial": 0.0,
"notes": {
- "actual": 0,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1 AND Sum=5"
+ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-011-helper-function-golden`.\n",
+ "phase": "call_reducer_golden"
}
},
- "helper_func_sum_parity": {
+ "helper_func_sum_abs": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-011-helper-function-golden`.\n",
- "phase": "call_reducer_golden"
+ "actual": 0,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1 AND Sum=5"
}
}
},
@@ -530,43 +489,34 @@
"started_at": "2026-01-04T18:03:46.460100343Z",
"finished_at": "2026-01-04T18:04:59.206987734Z"
},
- "t_017_scheduled_columns": {
+ "t_012_spacetime_product_type": {
"hash": "6f75b0c555fd7577df52872b447d9237496b7f004485c56464181b8bfb7834de",
- "task": "t_017_scheduled_columns",
+ "task": "t_012_spacetime_product_type",
"lang": "csharp",
- "golden_published": true,
+ "golden_published": false,
"model_name": "GPT-5",
- "total_tests": 2,
- "passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"tick_timer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [SpacetimeDB.PrimaryKey, SpacetimeDB.AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [SpacetimeDB.Reducer(SpacetimeDB.ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.tick_timer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMilliseconds(50))\n });\n }\n\n [SpacetimeDB.Reducer]\n public static void Tick(ReducerContext ctx, TickTimer _timer)\n {\n }\n}",
- "category": "schema",
- "route_api_model": "gpt-5",
- "golden_db": "schema-t-017-scheduled-columns-golden",
- "llm_db": "schema-t-017-scheduled-columns-gpt-5-llm",
- "work_dir_golden": "target/llm-runs/schema/t_017_scheduled_columns/csharp/server/golden",
- "work_dir_llm": "target/llm-runs/schema/t_017_scheduled_columns/csharp/server/gpt-5/llm",
+ "total_tests": 3,
+ "passed_tests": 0,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Type]\n public partial struct Score\n {\n public int Left;\n public int Right;\n }\n\n [Table(Name = \"results\", Public = true)]\n public partial struct Result\n {\n [PrimaryKey]\n public int Id;\n public Score Value;\n }\n\n [Reducer]\n public static void SetScore(ReducerContext ctx, int id, int left, int right)\n {\n ctx.Db.results.Insert(new Result\n {\n Id = id,\n Value = new Score\n {\n Left = left,\n Right = right\n }\n });\n }\n}",
+ "category": "schema",
+ "route_api_model": "gpt-5",
+ "golden_db": "schema-t-012-spacetime-product-type-golden",
+ "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm",
+ "work_dir_golden": "target/llm-runs/schema/t_012_spacetime_product_type/csharp/server/golden",
+ "work_dir_llm": "target/llm-runs/schema/t_012_spacetime_product_type/csharp/server/gpt-5/llm",
"scorer_details": {
- "scheduled_seeded_one_row": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM tick_timer WHERE ScheduledId>=0"
- }
- },
- "schema_parity": {
+ "publish_error": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-017-scheduled-columns-golden`.\n",
- "phase": "describe_golden"
+ "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nSystem.IO.IOException: The system cannot open the device or file specified. : 'NuGet-Migrations'\n at System.Threading.Mutex.CreateMutexCore(Boolean initiallyOwned, String name, Boolean& createdNew)\n at System.Threading.Mutex..ctor(Boolean initiallyOwned, String name)\n at NuGet.Common.Migrations.MigrationRunner.Run(String migrationsDirectory)\n at Microsoft.DotNet.Configurer.DotnetFirstTimeUseConfigurer.Configure()\n at Microsoft.DotNet.Cli.Program.ConfigureDotNetForFirstTimeUse(IFirstTimeUseNoticeSentinel firstTimeUseNoticeSentinel, IAspNetCertificateSentinel aspNetCertificateSentinel, IFileSentinel toolPathSentinel, Boolean isDotnetBeingInvokedFromNativeInstaller, DotnetFirstRunConfiguration dotnetFirstRunConfiguration, IEnvironmentProvider environmentProvider, Dictionary`2 performanceMeasurements)\n at Microsoft.DotNet.Cli.Program.ProcessArgs(String[] args, TimeSpan startupTime, ITelemetry telemetryClient)\n at Microsoft.DotNet.Cli.Program.Main(String[] args)\nError: command [\"dotnet\", \"workload\", \"list\"] exited with code 1\n\n--- stdout ---\n",
+ "phase": "build_or_publish"
}
}
},
"vendor": "openai",
- "started_at": "2026-01-04T18:02:06.142065603Z",
- "finished_at": "2026-01-04T18:03:31.781039210Z"
+ "started_at": "2026-01-04T18:02:06.156122014Z",
+ "finished_at": "2026-01-04T18:02:28.230519129Z"
},
"t_013_spacetime_sum_type": {
"hash": "6f75b0c555fd7577df52872b447d9237496b7f004485c56464181b8bfb7834de",
@@ -597,100 +547,63 @@
"started_at": "2026-01-04T18:03:02.247372147Z",
"finished_at": "2026-01-04T18:03:46.460073874Z"
},
- "t_000_empty_reducers": {
- "hash": "6f75b0c555fd7577df52872b447d9237496b7f004485c56464181b8bfb7834de",
- "task": "t_000_empty_reducers",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "GPT-5",
- "total_tests": 1,
- "passed_tests": 0,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_NoArgs(ReducerContext ctx)\n {\n }\n\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_WithInt(ReducerContext ctx, int count)\n {\n }\n\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_WithString(ReducerContext ctx, string name)\n {\n }\n\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_WithTwoArgs(ReducerContext ctx, int count, string name)\n {\n }\n\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_WithThreeArgs(ReducerContext ctx, bool active, float ratio, string label)\n {\n }\n}",
- "category": "basics",
- "route_api_model": "gpt-5",
- "golden_db": "basics-t-000-empty-reducers-golden",
- "llm_db": "basics-t-000-empty-reducers-gpt-5-llm",
- "work_dir_golden": "target/llm-runs/basics/t_000_empty_reducers/csharp/server/golden",
- "work_dir_llm": "target/llm-runs/basics/t_000_empty_reducers/csharp/server/gpt-5/llm",
- "scorer_details": {
- "schema_parity": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-000-empty-reducers-golden`.\n",
- "phase": "describe_golden"
- }
- }
- },
- "vendor": "openai",
- "started_at": "2026-01-04T18:03:02.314835122Z",
- "finished_at": "2026-01-04T18:03:49.955356015Z"
- },
- "t_004_insert": {
+ "t_014_elementary_columns": {
"hash": "6f75b0c555fd7577df52872b447d9237496b7f004485c56464181b8bfb7834de",
- "task": "t_004_insert",
+ "task": "t_014_elementary_columns",
"lang": "csharp",
- "golden_published": true,
+ "golden_published": false,
"model_name": "GPT-5",
- "total_tests": 2,
+ "total_tests": 3,
"passed_tests": 0,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void InsertUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n ctx.Db.users.Insert(new User\n {\n Id = id,\n Name = name,\n Age = age,\n Active = active\n });\n }\n}",
- "category": "basics",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"primitives\")]\n public partial struct Primitive\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Count;\n public long Total;\n public float Price;\n public double Ratio;\n public bool Active;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.primitives.Insert(new Primitive\n {\n Id = 1,\n Count = 2,\n Total = 3000000000L,\n Price = 1.5f,\n Ratio = 2.25,\n Active = true,\n Name = \"Alice\"\n });\n }\n}",
+ "category": "schema",
"route_api_model": "gpt-5",
- "golden_db": "basics-t-004-insert-golden",
- "llm_db": "basics-t-004-insert-gpt-5-llm",
- "work_dir_golden": "target/llm-runs/basics/t_004_insert/csharp/server/golden",
- "work_dir_llm": "target/llm-runs/basics/t_004_insert/csharp/server/gpt-5/llm",
+ "golden_db": "schema-t-014-elementary-columns-golden",
+ "llm_db": "schema-t-014-elementary-columns-gpt-5-llm",
+ "work_dir_golden": "target/llm-runs/schema/t_014_elementary_columns/csharp/server/golden",
+ "work_dir_llm": "target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm",
"scorer_details": {
- "data_parity_insert_user": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-004-insert-golden`.\n",
- "phase": "call_reducer_golden"
- }
- },
- "schema_parity": {
+ "publish_error": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-004-insert-golden`.\n",
- "phase": "describe_golden"
+ "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\n\nInstalling pack Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk version 8.0.22...\nSkipping NuGet package signature verification.\nWriting workload pack installation record for Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk version 8.0.22...\nInstalling pack Microsoft.NETCore.App.Runtime.Mono.wasi-wasm version 8.0.22...\nPack Microsoft.NETCore.App.Runtime.Mono.wasi-wasm version 8.0.22 is already installed.\nWriting workload pack installation record for Microsoft.NETCore.App.Runtime.Mono.wasi-wasm version 8.0.22...\nInstalling pack Microsoft.NET.Runtime.WebAssembly.Templates version 8.0.22...\nPack Microsoft.NET.Runtime.WebAssembly.Templates version 8.0.22 is already installed.\nWriting workload pack installation record for Microsoft.NET.Runtime.WebAssembly.Templates version 8.0.22...\nInstalling pack Microsoft.NET.Runtime.MonoAOTCompiler.Task version 8.0.22...\nWriting workload pack installation record for Microsoft.NET.Runtime.MonoAOTCompiler.Task version 8.0.22...\nInstalling pack Microsoft.NET.Runtime.MonoTargets.Sdk version 8.0.22...\nWriting workload pack installation record for Microsoft.NET.Runtime.MonoTargets.Sdk version 8.0.22...\nGarbage collecting for SDK feature band(s) 8.0.100 8.0.200 8.0.300 8.0.400 9.0.100 9.0.200 9.0.300 10.0.100...\n\nSuccessfully installed workload(s) .\n\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/obj/Release/net8.0/wasi-wasm/SpacetimeDB.Codegen/SpacetimeDB.Codegen.Module/FFI.cs(27,32): warning CS8981: The type name 'primitives' only contains lower-cased ascii characters. Such names may become reserved for the language. [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: The \"EmitBundleObjectFiles\" task failed unexpectedly. [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: System.AggregateException: One or more errors occurred. (Pipe is broken.) (Pipe is broken.) (Pipe is broken.) (Pipe is broken.) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: ---> System.IO.IOException: Pipe is broken. [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.Pipes.PipeStream.CheckWriteOperations() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.StreamWriter.Flush(Boolean flushStream, Boolean flushEncoder) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.StreamWriter.Dispose(Boolean disposing) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.TextWriter.Dispose() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.<>c__DisplayClass27_1.b__2(Stream codeStream) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at Utils.TryRunProcess(TaskLoggingHelper logger, String path, String args, IDictionary`2 envVars, String workingDir, Boolean silent, Boolean logStdErrAsMessage, MessageImportance debugMessageImportance, String label, Action`1 inputProvider) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleObjectFiles.EmitBundleFile(String destinationFile, Action`1 EmitBundleFile) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.<>c__DisplayClass27_0.b__1(Int32 i, ParallelLoopState state) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.<>c__DisplayClass19_0`2.b__1(RangeWorker& currentWorker, Int64 timeout, Boolean& replicationDelegateYieldedBeforeCompletion) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: --- End of stack trace from previous location --- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.<>c__DisplayClass19_0`2.b__1(RangeWorker& currentWorker, Int64 timeout, Boolean& replicationDelegateYieldedBeforeCompletion) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.TaskReplicator.Replica.Execute() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: --- End of inner exception stack trace --- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.TaskReplicator.Run[TState](ReplicatableUserAction`1 action, ParallelOptions options, Boolean stopOnFirstFailure) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.ForWorker[TLocal,TInt](TInt fromInclusive, TInt toExclusive, ParallelOptions parallelOptions, Action`1 body, Action`2 bodyWithState, Func`4 bodyWithLocal, Func`1 localInit, Action`1 localFinally) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: --- End of stack trace from previous location --- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.ForWorker[TLocal,TInt](TInt fromInclusive, TInt toExclusive, ParallelOptions parallelOptions, Action`1 body, Action`2 bodyWithState, Func`4 bodyWithLocal, Func`1 localInit, Action`1 localFinally) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.For(Int32 fromInclusive, Int32 toExclusive, ParallelOptions parallelOptions, Action`2 body) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.Execute() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleObjectFiles.Execute() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at Microsoft.Build.BackEnd.TaskExecutionHost.Execute() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at Microsoft.Build.BackEnd.TaskBuilder.ExecuteInstantiatedTask(TaskExecutionHost taskExecutionHost, TaskLoggingContext taskLoggingContext, TaskHost taskHost, ItemBucket bucket, TaskExecutionMode howToExecuteTask) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: ---> (Inner Exception #1) System.IO.IOException: Pipe is broken. [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.Pipes.PipeStream.CheckWriteOperations() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.StreamWriter.Flush(Boolean flushStream, Boolean flushEncoder) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.StreamWriter.Dispose(Boolean disposing) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.TextWriter.Dispose() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.<>c__DisplayClass27_1.b__2(Stream codeStream) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at Utils.TryRunProcess(TaskLoggingHelper logger, String path, String args, IDictionary`2 envVars, String workingDir, Boolean silent, Boolean logStdErrAsMessage, MessageImportance debugMessageImportance, String label, Action`1 inputProvider) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleObjectFiles.EmitBundleFile(String destinationFile, Action`1 EmitBundleFile) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.<>c__DisplayClass27_0.b__1(Int32 i, ParallelLoopState state) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.<>c__DisplayClass19_0`2.b__1(RangeWorker& currentWorker, Int64 timeout, Boolean& replicationDelegateYieldedBeforeCompletion) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: --- End of stack trace from previous location --- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.<>c__DisplayClass19_0`2.b__1(RangeWorker& currentWorker, Int64 timeout, Boolean& replicationDelegateYieldedBeforeCompletion) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.TaskReplicator.Replica.Execute()<--- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: ---> (Inner Exception #2) System.IO.IOException: Pipe is broken. [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.Pipes.PipeStream.CheckWriteOperations() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.StreamWriter.Flush(Boolean flushStream, Boolean flushEncoder) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.StreamWriter.Dispose(Boolean disposing) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.TextWriter.Dispose() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.<>c__DisplayClass27_1.b__2(Stream codeStream) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at Utils.TryRunProcess(TaskLoggingHelper logger, String path, String args, IDictionary`2 envVars, String workingDir, Boolean silent, Boolean logStdErrAsMessage, MessageImportance debugMessageImportance, String label, Action`1 inputProvider) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleObjectFiles.EmitBundleFile(String destinationFile, Action`1 EmitBundleFile) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.<>c__DisplayClass27_0.b__1(Int32 i, ParallelLoopState state) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.<>c__DisplayClass19_0`2.b__1(RangeWorker& currentWorker, Int64 timeout, Boolean& replicationDelegateYieldedBeforeCompletion) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: --- End of stack trace from previous location --- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.<>c__DisplayClass19_0`2.b__1(RangeWorker& currentWorker, Int64 timeout, Boolean& replicationDelegateYieldedBeforeCompletion) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.TaskReplicator.Replica.Execute()<--- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: ---> (Inner Exception #3) System.IO.IOException: Pipe is broken. [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.Pipes.PipeStream.CheckWriteOperations() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.StreamWriter.Flush(Boolean flushStream, Boolean flushEncoder) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.StreamWriter.Dispose(Boolean disposing) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.TextWriter.Dispose() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.<>c__DisplayClass27_1.b__2(Stream codeStream) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at Utils.TryRunProcess(TaskLoggingHelper logger, String path, String args, IDictionary`2 envVars, String workingDir, Boolean silent, Boolean logStdErrAsMessage, MessageImportance debugMessageImportance, String label, Action`1 inputProvider) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleObjectFiles.EmitBundleFile(String destinationFile, Action`1 EmitBundleFile) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.<>c__DisplayClass27_0.b__1(Int32 i, ParallelLoopState state) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.<>c__DisplayClass19_0`2.b__1(RangeWorker& currentWorker, Int64 timeout, Boolean& replicationDelegateYieldedBeforeCompletion) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: --- End of stack trace from previous location --- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.<>c__DisplayClass19_0`2.b__1(RangeWorker& currentWorker, Int64 timeout, Boolean& replicationDelegateYieldedBeforeCompletion) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.TaskReplicator.Replica.Execute()<--- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n",
+ "phase": "build_or_publish"
}
}
},
"vendor": "openai",
- "started_at": "2026-01-04T18:04:40.032531267Z",
- "finished_at": "2026-01-04T18:05:19.104533291Z"
+ "started_at": "2026-01-04T18:02:06.151425389Z",
+ "finished_at": "2026-01-04T18:03:02.557372040Z"
},
- "t_014_elementary_columns": {
+ "t_015_product_type_columns": {
"hash": "6f75b0c555fd7577df52872b447d9237496b7f004485c56464181b8bfb7834de",
- "task": "t_014_elementary_columns",
+ "task": "t_015_product_type_columns",
"lang": "csharp",
"golden_published": false,
"model_name": "GPT-5",
"total_tests": 3,
"passed_tests": 0,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"primitives\")]\n public partial struct Primitive\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Count;\n public long Total;\n public float Price;\n public double Ratio;\n public bool Active;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.primitives.Insert(new Primitive\n {\n Id = 1,\n Count = 2,\n Total = 3000000000L,\n Price = 1.5f,\n Ratio = 2.25,\n Active = true,\n Name = \"Alice\"\n });\n }\n}",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Address\n {\n public string Street;\n public int Zip;\n }\n\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"profiles\", Public = true)]\n public partial struct Profile\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Address Home;\n public Address Work;\n public Position Pos;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.profiles.Insert(new Profile\n {\n Id = 1,\n Home = new Address { Street = \"1 Main\", Zip = 11111 },\n Work = new Address { Street = \"2 Broad\", Zip = 22222 },\n Pos = new Position { X = 7, Y = 9 }\n });\n }\n}",
"category": "schema",
"route_api_model": "gpt-5",
- "golden_db": "schema-t-014-elementary-columns-golden",
- "llm_db": "schema-t-014-elementary-columns-gpt-5-llm",
- "work_dir_golden": "target/llm-runs/schema/t_014_elementary_columns/csharp/server/golden",
- "work_dir_llm": "target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm",
+ "golden_db": "schema-t-015-product-type-columns-golden",
+ "llm_db": "schema-t-015-product-type-columns-gpt-5-llm",
+ "work_dir_golden": "target/llm-runs/schema/t_015_product_type_columns/csharp/server/golden",
+ "work_dir_llm": "target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm",
"scorer_details": {
"publish_error": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\n\nInstalling pack Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk version 8.0.22...\nSkipping NuGet package signature verification.\nWriting workload pack installation record for Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk version 8.0.22...\nInstalling pack Microsoft.NETCore.App.Runtime.Mono.wasi-wasm version 8.0.22...\nPack Microsoft.NETCore.App.Runtime.Mono.wasi-wasm version 8.0.22 is already installed.\nWriting workload pack installation record for Microsoft.NETCore.App.Runtime.Mono.wasi-wasm version 8.0.22...\nInstalling pack Microsoft.NET.Runtime.WebAssembly.Templates version 8.0.22...\nPack Microsoft.NET.Runtime.WebAssembly.Templates version 8.0.22 is already installed.\nWriting workload pack installation record for Microsoft.NET.Runtime.WebAssembly.Templates version 8.0.22...\nInstalling pack Microsoft.NET.Runtime.MonoAOTCompiler.Task version 8.0.22...\nWriting workload pack installation record for Microsoft.NET.Runtime.MonoAOTCompiler.Task version 8.0.22...\nInstalling pack Microsoft.NET.Runtime.MonoTargets.Sdk version 8.0.22...\nWriting workload pack installation record for Microsoft.NET.Runtime.MonoTargets.Sdk version 8.0.22...\nGarbage collecting for SDK feature band(s) 8.0.100 8.0.200 8.0.300 8.0.400 9.0.100 9.0.200 9.0.300 10.0.100...\n\nSuccessfully installed workload(s) .\n\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/obj/Release/net8.0/wasi-wasm/SpacetimeDB.Codegen/SpacetimeDB.Codegen.Module/FFI.cs(27,32): warning CS8981: The type name 'primitives' only contains lower-cased ascii characters. Such names may become reserved for the language. [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: The \"EmitBundleObjectFiles\" task failed unexpectedly. [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: System.AggregateException: One or more errors occurred. (Pipe is broken.) (Pipe is broken.) (Pipe is broken.) (Pipe is broken.) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: ---> System.IO.IOException: Pipe is broken. [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.Pipes.PipeStream.CheckWriteOperations() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.StreamWriter.Flush(Boolean flushStream, Boolean flushEncoder) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.StreamWriter.Dispose(Boolean disposing) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.TextWriter.Dispose() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.<>c__DisplayClass27_1.b__2(Stream codeStream) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at Utils.TryRunProcess(TaskLoggingHelper logger, String path, String args, IDictionary`2 envVars, String workingDir, Boolean silent, Boolean logStdErrAsMessage, MessageImportance debugMessageImportance, String label, Action`1 inputProvider) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleObjectFiles.EmitBundleFile(String destinationFile, Action`1 EmitBundleFile) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.<>c__DisplayClass27_0.b__1(Int32 i, ParallelLoopState state) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.<>c__DisplayClass19_0`2.b__1(RangeWorker& currentWorker, Int64 timeout, Boolean& replicationDelegateYieldedBeforeCompletion) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: --- End of stack trace from previous location --- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.<>c__DisplayClass19_0`2.b__1(RangeWorker& currentWorker, Int64 timeout, Boolean& replicationDelegateYieldedBeforeCompletion) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.TaskReplicator.Replica.Execute() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: --- End of inner exception stack trace --- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.TaskReplicator.Run[TState](ReplicatableUserAction`1 action, ParallelOptions options, Boolean stopOnFirstFailure) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.ForWorker[TLocal,TInt](TInt fromInclusive, TInt toExclusive, ParallelOptions parallelOptions, Action`1 body, Action`2 bodyWithState, Func`4 bodyWithLocal, Func`1 localInit, Action`1 localFinally) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: --- End of stack trace from previous location --- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.ForWorker[TLocal,TInt](TInt fromInclusive, TInt toExclusive, ParallelOptions parallelOptions, Action`1 body, Action`2 bodyWithState, Func`4 bodyWithLocal, Func`1 localInit, Action`1 localFinally) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.For(Int32 fromInclusive, Int32 toExclusive, ParallelOptions parallelOptions, Action`2 body) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.Execute() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleObjectFiles.Execute() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at Microsoft.Build.BackEnd.TaskExecutionHost.Execute() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at Microsoft.Build.BackEnd.TaskBuilder.ExecuteInstantiatedTask(TaskExecutionHost taskExecutionHost, TaskLoggingContext taskLoggingContext, TaskHost taskHost, ItemBucket bucket, TaskExecutionMode howToExecuteTask) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: ---> (Inner Exception #1) System.IO.IOException: Pipe is broken. [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.Pipes.PipeStream.CheckWriteOperations() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.StreamWriter.Flush(Boolean flushStream, Boolean flushEncoder) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.StreamWriter.Dispose(Boolean disposing) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.TextWriter.Dispose() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.<>c__DisplayClass27_1.b__2(Stream codeStream) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at Utils.TryRunProcess(TaskLoggingHelper logger, String path, String args, IDictionary`2 envVars, String workingDir, Boolean silent, Boolean logStdErrAsMessage, MessageImportance debugMessageImportance, String label, Action`1 inputProvider) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleObjectFiles.EmitBundleFile(String destinationFile, Action`1 EmitBundleFile) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.<>c__DisplayClass27_0.b__1(Int32 i, ParallelLoopState state) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.<>c__DisplayClass19_0`2.b__1(RangeWorker& currentWorker, Int64 timeout, Boolean& replicationDelegateYieldedBeforeCompletion) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: --- End of stack trace from previous location --- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.<>c__DisplayClass19_0`2.b__1(RangeWorker& currentWorker, Int64 timeout, Boolean& replicationDelegateYieldedBeforeCompletion) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.TaskReplicator.Replica.Execute()<--- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: ---> (Inner Exception #2) System.IO.IOException: Pipe is broken. [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.Pipes.PipeStream.CheckWriteOperations() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.StreamWriter.Flush(Boolean flushStream, Boolean flushEncoder) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.StreamWriter.Dispose(Boolean disposing) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.TextWriter.Dispose() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.<>c__DisplayClass27_1.b__2(Stream codeStream) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at Utils.TryRunProcess(TaskLoggingHelper logger, String path, String args, IDictionary`2 envVars, String workingDir, Boolean silent, Boolean logStdErrAsMessage, MessageImportance debugMessageImportance, String label, Action`1 inputProvider) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleObjectFiles.EmitBundleFile(String destinationFile, Action`1 EmitBundleFile) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.<>c__DisplayClass27_0.b__1(Int32 i, ParallelLoopState state) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.<>c__DisplayClass19_0`2.b__1(RangeWorker& currentWorker, Int64 timeout, Boolean& replicationDelegateYieldedBeforeCompletion) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: --- End of stack trace from previous location --- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.<>c__DisplayClass19_0`2.b__1(RangeWorker& currentWorker, Int64 timeout, Boolean& replicationDelegateYieldedBeforeCompletion) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.TaskReplicator.Replica.Execute()<--- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: ---> (Inner Exception #3) System.IO.IOException: Pipe is broken. [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.Pipes.PipeStream.CheckWriteOperations() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.StreamWriter.Flush(Boolean flushStream, Boolean flushEncoder) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.StreamWriter.Dispose(Boolean disposing) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.TextWriter.Dispose() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.<>c__DisplayClass27_1.b__2(Stream codeStream) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at Utils.TryRunProcess(TaskLoggingHelper logger, String path, String args, IDictionary`2 envVars, String workingDir, Boolean silent, Boolean logStdErrAsMessage, MessageImportance debugMessageImportance, String label, Action`1 inputProvider) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleObjectFiles.EmitBundleFile(String destinationFile, Action`1 EmitBundleFile) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.<>c__DisplayClass27_0.b__1(Int32 i, ParallelLoopState state) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.<>c__DisplayClass19_0`2.b__1(RangeWorker& currentWorker, Int64 timeout, Boolean& replicationDelegateYieldedBeforeCompletion) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: --- End of stack trace from previous location --- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.<>c__DisplayClass19_0`2.b__1(RangeWorker& currentWorker, Int64 timeout, Boolean& replicationDelegateYieldedBeforeCompletion) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.TaskReplicator.Replica.Execute()<--- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n",
+ "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\n\nInstalling pack Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk version 8.0.22...\nSkipping NuGet package signature verification.\nWriting workload pack installation record for Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk version 8.0.22...\nInstalling pack Microsoft.NETCore.App.Runtime.Mono.wasi-wasm version 8.0.22...\nWriting workload pack installation record for Microsoft.NETCore.App.Runtime.Mono.wasi-wasm version 8.0.22...\nInstalling pack Microsoft.NET.Runtime.WebAssembly.Templates version 8.0.22...\nPack Microsoft.NET.Runtime.WebAssembly.Templates version 8.0.22 is already installed.\nWriting workload pack installation record for Microsoft.NET.Runtime.WebAssembly.Templates version 8.0.22...\nInstalling pack Microsoft.NET.Runtime.MonoAOTCompiler.Task version 8.0.22...\nPack Microsoft.NET.Runtime.MonoAOTCompiler.Task version 8.0.22 is already installed.\nWriting workload pack installation record for Microsoft.NET.Runtime.MonoAOTCompiler.Task version 8.0.22...\nInstalling pack Microsoft.NET.Runtime.MonoTargets.Sdk version 8.0.22...\nPack Microsoft.NET.Runtime.MonoTargets.Sdk version 8.0.22 is already installed.\nWriting workload pack installation record for Microsoft.NET.Runtime.MonoTargets.Sdk version 8.0.22...\nGarbage collecting for SDK feature band(s) 8.0.100 8.0.200 8.0.300 8.0.400 9.0.100 9.0.200 9.0.300 10.0.100...\n\nSuccessfully installed workload(s) .\n\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/obj/Release/net8.0/wasi-wasm/SpacetimeDB.Codegen/SpacetimeDB.Codegen.Module/FFI.cs(27,32): warning CS8981: The type name 'profiles' only contains lower-cased ascii characters. Such names may become reserved for the language. [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: The \"EmitBundleObjectFiles\" task failed unexpectedly. [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: System.AggregateException: One or more errors occurred. (Pipe is broken.) (Pipe is broken.) (Pipe is broken.) (Pipe is broken.) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: ---> System.IO.IOException: Pipe is broken. [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.Pipes.PipeStream.CheckWriteOperations() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.StreamWriter.Flush(Boolean flushStream, Boolean flushEncoder) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.StreamWriter.Dispose(Boolean disposing) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.TextWriter.Dispose() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.<>c__DisplayClass27_1.b__2(Stream codeStream) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at Utils.TryRunProcess(TaskLoggingHelper logger, String path, String args, IDictionary`2 envVars, String workingDir, Boolean silent, Boolean logStdErrAsMessage, MessageImportance debugMessageImportance, String label, Action`1 inputProvider) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleObjectFiles.EmitBundleFile(String destinationFile, Action`1 EmitBundleFile) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.<>c__DisplayClass27_0.b__1(Int32 i, ParallelLoopState state) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.<>c__DisplayClass19_0`2.b__1(RangeWorker& currentWorker, Int64 timeout, Boolean& replicationDelegateYieldedBeforeCompletion) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: --- End of stack trace from previous location --- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.<>c__DisplayClass19_0`2.b__1(RangeWorker& currentWorker, Int64 timeout, Boolean& replicationDelegateYieldedBeforeCompletion) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.TaskReplicator.Replica.Execute() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: --- End of inner exception stack trace --- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.TaskReplicator.Run[TState](ReplicatableUserAction`1 action, ParallelOptions options, Boolean stopOnFirstFailure) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.ForWorker[TLocal,TInt](TInt fromInclusive, TInt toExclusive, ParallelOptions parallelOptions, Action`1 body, Action`2 bodyWithState, Func`4 bodyWithLocal, Func`1 localInit, Action`1 localFinally) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: --- End of stack trace from previous location --- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.ForWorker[TLocal,TInt](TInt fromInclusive, TInt toExclusive, ParallelOptions parallelOptions, Action`1 body, Action`2 bodyWithState, Func`4 bodyWithLocal, Func`1 localInit, Action`1 localFinally) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.For(Int32 fromInclusive, Int32 toExclusive, ParallelOptions parallelOptions, Action`2 body) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.Execute() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleObjectFiles.Execute() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at Microsoft.Build.BackEnd.TaskExecutionHost.Execute() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at Microsoft.Build.BackEnd.TaskBuilder.ExecuteInstantiatedTask(TaskExecutionHost taskExecutionHost, TaskLoggingContext taskLoggingContext, TaskHost taskHost, ItemBucket bucket, TaskExecutionMode howToExecuteTask) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: ---> (Inner Exception #1) System.IO.IOException: Pipe is broken. [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.Pipes.PipeStream.CheckWriteOperations() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.StreamWriter.Flush(Boolean flushStream, Boolean flushEncoder) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.StreamWriter.Dispose(Boolean disposing) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.TextWriter.Dispose() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.<>c__DisplayClass27_1.b__2(Stream codeStream) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at Utils.TryRunProcess(TaskLoggingHelper logger, String path, String args, IDictionary`2 envVars, String workingDir, Boolean silent, Boolean logStdErrAsMessage, MessageImportance debugMessageImportance, String label, Action`1 inputProvider) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleObjectFiles.EmitBundleFile(String destinationFile, Action`1 EmitBundleFile) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.<>c__DisplayClass27_0.b__1(Int32 i, ParallelLoopState state) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.<>c__DisplayClass19_0`2.b__1(RangeWorker& currentWorker, Int64 timeout, Boolean& replicationDelegateYieldedBeforeCompletion) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: --- End of stack trace from previous location --- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.<>c__DisplayClass19_0`2.b__1(RangeWorker& currentWorker, Int64 timeout, Boolean& replicationDelegateYieldedBeforeCompletion) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.TaskReplicator.Replica.Execute()<--- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: ---> (Inner Exception #2) System.IO.IOException: Pipe is broken. [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.Pipes.PipeStream.CheckWriteOperations() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.StreamWriter.Flush(Boolean flushStream, Boolean flushEncoder) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.StreamWriter.Dispose(Boolean disposing) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.TextWriter.Dispose() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.<>c__DisplayClass27_1.b__2(Stream codeStream) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at Utils.TryRunProcess(TaskLoggingHelper logger, String path, String args, IDictionary`2 envVars, String workingDir, Boolean silent, Boolean logStdErrAsMessage, MessageImportance debugMessageImportance, String label, Action`1 inputProvider) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleObjectFiles.EmitBundleFile(String destinationFile, Action`1 EmitBundleFile) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.<>c__DisplayClass27_0.b__1(Int32 i, ParallelLoopState state) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.<>c__DisplayClass19_0`2.b__1(RangeWorker& currentWorker, Int64 timeout, Boolean& replicationDelegateYieldedBeforeCompletion) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: --- End of stack trace from previous location --- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.<>c__DisplayClass19_0`2.b__1(RangeWorker& currentWorker, Int64 timeout, Boolean& replicationDelegateYieldedBeforeCompletion) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.TaskReplicator.Replica.Execute()<--- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: ---> (Inner Exception #3) System.IO.IOException: Pipe is broken. [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.Pipes.PipeStream.CheckWriteOperations() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.StreamWriter.Flush(Boolean flushStream, Boolean flushEncoder) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.StreamWriter.Dispose(Boolean disposing) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.TextWriter.Dispose() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.<>c__DisplayClass27_1.b__2(Stream codeStream) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at Utils.TryRunProcess(TaskLoggingHelper logger, String path, String args, IDictionary`2 envVars, String workingDir, Boolean silent, Boolean logStdErrAsMessage, MessageImportance debugMessageImportance, String label, Action`1 inputProvider) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleObjectFiles.EmitBundleFile(String destinationFile, Action`1 EmitBundleFile) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.<>c__DisplayClass27_0.b__1(Int32 i, ParallelLoopState state) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.<>c__DisplayClass19_0`2.b__1(RangeWorker& currentWorker, Int64 timeout, Boolean& replicationDelegateYieldedBeforeCompletion) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: --- End of stack trace from previous location --- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.<>c__DisplayClass19_0`2.b__1(RangeWorker& currentWorker, Int64 timeout, Boolean& replicationDelegateYieldedBeforeCompletion) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.TaskReplicator.Replica.Execute()<--- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm/StdbModule.csproj]\n",
"phase": "build_or_publish"
}
}
},
"vendor": "openai",
- "started_at": "2026-01-04T18:02:06.151425389Z",
- "finished_at": "2026-01-04T18:03:02.557372040Z"
+ "started_at": "2026-01-04T18:02:06.160910108Z",
+ "finished_at": "2026-01-04T18:03:02.247345187Z"
},
"t_016_sum_type_columns": {
"hash": "6f75b0c555fd7577df52872b447d9237496b7f004485c56464181b8bfb7834de",
@@ -721,275 +634,295 @@
"started_at": "2026-01-04T18:02:28.230551359Z",
"finished_at": "2026-01-04T18:03:16.646260839Z"
},
- "t_009_init": {
+ "t_017_scheduled_columns": {
"hash": "6f75b0c555fd7577df52872b447d9237496b7f004485c56464181b8bfb7834de",
- "task": "t_009_init",
+ "task": "t_017_scheduled_columns",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-5",
- "total_tests": 4,
- "passed_tests": 3,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer(SpacetimeDB.ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.users.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.users.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n }\n}",
- "category": "basics",
+ "total_tests": 2,
+ "passed_tests": 1,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"tick_timer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [SpacetimeDB.PrimaryKey, SpacetimeDB.AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [SpacetimeDB.Reducer(SpacetimeDB.ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.tick_timer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMilliseconds(50))\n });\n }\n\n [SpacetimeDB.Reducer]\n public static void Tick(ReducerContext ctx, TickTimer _timer)\n {\n }\n}",
+ "category": "schema",
"route_api_model": "gpt-5",
- "golden_db": "basics-t-009-init-golden",
- "llm_db": "basics-t-009-init-gpt-5-llm",
- "work_dir_golden": "target/llm-runs/basics/t_009_init/csharp/server/golden",
- "work_dir_llm": "target/llm-runs/basics/t_009_init/csharp/server/gpt-5/llm",
+ "golden_db": "schema-t-017-scheduled-columns-golden",
+ "llm_db": "schema-t-017-scheduled-columns-gpt-5-llm",
+ "work_dir_golden": "target/llm-runs/schema/t_017_scheduled_columns/csharp/server/golden",
+ "work_dir_llm": "target/llm-runs/schema/t_017_scheduled_columns/csharp/server/gpt-5/llm",
"scorer_details": {
- "schema_parity": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-009-init-golden`.\n",
- "phase": "describe_golden"
- }
- },
- "init_total_two": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 2,
- "expected": 2,
- "sql": "SELECT COUNT(*) AS n FROM users"
- }
- },
- "init_seed_alice": {
+ "scheduled_seeded_one_row": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true"
+ "sql": "SELECT COUNT(*) AS n FROM tick_timer WHERE ScheduledId>=0"
}
},
- "init_seed_bob": {
- "pass": true,
- "partial": 1.0,
+ "schema_parity": {
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false"
+ "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-017-scheduled-columns-golden`.\n",
+ "phase": "describe_golden"
}
}
},
"vendor": "openai",
- "started_at": "2026-01-04T18:03:31.415313652Z",
- "finished_at": "2026-01-04T18:04:59.072525790Z"
+ "started_at": "2026-01-04T18:02:06.142065603Z",
+ "finished_at": "2026-01-04T18:03:31.781039210Z"
},
- "t_007_crud": {
+ "t_018_constraints": {
"hash": "6f75b0c555fd7577df52872b447d9237496b7f004485c56464181b8bfb7834de",
- "task": "t_007_crud",
+ "task": "t_018_constraints",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-5",
- "total_tests": 4,
- "passed_tests": 1,
- "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void Crud(ReducerContext ctx)\n {\n // insert (Id=1, Name=\"Alice\", Age=30, Active=true)\n ctx.Db.users.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n\n // insert (Id=2, Name=\"Bob\", Age=22, Active=false)\n ctx.Db.users.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n\n // update (Id=1, Name=\"Alice2\", Age=31, Active=false)\n var u1 = ctx.Db.users.Id.Find(1) ?? throw new Exception(\"User with Id=1 not found\");\n u1.Name = \"Alice2\";\n u1.Age = 31;\n u1.Active = false;\n ctx.Db.users.Id.Update(u1);\n\n // delete Id=2\n ctx.Db.users.Id.Delete(2);\n }\n}",
- "category": "basics",
+ "total_tests": 3,
+ "passed_tests": 0,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"accounts\")]\n [SpacetimeDB.Index.BTree(Name = \"by_name\", Columns = new[] { \"Name\" })]\n public partial struct Account\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n [SpacetimeDB.Unique]\n public string Email;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.accounts.Insert(new Account { Id = 1, Email = \"a@example.com\", Name = \"Alice\" });\n ctx.Db.accounts.Insert(new Account { Id = 2, Email = \"b@example.com\", Name = \"Bob\" });\n }\n}",
+ "category": "schema",
"route_api_model": "gpt-5",
- "golden_db": "basics-t-007-crud-golden",
- "llm_db": "basics-t-007-crud-gpt-5-llm",
- "work_dir_golden": "target/llm-runs/basics/t_007_crud/csharp/server/golden",
- "work_dir_llm": "target/llm-runs/basics/t_007_crud/csharp/server/gpt-5/llm",
+ "golden_db": "schema-t-018-constraints-golden",
+ "llm_db": "schema-t-018-constraints-gpt-5-llm",
+ "work_dir_golden": "target/llm-runs/schema/t_018_constraints/csharp/server/golden",
+ "work_dir_llm": "target/llm-runs/schema/t_018_constraints/csharp/server/gpt-5/llm",
"scorer_details": {
- "crud_row_id1_parity": {
+ "schema_parity": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-007-crud-golden`.\n",
+ "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-018-constraints-golden`.\n",
+ "phase": "describe_golden"
+ }
+ },
+ "constraints_row_parity_after_seed": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-018-constraints-golden`.\n",
"phase": "call_reducer_golden"
}
},
- "crud_row_id2_deleted": {
- "pass": true,
- "partial": 1.0,
+ "constraints_seed_two_rows": {
+ "pass": false,
+ "partial": 0.0,
"notes": {
"actual": 0,
- "expected": 0,
- "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=2"
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM accounts WHERE Id=2"
}
- },
- "schema_parity": {
+ }
+ },
+ "vendor": "openai",
+ "started_at": "2026-01-04T18:02:06.137382585Z",
+ "finished_at": "2026-01-04T18:03:31.415276583Z"
+ },
+ "t_019_many_to_many": {
+ "hash": "6f75b0c555fd7577df52872b447d9237496b7f004485c56464181b8bfb7834de",
+ "task": "t_019_many_to_many",
+ "lang": "csharp",
+ "golden_published": false,
+ "model_name": "GPT-5",
+ "total_tests": 5,
+ "passed_tests": 0,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"users\", Public = true)]\n public partial struct User\n {\n [PrimaryKey]\n public int UserId;\n public string Name;\n }\n\n [Table(Name = \"groups\", Public = true)]\n public partial struct Group\n {\n [PrimaryKey]\n public int GroupId;\n public string Title;\n }\n\n [Table(Name = \"memberships\", Public = true)]\n public partial struct Membership\n {\n [PrimaryKey]\n public int Id;\n\n [SpacetimeDB.Index.BTree(Name = \"by_user\")]\n public int UserId;\n\n [SpacetimeDB.Index.BTree(Name = \"by_group\")]\n public int GroupId;\n }\n\n [Reducer]\n public static void Seed(ReducerContext ctx)\n {\n // Clear existing rows to ensure the tables contain exactly the specified rows\n foreach (var m in ctx.Db.Membership.Iter())\n {\n ctx.Db.Membership.Id.Delete(m.Id);\n }\n foreach (var g in ctx.Db.Group.Iter())\n {\n ctx.Db.Group.GroupId.Delete(g.GroupId);\n }\n foreach (var u in ctx.Db.User.Iter())\n {\n ctx.Db.User.UserId.Delete(u.UserId);\n }\n\n // users\n ctx.Db.User.Insert(new User { UserId = 1, Name = \"Alice\" });\n ctx.Db.User.Insert(new User { UserId = 2, Name = \"Bob\" });\n\n // groups\n ctx.Db.Group.Insert(new Group { GroupId = 10, Title = \"Admin\" });\n ctx.Db.Group.Insert(new Group { GroupId = 20, Title = \"Dev\" });\n\n // memberships\n ctx.Db.Membership.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\n ctx.Db.Membership.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\n ctx.Db.Membership.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\n }\n}",
+ "category": "schema",
+ "route_api_model": "gpt-5",
+ "golden_db": "schema-t-019-many-to-many-golden",
+ "llm_db": "schema-t-019-many-to-many-gpt-5-llm",
+ "work_dir_golden": "target/llm-runs/schema/t_019_many_to_many/csharp/server/golden",
+ "work_dir_llm": "target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm",
+ "scorer_details": {
+ "publish_error": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-007-crud-golden`.\n",
- "phase": "describe_golden"
+ "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/obj/Release/net8.0/wasi-wasm/SpacetimeDB.Codegen/SpacetimeDB.Codegen.Module/FFI.cs(155,24): warning CS8981: The type name 'users' only contains lower-cased ascii characters. Such names may become reserved for the language. [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/obj/Release/net8.0/wasi-wasm/SpacetimeDB.Codegen/SpacetimeDB.Codegen.Module/FFI.cs(70,24): warning CS8981: The type name 'memberships' only contains lower-cased ascii characters. Such names may become reserved for the language. [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/obj/Release/net8.0/wasi-wasm/SpacetimeDB.Codegen/SpacetimeDB.Codegen.Module/FFI.cs(27,32): warning CS8981: The type name 'groups' only contains lower-cased ascii characters. Such names may become reserved for the language. [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/Lib.cs(39,34): error CS1061: 'Local' does not contain a definition for 'Membership' and no accessible extension method 'Membership' accepting a first argument of type 'Local' could be found (are you missing a using directive or an assembly reference?) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/Lib.cs(41,20): error CS1061: 'Local' does not contain a definition for 'Membership' and no accessible extension method 'Membership' accepting a first argument of type 'Local' could be found (are you missing a using directive or an assembly reference?) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/Lib.cs(43,34): error CS1061: 'Local' does not contain a definition for 'Group' and no accessible extension method 'Group' accepting a first argument of type 'Local' could be found (are you missing a using directive or an assembly reference?) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/Lib.cs(45,20): error CS1061: 'Local' does not contain a definition for 'Group' and no accessible extension method 'Group' accepting a first argument of type 'Local' could be found (are you missing a using directive or an assembly reference?) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/Lib.cs(47,34): error CS1061: 'Local' does not contain a definition for 'User' and no accessible extension method 'User' accepting a first argument of type 'Local' could be found (are you missing a using directive or an assembly reference?) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/Lib.cs(49,20): error CS1061: 'Local' does not contain a definition for 'User' and no accessible extension method 'User' accepting a first argument of type 'Local' could be found (are you missing a using directive or an assembly reference?) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/Lib.cs(53,16): error CS1061: 'Local' does not contain a definition for 'User' and no accessible extension method 'User' accepting a first argument of type 'Local' could be found (are you missing a using directive or an assembly reference?) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/Lib.cs(54,16): error CS1061: 'Local' does not contain a definition for 'User' and no accessible extension method 'User' accepting a first argument of type 'Local' could be found (are you missing a using directive or an assembly reference?) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/Lib.cs(57,16): error CS1061: 'Local' does not contain a definition for 'Group' and no accessible extension method 'Group' accepting a first argument of type 'Local' could be found (are you missing a using directive or an assembly reference?) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/Lib.cs(58,16): error CS1061: 'Local' does not contain a definition for 'Group' and no accessible extension method 'Group' accepting a first argument of type 'Local' could be found (are you missing a using directive or an assembly reference?) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/Lib.cs(61,16): error CS1061: 'Local' does not contain a definition for 'Membership' and no accessible extension method 'Membership' accepting a first argument of type 'Local' could be found (are you missing a using directive or an assembly reference?) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/Lib.cs(62,16): error CS1061: 'Local' does not contain a definition for 'Membership' and no accessible extension method 'Membership' accepting a first argument of type 'Local' could be found (are you missing a using directive or an assembly reference?) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/Lib.cs(63,16): error CS1061: 'Local' does not contain a definition for 'Membership' and no accessible extension method 'Membership' accepting a first argument of type 'Local' could be found (are you missing a using directive or an assembly reference?) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm/StdbModule.csproj]\n",
+ "phase": "build_or_publish"
}
- },
- "crud_total_count_one": {
+ }
+ },
+ "vendor": "openai",
+ "started_at": "2026-01-04T18:02:06.130072420Z",
+ "finished_at": "2026-01-04T18:03:19.445236666Z"
+ },
+ "t_020_ecs": {
+ "hash": "6f75b0c555fd7577df52872b447d9237496b7f004485c56464181b8bfb7834de",
+ "task": "t_020_ecs",
+ "lang": "csharp",
+ "golden_published": false,
+ "model_name": "GPT-5",
+ "total_tests": 5,
+ "passed_tests": 0,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"entities\", Public = true)]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n }\n\n [SpacetimeDB.Table(Name = \"positions\", Public = true)]\n public partial struct Position\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"velocities\", Public = true)]\n public partial struct Velocity\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int VX;\n public int VY;\n }\n\n [SpacetimeDB.Table(Name = \"next_positions\", Public = true)]\n public partial struct NextPosition\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n // Entity 1\n ctx.Db.entities.Insert(new Entity\n {\n Id = 1\n });\n ctx.Db.positions.Insert(new Position\n {\n EntityId = 1,\n X = 0,\n Y = 0\n });\n ctx.Db.velocities.Insert(new Velocity\n {\n EntityId = 1,\n VX = 1,\n VY = 0\n });\n\n // Entity 2\n ctx.Db.entities.Insert(new Entity\n {\n Id = 2\n });\n ctx.Db.positions.Insert(new Position\n {\n EntityId = 2,\n X = 10,\n Y = 0\n });\n ctx.Db.velocities.Insert(new Velocity\n {\n EntityId = 2,\n VX = -2,\n VY = 3\n });\n }\n\n [SpacetimeDB.Reducer]\n public static void Step(ReducerContext ctx)\n {\n foreach (var pos in ctx.Db.positions.Iter())\n {\n if (ctx.Db.velocities.EntityId.Find(pos.EntityId) is Velocity vel)\n {\n var nx = pos.X + vel.VX;\n var ny = pos.Y + vel.VY;\n\n if (ctx.Db.next_positions.EntityId.Find(pos.EntityId) is NextPosition existing)\n {\n existing.X = nx;\n existing.Y = ny;\n ctx.Db.next_positions.EntityId.Update(existing);\n }\n else\n {\n ctx.Db.next_positions.Insert(new NextPosition\n {\n EntityId = pos.EntityId,\n X = nx,\n Y = ny\n });\n }\n }\n }\n }\n}",
+ "category": "schema",
+ "route_api_model": "gpt-5",
+ "golden_db": "schema-t-020-ecs-golden",
+ "llm_db": "schema-t-020-ecs-gpt-5-llm",
+ "work_dir_golden": "target/llm-runs/schema/t_020_ecs/csharp/server/golden",
+ "work_dir_llm": "target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm",
+ "scorer_details": {
+ "publish_error": {
"pass": false,
"partial": 0.0,
"notes": {
- "actual": 0,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM users"
+ "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/obj/Release/net8.0/wasi-wasm/SpacetimeDB.Codegen/SpacetimeDB.Codegen.Module/FFI.cs(27,32): warning CS8981: The type name 'entities' only contains lower-cased ascii characters. Such names may become reserved for the language. [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/obj/Release/net8.0/wasi-wasm/SpacetimeDB.Codegen/SpacetimeDB.Codegen.Module/FFI.cs(113,24): warning CS8981: The type name 'positions' only contains lower-cased ascii characters. Such names may become reserved for the language. [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/obj/Release/net8.0/wasi-wasm/SpacetimeDB.Codegen/SpacetimeDB.Codegen.Module/FFI.cs(156,24): warning CS8981: The type name 'velocities' only contains lower-cased ascii characters. Such names may become reserved for the language. [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: The \"EmitBundleObjectFiles\" task failed unexpectedly. [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: System.AggregateException: One or more errors occurred. (Pipe is broken.) (Pipe is broken.) (Pipe is broken.) (Pipe is broken.) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: ---> System.IO.IOException: Pipe is broken. [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.Pipes.PipeStream.CheckWriteOperations() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.StreamWriter.Flush(Boolean flushStream, Boolean flushEncoder) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.StreamWriter.Dispose(Boolean disposing) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.TextWriter.Dispose() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.<>c__DisplayClass27_1.b__2(Stream codeStream) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at Utils.TryRunProcess(TaskLoggingHelper logger, String path, String args, IDictionary`2 envVars, String workingDir, Boolean silent, Boolean logStdErrAsMessage, MessageImportance debugMessageImportance, String label, Action`1 inputProvider) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleObjectFiles.EmitBundleFile(String destinationFile, Action`1 EmitBundleFile) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.<>c__DisplayClass27_0.b__1(Int32 i, ParallelLoopState state) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.<>c__DisplayClass19_0`2.b__1(RangeWorker& currentWorker, Int64 timeout, Boolean& replicationDelegateYieldedBeforeCompletion) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: --- End of stack trace from previous location --- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.<>c__DisplayClass19_0`2.b__1(RangeWorker& currentWorker, Int64 timeout, Boolean& replicationDelegateYieldedBeforeCompletion) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.TaskReplicator.Replica.Execute() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: --- End of inner exception stack trace --- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.TaskReplicator.Run[TState](ReplicatableUserAction`1 action, ParallelOptions options, Boolean stopOnFirstFailure) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.ForWorker[TLocal,TInt](TInt fromInclusive, TInt toExclusive, ParallelOptions parallelOptions, Action`1 body, Action`2 bodyWithState, Func`4 bodyWithLocal, Func`1 localInit, Action`1 localFinally) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: --- End of stack trace from previous location --- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.ForWorker[TLocal,TInt](TInt fromInclusive, TInt toExclusive, ParallelOptions parallelOptions, Action`1 body, Action`2 bodyWithState, Func`4 bodyWithLocal, Func`1 localInit, Action`1 localFinally) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.For(Int32 fromInclusive, Int32 toExclusive, ParallelOptions parallelOptions, Action`2 body) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.Execute() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleObjectFiles.Execute() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at Microsoft.Build.BackEnd.TaskExecutionHost.Execute() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at Microsoft.Build.BackEnd.TaskBuilder.ExecuteInstantiatedTask(TaskExecutionHost taskExecutionHost, TaskLoggingContext taskLoggingContext, TaskHost taskHost, ItemBucket bucket, TaskExecutionMode howToExecuteTask) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: ---> (Inner Exception #1) System.IO.IOException: Pipe is broken. [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.Pipes.PipeStream.CheckWriteOperations() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.StreamWriter.Flush(Boolean flushStream, Boolean flushEncoder) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.StreamWriter.Dispose(Boolean disposing) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.TextWriter.Dispose() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.<>c__DisplayClass27_1.b__2(Stream codeStream) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at Utils.TryRunProcess(TaskLoggingHelper logger, String path, String args, IDictionary`2 envVars, String workingDir, Boolean silent, Boolean logStdErrAsMessage, MessageImportance debugMessageImportance, String label, Action`1 inputProvider) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleObjectFiles.EmitBundleFile(String destinationFile, Action`1 EmitBundleFile) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.<>c__DisplayClass27_0.b__1(Int32 i, ParallelLoopState state) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.<>c__DisplayClass19_0`2.b__1(RangeWorker& currentWorker, Int64 timeout, Boolean& replicationDelegateYieldedBeforeCompletion) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: --- End of stack trace from previous location --- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.<>c__DisplayClass19_0`2.b__1(RangeWorker& currentWorker, Int64 timeout, Boolean& replicationDelegateYieldedBeforeCompletion) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.TaskReplicator.Replica.Execute()<--- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: ---> (Inner Exception #2) System.IO.IOException: Pipe is broken. [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.Pipes.PipeStream.CheckWriteOperations() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.StreamWriter.Flush(Boolean flushStream, Boolean flushEncoder) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.StreamWriter.Dispose(Boolean disposing) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.TextWriter.Dispose() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.<>c__DisplayClass27_1.b__2(Stream codeStream) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at Utils.TryRunProcess(TaskLoggingHelper logger, String path, String args, IDictionary`2 envVars, String workingDir, Boolean silent, Boolean logStdErrAsMessage, MessageImportance debugMessageImportance, String label, Action`1 inputProvider) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleObjectFiles.EmitBundleFile(String destinationFile, Action`1 EmitBundleFile) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.<>c__DisplayClass27_0.b__1(Int32 i, ParallelLoopState state) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.<>c__DisplayClass19_0`2.b__1(RangeWorker& currentWorker, Int64 timeout, Boolean& replicationDelegateYieldedBeforeCompletion) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: --- End of stack trace from previous location --- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.<>c__DisplayClass19_0`2.b__1(RangeWorker& currentWorker, Int64 timeout, Boolean& replicationDelegateYieldedBeforeCompletion) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.TaskReplicator.Replica.Execute()<--- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: ---> (Inner Exception #3) System.IO.IOException: Pipe is broken. [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.Pipes.PipeStream.CheckWriteOperations() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.StreamWriter.Flush(Boolean flushStream, Boolean flushEncoder) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.StreamWriter.Dispose(Boolean disposing) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.IO.TextWriter.Dispose() [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.<>c__DisplayClass27_1.b__2(Stream codeStream) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at Utils.TryRunProcess(TaskLoggingHelper logger, String path, String args, IDictionary`2 envVars, String workingDir, Boolean silent, Boolean logStdErrAsMessage, MessageImportance debugMessageImportance, String label, Action`1 inputProvider) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleObjectFiles.EmitBundleFile(String destinationFile, Action`1 EmitBundleFile) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at EmitBundleBase.<>c__DisplayClass27_0.b__1(Int32 i, ParallelLoopState state) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.<>c__DisplayClass19_0`2.b__1(RangeWorker& currentWorker, Int64 timeout, Boolean& replicationDelegateYieldedBeforeCompletion) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: --- End of stack trace from previous location --- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.Parallel.<>c__DisplayClass19_0`2.b__1(RangeWorker& currentWorker, Int64 timeout, Boolean& replicationDelegateYieldedBeforeCompletion) [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: at System.Threading.Tasks.TaskReplicator.Replica.Execute()<--- [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n/usr/share/dotnet/packs/Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk/8.0.22/Sdk/WasiApp.Native.targets(370,5): error MSB4018: [/home/runner/work/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm/StdbModule.csproj]\n",
+ "phase": "build_or_publish"
}
}
},
"vendor": "openai",
- "started_at": "2026-01-04T18:03:02.557397448Z",
- "finished_at": "2026-01-04T18:03:57.769578396Z"
- }
- }
- },
- {
- "name": "o4-mini",
- "route_api_model": "o4-mini",
- "tasks": {
- "t_014_elementary_columns": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_014_elementary_columns",
+ "started_at": "2026-01-04T18:02:06.133458136Z",
+ "finished_at": "2026-01-04T18:03:02.314808993Z"
+ },
+ "t_021_multi_column_index": {
+ "hash": "6f75b0c555fd7577df52872b447d9237496b7f004485c56464181b8bfb7834de",
+ "task": "t_021_multi_column_index",
"lang": "csharp",
"golden_published": true,
- "model_name": "o4-mini",
- "total_tests": 3,
+ "model_name": "GPT-5",
+ "total_tests": 4,
"passed_tests": 3,
- "llm_output": null,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"logs\", Public = true)]\n [SpacetimeDB.Index.BTree(Name = \"by_user_day\", Columns = new[] { \"UserId\", \"Day\" })]\n public partial struct Log\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int Day;\n public string Message;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.logs.Insert(new Log { Id = 1, UserId = 7, Day = 1, Message = \"a\" });\n ctx.Db.logs.Insert(new Log { Id = 2, UserId = 7, Day = 2, Message = \"b\" });\n ctx.Db.logs.Insert(new Log { Id = 3, UserId = 9, Day = 1, Message = \"c\" });\n }\n}",
"category": "schema",
- "route_api_model": "o4-mini",
- "golden_db": "schema-t-014-elementary-columns-golden",
- "llm_db": "schema-t-014-elementary-columns-o4-mini-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_014_elementary_columns\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_014_elementary_columns\\csharp\\server\\o4-mini\\llm",
+ "route_api_model": "gpt-5",
+ "golden_db": "schema-t-021-multi-column-index-golden",
+ "llm_db": "schema-t-021-multi-column-index-gpt-5-llm",
+ "work_dir_golden": "target/llm-runs/schema/t_021_multi_column_index/csharp/server/golden",
+ "work_dir_llm": "target/llm-runs/schema/t_021_multi_column_index/csharp/server/gpt-5/llm",
"scorer_details": {
- "schema_parity": {
+ "mcindex_seed_count": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-014-elementary-columns-golden",
- "llm_db": "schema-t-014-elementary-columns-o4-mini-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
+ "actual": 3,
+ "expected": 3,
+ "sql": "SELECT COUNT(*) AS n FROM logs"
}
},
- "elementary_columns_row_parity": {
+ "mcindex_lookup_u7_d2": {
"pass": true,
"partial": 1.0,
"notes": {
- "args": [],
- "golden_db": "schema-t-014-elementary-columns-golden",
- "golden_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"",
- "llm_db": "schema-t-014-elementary-columns-o4-mini-llm",
- "llm_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"",
- "query": "SELECT Id, Count, Total, Price, Ratio, Active, Name FROM primitives WHERE Id=1",
- "reducer": "Seed",
- "server": "local"
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM logs WHERE UserId=7 AND Day=2"
}
},
- "elementary_columns_row_count": {
+ "mcindex_lookup_u7_d1": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM primitives WHERE Id=1"
+ "sql": "SELECT COUNT(*) AS n FROM logs WHERE UserId=7 AND Day=1"
+ }
+ },
+ "schema_parity": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-021-multi-column-index-golden`.\n",
+ "phase": "describe_golden"
}
}
},
"vendor": "openai",
- "started_at": "2025-10-19T23:14:48.289792Z",
- "finished_at": "2025-10-19T23:22:01.068608400Z"
- },
- "t_006_delete": {
+ "started_at": "2026-01-04T18:02:06.146746037Z",
+ "finished_at": "2026-01-04T18:03:32.116692104Z"
+ }
+ }
+ },
+ {
+ "name": "o4-mini",
+ "route_api_model": "o4-mini",
+ "tasks": {
+ "t_000_empty_reducers": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_006_delete",
+ "task": "t_000_empty_reducers",
"lang": "csharp",
"golden_published": true,
"model_name": "o4-mini",
- "total_tests": 3,
- "passed_tests": 3,
- "llm_output": null,
+ "total_tests": 1,
+ "passed_tests": 1,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_NoArgs(ReducerContext ctx)\n {\n }\n\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_WithInt(ReducerContext ctx, int count)\n {\n }\n\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_WithString(ReducerContext ctx, string name)\n {\n }\n\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_WithTwoArgs(ReducerContext ctx, int count, string name)\n {\n }\n\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_WithThreeArgs(ReducerContext ctx, bool active, float ratio, string label)\n {\n }\n}",
"category": "basics",
"route_api_model": "o4-mini",
- "golden_db": "basics-t-006-delete-golden",
- "llm_db": "basics-t-006-delete-o4-mini-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\o4-mini\\llm",
+ "golden_db": "basics-t-000-empty-reducers-golden",
+ "llm_db": "basics-t-000-empty-reducers-o4-mini-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\o4-mini\\llm",
"scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-006-delete-golden",
- "llm_db": "basics-t-006-delete-o4-mini-llm",
+ "golden_db": "basics-t-000-empty-reducers-golden",
+ "llm_db": "basics-t-000-empty-reducers-o4-mini-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
"tables_diff": null,
"tables_equal": true
}
- },
- "delete_user_count_zero": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 0,
- "expected": 0,
- "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=1"
- }
- },
- "seed_users_row": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
- }
}
},
"vendor": "openai",
- "started_at": "2025-10-19T23:14:47.644379900Z",
- "finished_at": "2025-10-19T23:21:59.142268900Z"
+ "started_at": "2025-10-21T23:58:20.283902800Z",
+ "finished_at": "2025-10-21T23:59:25.402595100Z"
},
- "t_018_constraints": {
+ "t_001_basic_tables": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_018_constraints",
+ "task": "t_001_basic_tables",
"lang": "csharp",
"golden_published": true,
"model_name": "o4-mini",
- "total_tests": 3,
- "passed_tests": 3,
+ "total_tests": 1,
+ "passed_tests": 0,
"llm_output": null,
- "category": "schema",
+ "category": "basics",
"route_api_model": "o4-mini",
- "golden_db": "schema-t-018-constraints-golden",
- "llm_db": "schema-t-018-constraints-o4-mini-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\o4-mini\\llm",
+ "golden_db": "basics-t-001-basic-tables-golden",
+ "llm_db": "basics-t-001-basic-tables-o4-mini-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\o4-mini\\llm",
"scorer_details": {
- "constraints_seed_two_rows": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM accounts WHERE Id=2"
- }
- },
- "constraints_row_parity_after_seed": {
- "pass": true,
- "partial": 1.0,
+ "publish_error": {
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "args": [],
- "golden_db": "schema-t-018-constraints-golden",
- "golden_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"",
- "llm_db": "schema-t-018-constraints-o4-mini-llm",
- "llm_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"",
- "query": "SELECT Id, Email, Name FROM accounts WHERE Id=1",
- "reducer": "Seed",
- "server": "local"
+ "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\o4-mini\\llm\\Lib.cs(6,40): error CS1016: Named attribute argument expected [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\o4-mini\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\o4-mini\\llm\\Lib.cs(6,40): error CS0103: The name 'Public' does not exist in the current context [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\o4-mini\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\o4-mini\\llm\\Lib.cs(6,6): error CS1729: 'TableAttribute' does not contain a constructor that takes 1 arguments [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\o4-mini\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\o4-mini\\llm\\Lib.cs(16,43): error CS1016: Named attribute argument expected [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\o4-mini\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\o4-mini\\llm\\Lib.cs(16,43): error CS0103: The name 'Public' does not exist in the current context [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\o4-mini\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\o4-mini\\llm\\Lib.cs(16,6): error CS1729: 'TableAttribute' does not contain a constructor that takes 1 arguments [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\o4-mini\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\o4-mini\\llm\\Lib.cs(26,40): error CS1016: Named attribute argument expected [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\o4-mini\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\o4-mini\\llm\\Lib.cs(26,40): error CS0103: The name 'Public' does not exist in the current context [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\o4-mini\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\o4-mini\\llm\\Lib.cs(26,6): error CS1729: 'TableAttribute' does not contain a constructor that takes 1 arguments [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\o4-mini\\llm\\StdbModule.csproj]\r\n",
+ "phase": "build_or_publish"
}
- },
+ }
+ },
+ "vendor": "openai",
+ "started_at": "2025-10-19T23:14:47.248478100Z",
+ "finished_at": "2025-10-19T23:15:36.126687400Z"
+ },
+ "t_002_scheduled_table": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_002_scheduled_table",
+ "lang": "csharp",
+ "golden_published": true,
+ "model_name": "o4-mini",
+ "total_tests": 1,
+ "passed_tests": 1,
+ "llm_output": null,
+ "category": "basics",
+ "route_api_model": "o4-mini",
+ "golden_db": "basics-t-002-scheduled-table-golden",
+ "llm_db": "basics-t-002-scheduled-table-o4-mini-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_002_scheduled_table\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_002_scheduled_table\\csharp\\server\\o4-mini\\llm",
+ "scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-018-constraints-golden",
- "llm_db": "schema-t-018-constraints-o4-mini-llm",
+ "golden_db": "basics-t-002-scheduled-table-golden",
+ "llm_db": "basics-t-002-scheduled-table-o4-mini-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -999,8 +932,8 @@
}
},
"vendor": "openai",
- "started_at": "2025-10-19T23:14:48.744809600Z",
- "finished_at": "2025-10-19T23:21:58.431013200Z"
+ "started_at": "2025-10-19T23:14:47.329189800Z",
+ "finished_at": "2025-10-19T23:21:58.845517100Z"
},
"t_003_struct_in_table": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
@@ -1036,6 +969,35 @@
"started_at": "2025-10-19T23:14:47.408425700Z",
"finished_at": "2025-10-19T23:22:00.509202800Z"
},
+ "t_004_insert": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_004_insert",
+ "lang": "csharp",
+ "golden_published": true,
+ "model_name": "o4-mini",
+ "total_tests": 2,
+ "passed_tests": 0,
+ "llm_output": null,
+ "category": "basics",
+ "route_api_model": "o4-mini",
+ "golden_db": "basics-t-004-insert-golden",
+ "llm_db": "basics-t-004-insert-o4-mini-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_004_insert\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_004_insert\\csharp\\server\\o4-mini\\llm",
+ "scorer_details": {
+ "publish_error": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_004_insert\\csharp\\server\\o4-mini\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'users' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_004_insert\\csharp\\server\\o4-mini\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_004_insert\\csharp\\server\\o4-mini\\llm\\Lib.cs(19,16): error CS1061: 'Local' does not contain a definition for 'Users' and no accessible extension method 'Users' accepting a first argument of type 'Local' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_004_insert\\csharp\\server\\o4-mini\\llm\\StdbModule.csproj]\r\n",
+ "phase": "build_or_publish"
+ }
+ }
+ },
+ "vendor": "openai",
+ "started_at": "2025-10-19T23:14:47.487751600Z",
+ "finished_at": "2025-10-19T23:15:49.003673700Z"
+ },
"t_005_update": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
"task": "t_005_update",
@@ -1052,12 +1014,25 @@
"work_dir_golden": "target\\llm-runs\\basics\\t_005_update\\csharp\\server\\golden",
"work_dir_llm": "target\\llm-runs\\basics\\t_005_update\\csharp\\server\\o4-mini\\llm",
"scorer_details": {
- "seed_users_row": {
- "pass": false,
- "partial": 0.0,
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
"notes": {
- "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: IndexError: Unique constraint violation 'users_Id_idx_btree' in table 'users': column(s): '[\"Id\"]' value: 1\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:3000/v1/database/c2009d82dc6baa267ea089f997a830b64b31543e098dcb102ae1025dce7d1f73/sql)\n",
- "phase": "sql_golden",
+ "golden_db": "basics-t-005-update-golden",
+ "llm_db": "basics-t-005-update-o4-mini-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
+ }
+ },
+ "seed_users_row": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: IndexError: Unique constraint violation 'users_Id_idx_btree' in table 'users': column(s): '[\"Id\"]' value: 1\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:3000/v1/database/c2009d82dc6baa267ea089f997a830b64b31543e098dcb102ae1025dce7d1f73/sql)\n",
+ "phase": "sql_golden",
"sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
}
},
@@ -1068,47 +1043,50 @@
"error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: SpacetimeDB.NoSuchRowException: The row was not found, e.g., in an update call\n at SpacetimeDB.Internal.FFI.CheckedStatus.Marshaller.ConvertToManaged(Errno )\n at SpacetimeDB.Internal.FFI.datastore_update_bsatn(TableId , IndexId , Span`1 , UInt32& )\n at SpacetimeDB.Internal.UniqueIndex`4[[SpacetimeDB.Internal.TableHandles.users, StdbModule, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null],[Module.Users, StdbModule, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null],[System.Int32, System.Private.CoreLib, Version=8.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e],[SpacetimeDB.BSATN.I32, SpacetimeDB.BSATN.Runtime, Version=1.5.0.0, Culture=neutral, PublicKeyToken=null]].DoUpdate(Users )\n at SpacetimeDB.Internal.TableHandles.users.IdUniqueIndex.Update(Users )\n at Module.UpdateUser(ReducerContext , Int32 , String , Int32 , Boolean )\n at ModuleRegistration.UpdateUser.Invoke(BinaryReader , IReducerContext )\n at SpacetimeDB.Internal.Module.__call_reducer__(UInt32 id, UInt64 sender_0, UInt64 sender_1, UInt64 sender_2, UInt64 sender_3, UInt64 conn_id_0, UInt64 conn_id_1, Timestamp timestamp, BytesSource args, BytesSink error)\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:3000/v1/database/c2002ae1190f79d06705c8050b4825767f36b610058869860bdb2f25a3d2606c/call/UpdateUser)\n",
"phase": "call_reducer_llm"
}
- },
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "golden_db": "basics-t-005-update-golden",
- "llm_db": "basics-t-005-update-o4-mini-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
- }
}
},
"vendor": "openai",
"started_at": "2025-10-19T23:14:47.566753600Z",
"finished_at": "2025-10-19T23:21:58.713208900Z"
},
- "t_002_scheduled_table": {
+ "t_006_delete": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_002_scheduled_table",
+ "task": "t_006_delete",
"lang": "csharp",
"golden_published": true,
"model_name": "o4-mini",
- "total_tests": 1,
- "passed_tests": 1,
+ "total_tests": 3,
+ "passed_tests": 3,
"llm_output": null,
"category": "basics",
"route_api_model": "o4-mini",
- "golden_db": "basics-t-002-scheduled-table-golden",
- "llm_db": "basics-t-002-scheduled-table-o4-mini-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_002_scheduled_table\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_002_scheduled_table\\csharp\\server\\o4-mini\\llm",
+ "golden_db": "basics-t-006-delete-golden",
+ "llm_db": "basics-t-006-delete-o4-mini-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\o4-mini\\llm",
"scorer_details": {
+ "seed_users_row": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
+ }
+ },
+ "delete_user_count_zero": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 0,
+ "expected": 0,
+ "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=1"
+ }
+ },
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-002-scheduled-table-golden",
- "llm_db": "basics-t-002-scheduled-table-o4-mini-llm",
+ "golden_db": "basics-t-006-delete-golden",
+ "llm_db": "basics-t-006-delete-o4-mini-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -1118,37 +1096,68 @@
}
},
"vendor": "openai",
- "started_at": "2025-10-19T23:14:47.329189800Z",
- "finished_at": "2025-10-19T23:21:58.845517100Z"
+ "started_at": "2025-10-19T23:14:47.644379900Z",
+ "finished_at": "2025-10-19T23:21:59.142268900Z"
},
- "t_004_insert": {
+ "t_007_crud": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_004_insert",
+ "task": "t_007_crud",
"lang": "csharp",
"golden_published": true,
"model_name": "o4-mini",
- "total_tests": 2,
- "passed_tests": 0,
+ "total_tests": 4,
+ "passed_tests": 2,
"llm_output": null,
"category": "basics",
"route_api_model": "o4-mini",
- "golden_db": "basics-t-004-insert-golden",
- "llm_db": "basics-t-004-insert-o4-mini-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_004_insert\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_004_insert\\csharp\\server\\o4-mini\\llm",
+ "golden_db": "basics-t-007-crud-golden",
+ "llm_db": "basics-t-007-crud-o4-mini-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\o4-mini\\llm",
"scorer_details": {
- "publish_error": {
+ "crud_total_count_one": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_004_insert\\csharp\\server\\o4-mini\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'users' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_004_insert\\csharp\\server\\o4-mini\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_004_insert\\csharp\\server\\o4-mini\\llm\\Lib.cs(19,16): error CS1061: 'Local' does not contain a definition for 'Users' and no accessible extension method 'Users' accepting a first argument of type 'Local' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_004_insert\\csharp\\server\\o4-mini\\llm\\StdbModule.csproj]\r\n",
- "phase": "build_or_publish"
+ "actual": 0,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM users"
+ }
+ },
+ "crud_row_id1_parity": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: SpacetimeDB.UniqueConstraintViolationException: Value with given unique identifier already exists\n at SpacetimeDB.Internal.FFI.CheckedStatus.Marshaller.ConvertToManaged(Errno )\n at SpacetimeDB.Internal.FFI.datastore_insert_bsatn(TableId , Span`1 , UInt32& )\n at SpacetimeDB.Internal.ITableView`2[[SpacetimeDB.Internal.TableHandles.users, StdbModule, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null],[Module.User, StdbModule, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null]].DoInsert(User )\n at SpacetimeDB.Internal.TableHandles.users.Insert(User )\n at Module.Crud(ReducerContext )\n at ModuleRegistration.Crud.Invoke(BinaryReader , IReducerContext )\n at SpacetimeDB.Internal.Module.__call_reducer__(UInt32 id, UInt64 sender_0, UInt64 sender_1, UInt64 sender_2, UInt64 sender_3, UInt64 conn_id_0, UInt64 conn_id_1, Timestamp timestamp, BytesSource args, BytesSink error)\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:3000/v1/database/c200b8341c6ccfc8fe52f225fa1fbfc9b53d0de44b74ac5b291d8c94ef9e9fab/call/Crud)\n",
+ "phase": "call_reducer_golden"
+ }
+ },
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "golden_db": "basics-t-007-crud-golden",
+ "llm_db": "basics-t-007-crud-o4-mini-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
+ }
+ },
+ "crud_row_id2_deleted": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 0,
+ "expected": 0,
+ "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=2"
}
}
},
"vendor": "openai",
- "started_at": "2025-10-19T23:14:47.487751600Z",
- "finished_at": "2025-10-19T23:15:49.003673700Z"
+ "started_at": "2025-10-19T23:14:47.721825Z",
+ "finished_at": "2025-10-19T23:22:01.386410500Z"
},
"t_008_index_lookup": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
@@ -1179,235 +1188,89 @@
"started_at": "2025-10-19T23:14:47.801378400Z",
"finished_at": "2025-10-19T23:16:06.930643500Z"
},
- "t_001_basic_tables": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_001_basic_tables",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "o4-mini",
- "total_tests": 1,
- "passed_tests": 0,
- "llm_output": null,
- "category": "basics",
- "route_api_model": "o4-mini",
- "golden_db": "basics-t-001-basic-tables-golden",
- "llm_db": "basics-t-001-basic-tables-o4-mini-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\o4-mini\\llm",
- "scorer_details": {
- "publish_error": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\o4-mini\\llm\\Lib.cs(6,40): error CS1016: Named attribute argument expected [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\o4-mini\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\o4-mini\\llm\\Lib.cs(6,40): error CS0103: The name 'Public' does not exist in the current context [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\o4-mini\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\o4-mini\\llm\\Lib.cs(6,6): error CS1729: 'TableAttribute' does not contain a constructor that takes 1 arguments [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\o4-mini\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\o4-mini\\llm\\Lib.cs(16,43): error CS1016: Named attribute argument expected [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\o4-mini\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\o4-mini\\llm\\Lib.cs(16,43): error CS0103: The name 'Public' does not exist in the current context [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\o4-mini\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\o4-mini\\llm\\Lib.cs(16,6): error CS1729: 'TableAttribute' does not contain a constructor that takes 1 arguments [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\o4-mini\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\o4-mini\\llm\\Lib.cs(26,40): error CS1016: Named attribute argument expected [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\o4-mini\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\o4-mini\\llm\\Lib.cs(26,40): error CS0103: The name 'Public' does not exist in the current context [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\o4-mini\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\o4-mini\\llm\\Lib.cs(26,6): error CS1729: 'TableAttribute' does not contain a constructor that takes 1 arguments [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\o4-mini\\llm\\StdbModule.csproj]\r\n",
- "phase": "build_or_publish"
- }
- }
- },
- "vendor": "openai",
- "started_at": "2025-10-19T23:14:47.248478100Z",
- "finished_at": "2025-10-19T23:15:36.126687400Z"
- },
- "t_010_connect": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_010_connect",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "o4-mini",
- "total_tests": 1,
- "passed_tests": 1,
- "llm_output": null,
- "category": "basics",
- "route_api_model": "o4-mini",
- "golden_db": "basics-t-010-connect-golden",
- "llm_db": "basics-t-010-connect-o4-mini-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_010_connect\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_010_connect\\csharp\\server\\o4-mini\\llm",
- "scorer_details": {
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "golden_db": "basics-t-010-connect-golden",
- "llm_db": "basics-t-010-connect-o4-mini-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
- }
- }
- },
- "vendor": "openai",
- "started_at": "2025-10-19T23:14:47.956244400Z",
- "finished_at": "2025-10-19T23:21:59.458614600Z"
- },
- "t_021_multi_column_index": {
+ "t_009_init": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_021_multi_column_index",
+ "task": "t_009_init",
"lang": "csharp",
"golden_published": true,
"model_name": "o4-mini",
"total_tests": 4,
"passed_tests": 4,
"llm_output": null,
- "category": "schema",
+ "category": "basics",
"route_api_model": "o4-mini",
- "golden_db": "schema-t-021-multi-column-index-golden",
- "llm_db": "schema-t-021-multi-column-index-o4-mini-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\o4-mini\\llm",
+ "golden_db": "basics-t-009-init-golden",
+ "llm_db": "basics-t-009-init-o4-mini-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_009_init\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_009_init\\csharp\\server\\o4-mini\\llm",
"scorer_details": {
- "mcindex_seed_count": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 3,
- "expected": 3,
- "sql": "SELECT COUNT(*) AS n FROM logs"
- }
- },
- "mcindex_lookup_u7_d2": {
+ "init_seed_bob": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM logs WHERE UserId=7 AND Day=2"
- }
- },
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "golden_db": "schema-t-021-multi-column-index-golden",
- "llm_db": "schema-t-021-multi-column-index-o4-mini-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
+ "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false"
}
},
- "mcindex_lookup_u7_d1": {
+ "init_seed_alice": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM logs WHERE UserId=7 AND Day=1"
+ "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true"
}
- }
- },
- "vendor": "openai",
- "started_at": "2025-10-19T23:15:49.473810800Z",
- "finished_at": "2025-10-19T23:21:56.880671200Z"
- },
- "t_016_sum_type_columns": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_016_sum_type_columns",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "o4-mini",
- "total_tests": 3,
- "passed_tests": 3,
- "llm_output": null,
- "category": "schema",
- "route_api_model": "o4-mini",
- "golden_db": "schema-t-016-sum-type-columns-golden",
- "llm_db": "schema-t-016-sum-type-columns-o4-mini-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\o4-mini\\llm",
- "scorer_details": {
- "sum_type_columns_row_parity": {
+ },
+ "init_total_two": {
"pass": true,
"partial": 1.0,
"notes": {
- "args": [],
- "golden_db": "schema-t-016-sum-type-columns-golden",
- "golden_out": "Id | A | B ----+--------------------------+--------------------------------------- 1 | (Circle = (Radius = 10)) | (Rectangle = (Width = 4, Height = 6))",
- "llm_db": "schema-t-016-sum-type-columns-o4-mini-llm",
- "llm_out": "Id | A | B ----+--------------------------+--------------------------------------- 1 | (Circle = (Radius = 10)) | (Rectangle = (Width = 4, Height = 6))",
- "query": "SELECT Id, A, B FROM drawings WHERE Id=1",
- "reducer": "Seed",
- "server": "local"
+ "actual": 2,
+ "expected": 2,
+ "sql": "SELECT COUNT(*) AS n FROM users"
}
},
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-016-sum-type-columns-golden",
- "llm_db": "schema-t-016-sum-type-columns-o4-mini-llm",
+ "golden_db": "basics-t-009-init-golden",
+ "llm_db": "basics-t-009-init-o4-mini-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
"tables_diff": null,
"tables_equal": true
}
- },
- "sum_type_columns_row_count": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM drawings WHERE Id=1"
- }
}
},
"vendor": "openai",
- "started_at": "2025-10-19T23:14:48.503910600Z",
- "finished_at": "2025-10-19T23:22:00.341656200Z"
+ "started_at": "2025-10-19T23:14:47.878935900Z",
+ "finished_at": "2025-10-19T23:21:58.086857300Z"
},
- "t_007_crud": {
+ "t_010_connect": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_007_crud",
+ "task": "t_010_connect",
"lang": "csharp",
"golden_published": true,
"model_name": "o4-mini",
- "total_tests": 4,
- "passed_tests": 2,
+ "total_tests": 1,
+ "passed_tests": 1,
"llm_output": null,
"category": "basics",
"route_api_model": "o4-mini",
- "golden_db": "basics-t-007-crud-golden",
- "llm_db": "basics-t-007-crud-o4-mini-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\o4-mini\\llm",
+ "golden_db": "basics-t-010-connect-golden",
+ "llm_db": "basics-t-010-connect-o4-mini-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_010_connect\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_010_connect\\csharp\\server\\o4-mini\\llm",
"scorer_details": {
- "crud_row_id1_parity": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: SpacetimeDB.UniqueConstraintViolationException: Value with given unique identifier already exists\n at SpacetimeDB.Internal.FFI.CheckedStatus.Marshaller.ConvertToManaged(Errno )\n at SpacetimeDB.Internal.FFI.datastore_insert_bsatn(TableId , Span`1 , UInt32& )\n at SpacetimeDB.Internal.ITableView`2[[SpacetimeDB.Internal.TableHandles.users, StdbModule, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null],[Module.User, StdbModule, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null]].DoInsert(User )\n at SpacetimeDB.Internal.TableHandles.users.Insert(User )\n at Module.Crud(ReducerContext )\n at ModuleRegistration.Crud.Invoke(BinaryReader , IReducerContext )\n at SpacetimeDB.Internal.Module.__call_reducer__(UInt32 id, UInt64 sender_0, UInt64 sender_1, UInt64 sender_2, UInt64 sender_3, UInt64 conn_id_0, UInt64 conn_id_1, Timestamp timestamp, BytesSource args, BytesSink error)\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:3000/v1/database/c200b8341c6ccfc8fe52f225fa1fbfc9b53d0de44b74ac5b291d8c94ef9e9fab/call/Crud)\n",
- "phase": "call_reducer_golden"
- }
- },
- "crud_total_count_one": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "actual": 0,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM users"
- }
- },
- "crud_row_id2_deleted": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 0,
- "expected": 0,
- "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=2"
- }
- },
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-007-crud-golden",
- "llm_db": "basics-t-007-crud-o4-mini-llm",
+ "golden_db": "basics-t-010-connect-golden",
+ "llm_db": "basics-t-010-connect-o4-mini-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -1417,37 +1280,8 @@
}
},
"vendor": "openai",
- "started_at": "2025-10-19T23:14:47.721825Z",
- "finished_at": "2025-10-19T23:22:01.386410500Z"
- },
- "t_020_ecs": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_020_ecs",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "o4-mini",
- "total_tests": 5,
- "passed_tests": 0,
- "llm_output": null,
- "category": "schema",
- "route_api_model": "o4-mini",
- "golden_db": "schema-t-020-ecs-golden",
- "llm_db": "schema-t-020-ecs-o4-mini-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\o4-mini\\llm",
- "scorer_details": {
- "publish_error": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\o4-mini\\llm\\Lib.cs(5,43): error CS1016: Named attribute argument expected [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\o4-mini\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\o4-mini\\llm\\Lib.cs(5,43): error CS0103: The name 'Public' does not exist in the current context [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\o4-mini\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\o4-mini\\llm\\Lib.cs(5,6): error CS1729: 'TableAttribute' does not contain a constructor that takes 1 arguments [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\o4-mini\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\o4-mini\\llm\\Lib.cs(11,44): error CS1016: Named attribute argument expected [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\o4-mini\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\o4-mini\\llm\\Lib.cs(11,44): error CS0103: The name 'Public' does not exist in the current context [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\o4-mini\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\o4-mini\\llm\\Lib.cs(11,6): error CS1729: 'TableAttribute' does not contain a constructor that takes 1 arguments [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\o4-mini\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\o4-mini\\llm\\Lib.cs(19,45): error CS1016: Named attribute argument expected [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\o4-mini\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\o4-mini\\llm\\Lib.cs(19,45): error CS0103: The name 'Public' does not exist in the current context [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\o4-mini\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\o4-mini\\llm\\Lib.cs(19,6): error CS1729: 'TableAttribute' does not contain a constructor that takes 1 arguments [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\o4-mini\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\o4-mini\\llm\\Lib.cs(27,49): error CS1016: Named attribute argument expected [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\o4-mini\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\o4-mini\\llm\\Lib.cs(27,49): error CS0103: The name 'Public' does not exist in the current context [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\o4-mini\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\o4-mini\\llm\\Lib.cs(27,6): error CS1729: 'TableAttribute' does not contain a constructor that takes 1 arguments [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\o4-mini\\llm\\StdbModule.csproj]\r\n",
- "phase": "build_or_publish"
- }
- }
- },
- "vendor": "openai",
- "started_at": "2025-10-19T23:15:36.126968800Z",
- "finished_at": "2025-10-19T23:16:46.141457Z"
+ "started_at": "2025-10-19T23:14:47.956244400Z",
+ "finished_at": "2025-10-19T23:21:59.458614600Z"
},
"t_011_helper_function": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
@@ -1465,22 +1299,13 @@
"work_dir_golden": "target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\golden",
"work_dir_llm": "target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\o4-mini\\llm",
"scorer_details": {
- "helper_func_sum_parity": {
+ "helper_func_sum_abs": {
"pass": true,
"partial": 1.0,
"notes": {
- "args": [
- 1,
- 2,
- 3
- ],
- "golden_db": "basics-t-011-helper-function-golden",
- "golden_out": "Id | Sum ----+----- 1 | 5",
- "llm_db": "basics-t-011-helper-function-o4-mini-llm",
- "llm_out": "Id | Sum ----+----- 1 | 5",
- "query": "SELECT Id, Sum FROM results WHERE Id=1",
- "reducer": "ComputeSum",
- "server": "local"
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1 AND Sum=5"
}
},
"schema_parity": {
@@ -1496,13 +1321,22 @@
"tables_equal": true
}
},
- "helper_func_sum_abs": {
+ "helper_func_sum_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1 AND Sum=5"
+ "args": [
+ 1,
+ 2,
+ 3
+ ],
+ "golden_db": "basics-t-011-helper-function-golden",
+ "golden_out": "Id | Sum ----+----- 1 | 5",
+ "llm_db": "basics-t-011-helper-function-o4-mini-llm",
+ "llm_out": "Id | Sum ----+----- 1 | 5",
+ "query": "SELECT Id, Sum FROM results WHERE Id=1",
+ "reducer": "ComputeSum",
+ "server": "local"
}
}
},
@@ -1510,6 +1344,35 @@
"started_at": "2025-10-19T23:14:48.032931500Z",
"finished_at": "2025-10-19T23:21:59.849948300Z"
},
+ "t_012_spacetime_product_type": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_012_spacetime_product_type",
+ "lang": "csharp",
+ "golden_published": true,
+ "model_name": "o4-mini",
+ "total_tests": 3,
+ "passed_tests": 0,
+ "llm_output": null,
+ "category": "schema",
+ "route_api_model": "o4-mini",
+ "golden_db": "schema-t-012-spacetime-product-type-golden",
+ "llm_db": "schema-t-012-spacetime-product-type-o4-mini-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\o4-mini\\llm",
+ "scorer_details": {
+ "publish_error": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\o4-mini\\llm\\Lib.cs(13,42): error CS1016: Named attribute argument expected [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\o4-mini\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\o4-mini\\llm\\Lib.cs(13,42): error CS0103: The name 'Public' does not exist in the current context [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\o4-mini\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\o4-mini\\llm\\Lib.cs(13,6): error CS1729: 'TableAttribute' does not contain a constructor that takes 1 arguments [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\o4-mini\\llm\\StdbModule.csproj]\r\n",
+ "phase": "build_or_publish"
+ }
+ }
+ },
+ "vendor": "openai",
+ "started_at": "2025-10-19T23:14:48.113283300Z",
+ "finished_at": "2025-10-19T23:15:51.073350Z"
+ },
"t_013_spacetime_sum_type": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
"task": "t_013_spacetime_sum_type",
@@ -1539,39 +1402,62 @@
"started_at": "2025-10-19T23:14:48.193276500Z",
"finished_at": "2025-10-19T23:15:49.471287400Z"
},
- "t_000_empty_reducers": {
+ "t_014_elementary_columns": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_000_empty_reducers",
+ "task": "t_014_elementary_columns",
"lang": "csharp",
"golden_published": true,
"model_name": "o4-mini",
- "total_tests": 1,
- "passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_NoArgs(ReducerContext ctx)\n {\n }\n\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_WithInt(ReducerContext ctx, int count)\n {\n }\n\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_WithString(ReducerContext ctx, string name)\n {\n }\n\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_WithTwoArgs(ReducerContext ctx, int count, string name)\n {\n }\n\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_WithThreeArgs(ReducerContext ctx, bool active, float ratio, string label)\n {\n }\n}",
- "category": "basics",
+ "total_tests": 3,
+ "passed_tests": 3,
+ "llm_output": null,
+ "category": "schema",
"route_api_model": "o4-mini",
- "golden_db": "basics-t-000-empty-reducers-golden",
- "llm_db": "basics-t-000-empty-reducers-o4-mini-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\o4-mini\\llm",
+ "golden_db": "schema-t-014-elementary-columns-golden",
+ "llm_db": "schema-t-014-elementary-columns-o4-mini-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_014_elementary_columns\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_014_elementary_columns\\csharp\\server\\o4-mini\\llm",
"scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-000-empty-reducers-golden",
- "llm_db": "basics-t-000-empty-reducers-o4-mini-llm",
+ "golden_db": "schema-t-014-elementary-columns-golden",
+ "llm_db": "schema-t-014-elementary-columns-o4-mini-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
"tables_diff": null,
"tables_equal": true
}
+ },
+ "elementary_columns_row_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "args": [],
+ "golden_db": "schema-t-014-elementary-columns-golden",
+ "golden_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"",
+ "llm_db": "schema-t-014-elementary-columns-o4-mini-llm",
+ "llm_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"",
+ "query": "SELECT Id, Count, Total, Price, Ratio, Active, Name FROM primitives WHERE Id=1",
+ "reducer": "Seed",
+ "server": "local"
+ }
+ },
+ "elementary_columns_row_count": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM primitives WHERE Id=1"
+ }
}
},
"vendor": "openai",
- "started_at": "2025-10-21T23:58:20.283902800Z",
- "finished_at": "2025-10-21T23:59:25.402595100Z"
+ "started_at": "2025-10-19T23:14:48.289792Z",
+ "finished_at": "2025-10-19T23:22:01.068608400Z"
},
"t_015_product_type_columns": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
@@ -1589,18 +1475,13 @@
"work_dir_golden": "target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\golden",
"work_dir_llm": "target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\o4-mini\\llm",
"scorer_details": {
- "product_type_columns_row_parity": {
+ "product_type_columns_row_count": {
"pass": true,
"partial": 1.0,
"notes": {
- "args": [],
- "golden_db": "schema-t-015-product-type-columns-golden",
- "golden_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)",
- "llm_db": "schema-t-015-product-type-columns-o4-mini-llm",
- "llm_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)",
- "query": "SELECT Id, Home, Work, Pos FROM profiles WHERE Id=1",
- "reducer": "Seed",
- "server": "local"
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM profiles WHERE Id=1"
}
},
"schema_parity": {
@@ -1616,13 +1497,18 @@
"tables_equal": true
}
},
- "product_type_columns_row_count": {
+ "product_type_columns_row_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM profiles WHERE Id=1"
+ "args": [],
+ "golden_db": "schema-t-015-product-type-columns-golden",
+ "golden_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)",
+ "llm_db": "schema-t-015-product-type-columns-o4-mini-llm",
+ "llm_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)",
+ "query": "SELECT Id, Home, Work, Pos FROM profiles WHERE Id=1",
+ "reducer": "Seed",
+ "server": "local"
}
}
},
@@ -1630,14 +1516,71 @@
"started_at": "2025-10-19T23:14:48.389461500Z",
"finished_at": "2025-10-19T23:21:57.778496700Z"
},
- "t_017_scheduled_columns": {
+ "t_016_sum_type_columns": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_017_scheduled_columns",
+ "task": "t_016_sum_type_columns",
"lang": "csharp",
"golden_published": true,
"model_name": "o4-mini",
- "total_tests": 2,
- "passed_tests": 2,
+ "total_tests": 3,
+ "passed_tests": 3,
+ "llm_output": null,
+ "category": "schema",
+ "route_api_model": "o4-mini",
+ "golden_db": "schema-t-016-sum-type-columns-golden",
+ "llm_db": "schema-t-016-sum-type-columns-o4-mini-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\o4-mini\\llm",
+ "scorer_details": {
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "golden_db": "schema-t-016-sum-type-columns-golden",
+ "llm_db": "schema-t-016-sum-type-columns-o4-mini-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
+ }
+ },
+ "sum_type_columns_row_count": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM drawings WHERE Id=1"
+ }
+ },
+ "sum_type_columns_row_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "args": [],
+ "golden_db": "schema-t-016-sum-type-columns-golden",
+ "golden_out": "Id | A | B ----+--------------------------+--------------------------------------- 1 | (Circle = (Radius = 10)) | (Rectangle = (Width = 4, Height = 6))",
+ "llm_db": "schema-t-016-sum-type-columns-o4-mini-llm",
+ "llm_out": "Id | A | B ----+--------------------------+--------------------------------------- 1 | (Circle = (Radius = 10)) | (Rectangle = (Width = 4, Height = 6))",
+ "query": "SELECT Id, A, B FROM drawings WHERE Id=1",
+ "reducer": "Seed",
+ "server": "local"
+ }
+ }
+ },
+ "vendor": "openai",
+ "started_at": "2025-10-19T23:14:48.503910600Z",
+ "finished_at": "2025-10-19T23:22:00.341656200Z"
+ },
+ "t_017_scheduled_columns": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_017_scheduled_columns",
+ "lang": "csharp",
+ "golden_published": true,
+ "model_name": "o4-mini",
+ "total_tests": 2,
+ "passed_tests": 2,
"llm_output": null,
"category": "schema",
"route_api_model": "o4-mini",
@@ -1673,37 +1616,28 @@
"started_at": "2025-10-19T23:14:48.629575200Z",
"finished_at": "2025-10-19T23:21:59.322132600Z"
},
- "t_009_init": {
+ "t_018_constraints": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_009_init",
+ "task": "t_018_constraints",
"lang": "csharp",
"golden_published": true,
"model_name": "o4-mini",
- "total_tests": 4,
- "passed_tests": 4,
+ "total_tests": 3,
+ "passed_tests": 3,
"llm_output": null,
- "category": "basics",
+ "category": "schema",
"route_api_model": "o4-mini",
- "golden_db": "basics-t-009-init-golden",
- "llm_db": "basics-t-009-init-o4-mini-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_009_init\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_009_init\\csharp\\server\\o4-mini\\llm",
+ "golden_db": "schema-t-018-constraints-golden",
+ "llm_db": "schema-t-018-constraints-o4-mini-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\o4-mini\\llm",
"scorer_details": {
- "init_seed_alice": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true"
- }
- },
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-009-init-golden",
- "llm_db": "basics-t-009-init-o4-mini-llm",
+ "golden_db": "schema-t-018-constraints-golden",
+ "llm_db": "schema-t-018-constraints-o4-mini-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -1711,57 +1645,33 @@
"tables_equal": true
}
},
- "init_seed_bob": {
+ "constraints_seed_two_rows": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false"
+ "sql": "SELECT COUNT(*) AS n FROM accounts WHERE Id=2"
}
},
- "init_total_two": {
+ "constraints_row_parity_after_seed": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 2,
- "expected": 2,
- "sql": "SELECT COUNT(*) AS n FROM users"
- }
- }
- },
- "vendor": "openai",
- "started_at": "2025-10-19T23:14:47.878935900Z",
- "finished_at": "2025-10-19T23:21:58.086857300Z"
- },
- "t_012_spacetime_product_type": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_012_spacetime_product_type",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "o4-mini",
- "total_tests": 3,
- "passed_tests": 0,
- "llm_output": null,
- "category": "schema",
- "route_api_model": "o4-mini",
- "golden_db": "schema-t-012-spacetime-product-type-golden",
- "llm_db": "schema-t-012-spacetime-product-type-o4-mini-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\o4-mini\\llm",
- "scorer_details": {
- "publish_error": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\o4-mini\\llm\\Lib.cs(13,42): error CS1016: Named attribute argument expected [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\o4-mini\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\o4-mini\\llm\\Lib.cs(13,42): error CS0103: The name 'Public' does not exist in the current context [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\o4-mini\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\o4-mini\\llm\\Lib.cs(13,6): error CS1729: 'TableAttribute' does not contain a constructor that takes 1 arguments [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\o4-mini\\llm\\StdbModule.csproj]\r\n",
- "phase": "build_or_publish"
+ "args": [],
+ "golden_db": "schema-t-018-constraints-golden",
+ "golden_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"",
+ "llm_db": "schema-t-018-constraints-o4-mini-llm",
+ "llm_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"",
+ "query": "SELECT Id, Email, Name FROM accounts WHERE Id=1",
+ "reducer": "Seed",
+ "server": "local"
}
}
},
"vendor": "openai",
- "started_at": "2025-10-19T23:14:48.113283300Z",
- "finished_at": "2025-10-19T23:15:51.073350Z"
+ "started_at": "2025-10-19T23:14:48.744809600Z",
+ "finished_at": "2025-10-19T23:21:58.431013200Z"
},
"t_019_many_to_many": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
@@ -1791,191 +1701,188 @@
"vendor": "openai",
"started_at": "2025-10-19T23:14:48.852351600Z",
"finished_at": "2025-10-19T23:16:06.979641200Z"
- }
- }
- },
- {
- "name": "GPT-4.1",
- "route_api_model": "gpt-4.1",
- "tasks": {
- "t_017_scheduled_columns": {
+ },
+ "t_020_ecs": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_017_scheduled_columns",
+ "task": "t_020_ecs",
"lang": "csharp",
"golden_published": true,
- "model_name": "GPT-4.1",
- "total_tests": 2,
- "passed_tests": 2,
+ "model_name": "o4-mini",
+ "total_tests": 5,
+ "passed_tests": 0,
"llm_output": null,
"category": "schema",
- "route_api_model": "gpt-4.1",
- "golden_db": "schema-t-017-scheduled-columns-golden",
- "llm_db": "schema-t-017-scheduled-columns-gpt-4-1-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_017_scheduled_columns\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_017_scheduled_columns\\csharp\\server\\gpt-4-1\\llm",
+ "route_api_model": "o4-mini",
+ "golden_db": "schema-t-020-ecs-golden",
+ "llm_db": "schema-t-020-ecs-o4-mini-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\o4-mini\\llm",
"scorer_details": {
- "scheduled_seeded_one_row": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM tick_timer WHERE ScheduledId>=0"
- }
- },
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
+ "publish_error": {
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "golden_db": "schema-t-017-scheduled-columns-golden",
- "llm_db": "schema-t-017-scheduled-columns-gpt-4-1-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
+ "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\o4-mini\\llm\\Lib.cs(5,43): error CS1016: Named attribute argument expected [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\o4-mini\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\o4-mini\\llm\\Lib.cs(5,43): error CS0103: The name 'Public' does not exist in the current context [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\o4-mini\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\o4-mini\\llm\\Lib.cs(5,6): error CS1729: 'TableAttribute' does not contain a constructor that takes 1 arguments [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\o4-mini\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\o4-mini\\llm\\Lib.cs(11,44): error CS1016: Named attribute argument expected [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\o4-mini\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\o4-mini\\llm\\Lib.cs(11,44): error CS0103: The name 'Public' does not exist in the current context [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\o4-mini\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\o4-mini\\llm\\Lib.cs(11,6): error CS1729: 'TableAttribute' does not contain a constructor that takes 1 arguments [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\o4-mini\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\o4-mini\\llm\\Lib.cs(19,45): error CS1016: Named attribute argument expected [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\o4-mini\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\o4-mini\\llm\\Lib.cs(19,45): error CS0103: The name 'Public' does not exist in the current context [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\o4-mini\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\o4-mini\\llm\\Lib.cs(19,6): error CS1729: 'TableAttribute' does not contain a constructor that takes 1 arguments [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\o4-mini\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\o4-mini\\llm\\Lib.cs(27,49): error CS1016: Named attribute argument expected [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\o4-mini\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\o4-mini\\llm\\Lib.cs(27,49): error CS0103: The name 'Public' does not exist in the current context [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\o4-mini\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\o4-mini\\llm\\Lib.cs(27,6): error CS1729: 'TableAttribute' does not contain a constructor that takes 1 arguments [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\o4-mini\\llm\\StdbModule.csproj]\r\n",
+ "phase": "build_or_publish"
}
}
},
"vendor": "openai",
- "started_at": "2025-10-19T23:14:46.912901700Z",
- "finished_at": "2025-10-19T23:22:02.767038400Z"
+ "started_at": "2025-10-19T23:15:36.126968800Z",
+ "finished_at": "2025-10-19T23:16:46.141457Z"
},
- "t_015_product_type_columns": {
+ "t_021_multi_column_index": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_015_product_type_columns",
+ "task": "t_021_multi_column_index",
"lang": "csharp",
"golden_published": true,
- "model_name": "GPT-4.1",
- "total_tests": 3,
- "passed_tests": 3,
+ "model_name": "o4-mini",
+ "total_tests": 4,
+ "passed_tests": 4,
"llm_output": null,
"category": "schema",
- "route_api_model": "gpt-4.1",
- "golden_db": "schema-t-015-product-type-columns-golden",
- "llm_db": "schema-t-015-product-type-columns-gpt-4-1-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\gpt-4-1\\llm",
+ "route_api_model": "o4-mini",
+ "golden_db": "schema-t-021-multi-column-index-golden",
+ "llm_db": "schema-t-021-multi-column-index-o4-mini-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\o4-mini\\llm",
"scorer_details": {
- "product_type_columns_row_parity": {
+ "mcindex_lookup_u7_d1": {
"pass": true,
"partial": 1.0,
"notes": {
- "args": [],
- "golden_db": "schema-t-015-product-type-columns-golden",
- "golden_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)",
- "llm_db": "schema-t-015-product-type-columns-gpt-4-1-llm",
- "llm_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)",
- "query": "SELECT Id, Home, Work, Pos FROM profiles WHERE Id=1",
- "reducer": "Seed",
- "server": "local"
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM logs WHERE UserId=7 AND Day=1"
}
},
- "product_type_columns_row_count": {
+ "mcindex_lookup_u7_d2": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM profiles WHERE Id=1"
+ "sql": "SELECT COUNT(*) AS n FROM logs WHERE UserId=7 AND Day=2"
}
},
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-015-product-type-columns-golden",
- "llm_db": "schema-t-015-product-type-columns-gpt-4-1-llm",
+ "golden_db": "schema-t-021-multi-column-index-golden",
+ "llm_db": "schema-t-021-multi-column-index-o4-mini-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
"tables_diff": null,
"tables_equal": true
}
+ },
+ "mcindex_seed_count": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 3,
+ "expected": 3,
+ "sql": "SELECT COUNT(*) AS n FROM logs"
+ }
}
},
"vendor": "openai",
- "started_at": "2025-10-19T23:14:46.754300400Z",
- "finished_at": "2025-10-19T23:21:32.610201800Z"
- },
- "t_020_ecs": {
+ "started_at": "2025-10-19T23:15:49.473810800Z",
+ "finished_at": "2025-10-19T23:21:56.880671200Z"
+ }
+ }
+ },
+ {
+ "name": "GPT-4.1",
+ "route_api_model": "gpt-4.1",
+ "tasks": {
+ "t_000_empty_reducers": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_020_ecs",
+ "task": "t_000_empty_reducers",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-4.1",
- "total_tests": 5,
+ "total_tests": 1,
"passed_tests": 0,
- "llm_output": null,
- "category": "schema",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Reducer]\n public static void EmptyReducer_NoArgs()\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithInt(int count)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithString(string name)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithTwoArgs(int count, string name)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithThreeArgs(bool active, float ratio, string label)\n {\n }\n}",
+ "category": "basics",
"route_api_model": "gpt-4.1",
- "golden_db": "schema-t-020-ecs-golden",
- "llm_db": "schema-t-020-ecs-gpt-4-1-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\gpt-4-1\\llm",
+ "golden_db": "basics-t-000-empty-reducers-golden",
+ "llm_db": "basics-t-000-empty-reducers-gpt-4-1-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\gpt-4-1\\llm",
"scorer_details": {
"publish_error": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\gpt-4-1\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'entities' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\gpt-4-1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\gpt-4-1\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(113,24): warning CS8981: The type name 'positions' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\gpt-4-1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\gpt-4-1\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(156,24): warning CS8981: The type name 'velocities' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\gpt-4-1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\gpt-4-1\\llm\\Lib.cs(65,37): error CS1061: 'Module.Velocity?' does not contain a definition for 'VX' and no accessible extension method 'VX' accepting a first argument of type 'Module.Velocity?' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\gpt-4-1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\gpt-4-1\\llm\\Lib.cs(66,37): error CS1061: 'Module.Velocity?' does not contain a definition for 'VY' and no accessible extension method 'VY' accepting a first argument of type 'Module.Velocity?' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\gpt-4-1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\gpt-4-1\\llm\\Lib.cs(71,30): error CS1061: 'Module.NextPosition?' does not contain a definition for 'X' and no accessible extension method 'X' accepting a first argument of type 'Module.NextPosition?' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\gpt-4-1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\gpt-4-1\\llm\\Lib.cs(72,30): error CS1061: 'Module.NextPosition?' does not contain a definition for 'Y' and no accessible extension method 'Y' accepting a first argument of type 'Module.NextPosition?' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\gpt-4-1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\gpt-4-1\\llm\\Lib.cs(73,55): error CS1503: Argument 1: cannot convert from 'Module.NextPosition?' to 'Module.NextPosition' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\gpt-4-1\\llm\\StdbModule.csproj]\r\n",
+ "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\gpt-4-1\\llm\\Lib.cs(7,43): error STDB0008: Reducer method EmptyReducer_NoArgs does not have a ReducerContext parameter. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\gpt-4-1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\gpt-4-1\\llm\\Lib.cs(12,44): error STDB0008: Reducer method EmptyReducer_WithInt does not have a ReducerContext parameter. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\gpt-4-1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\gpt-4-1\\llm\\Lib.cs(17,47): error STDB0008: Reducer method EmptyReducer_WithString does not have a ReducerContext parameter. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\gpt-4-1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\gpt-4-1\\llm\\Lib.cs(22,48): error STDB0008: Reducer method EmptyReducer_WithTwoArgs does not have a ReducerContext parameter. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\gpt-4-1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\gpt-4-1\\llm\\Lib.cs(27,50): error STDB0008: Reducer method EmptyReducer_WithThreeArgs does not have a ReducerContext parameter. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\gpt-4-1\\llm\\StdbModule.csproj]\r\n",
"phase": "build_or_publish"
}
}
},
"vendor": "openai",
- "started_at": "2025-10-19T23:15:38.719205400Z",
- "finished_at": "2025-10-19T23:16:26.679038600Z"
+ "started_at": "2025-10-21T23:58:20.228678700Z",
+ "finished_at": "2025-10-21T23:58:31.045062900Z"
},
- "t_012_spacetime_product_type": {
+ "t_001_basic_tables": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_012_spacetime_product_type",
+ "task": "t_001_basic_tables",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-4.1",
- "total_tests": 3,
- "passed_tests": 3,
+ "total_tests": 1,
+ "passed_tests": 1,
"llm_output": null,
- "category": "schema",
+ "category": "basics",
"route_api_model": "gpt-4.1",
- "golden_db": "schema-t-012-spacetime-product-type-golden",
- "llm_db": "schema-t-012-spacetime-product-type-gpt-4-1-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\gpt-4-1\\llm",
+ "golden_db": "basics-t-001-basic-tables-golden",
+ "llm_db": "basics-t-001-basic-tables-gpt-4-1-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\gpt-4-1\\llm",
"scorer_details": {
- "product_type_row_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "args": [
- 1,
- 2,
- 3
- ],
- "golden_db": "schema-t-012-spacetime-product-type-golden",
- "golden_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)",
- "llm_db": "schema-t-012-spacetime-product-type-gpt-4-1-llm",
- "llm_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)",
- "query": "SELECT Id, Value FROM results WHERE Id=1",
- "reducer": "SetScore",
- "server": "local"
- }
- },
- "product_type_row_count": {
+ "schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1"
+ "golden_db": "basics-t-001-basic-tables-golden",
+ "llm_db": "basics-t-001-basic-tables-gpt-4-1-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
}
- },
+ }
+ },
+ "vendor": "openai",
+ "started_at": "2025-10-19T23:14:45.625566500Z",
+ "finished_at": "2025-10-19T23:22:02.903292300Z"
+ },
+ "t_002_scheduled_table": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_002_scheduled_table",
+ "lang": "csharp",
+ "golden_published": true,
+ "model_name": "GPT-4.1",
+ "total_tests": 1,
+ "passed_tests": 1,
+ "llm_output": null,
+ "category": "basics",
+ "route_api_model": "gpt-4.1",
+ "golden_db": "basics-t-002-scheduled-table-golden",
+ "llm_db": "basics-t-002-scheduled-table-gpt-4-1-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_002_scheduled_table\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_002_scheduled_table\\csharp\\server\\gpt-4-1\\llm",
+ "scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-012-spacetime-product-type-golden",
- "llm_db": "schema-t-012-spacetime-product-type-gpt-4-1-llm",
+ "golden_db": "basics-t-002-scheduled-table-golden",
+ "llm_db": "basics-t-002-scheduled-table-gpt-4-1-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -1985,8 +1892,8 @@
}
},
"vendor": "openai",
- "started_at": "2025-10-19T23:14:46.506688400Z",
- "finished_at": "2025-10-19T23:21:33.499319900Z"
+ "started_at": "2025-10-19T23:14:45.702356900Z",
+ "finished_at": "2025-10-19T23:21:28.649313300Z"
},
"t_003_struct_in_table": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
@@ -2022,75 +1929,81 @@
"started_at": "2025-10-19T23:14:45.781952400Z",
"finished_at": "2025-10-19T23:20:36.408580600Z"
},
- "t_000_empty_reducers": {
+ "t_004_insert": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_000_empty_reducers",
+ "task": "t_004_insert",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-4.1",
- "total_tests": 1,
- "passed_tests": 0,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Reducer]\n public static void EmptyReducer_NoArgs()\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithInt(int count)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithString(string name)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithTwoArgs(int count, string name)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithThreeArgs(bool active, float ratio, string label)\n {\n }\n}",
+ "total_tests": 2,
+ "passed_tests": 2,
+ "llm_output": null,
"category": "basics",
"route_api_model": "gpt-4.1",
- "golden_db": "basics-t-000-empty-reducers-golden",
- "llm_db": "basics-t-000-empty-reducers-gpt-4-1-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\gpt-4-1\\llm",
+ "golden_db": "basics-t-004-insert-golden",
+ "llm_db": "basics-t-004-insert-gpt-4-1-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_004_insert\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_004_insert\\csharp\\server\\gpt-4-1\\llm",
"scorer_details": {
- "publish_error": {
- "pass": false,
- "partial": 0.0,
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
"notes": {
- "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\gpt-4-1\\llm\\Lib.cs(7,43): error STDB0008: Reducer method EmptyReducer_NoArgs does not have a ReducerContext parameter. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\gpt-4-1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\gpt-4-1\\llm\\Lib.cs(12,44): error STDB0008: Reducer method EmptyReducer_WithInt does not have a ReducerContext parameter. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\gpt-4-1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\gpt-4-1\\llm\\Lib.cs(17,47): error STDB0008: Reducer method EmptyReducer_WithString does not have a ReducerContext parameter. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\gpt-4-1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\gpt-4-1\\llm\\Lib.cs(22,48): error STDB0008: Reducer method EmptyReducer_WithTwoArgs does not have a ReducerContext parameter. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\gpt-4-1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\gpt-4-1\\llm\\Lib.cs(27,50): error STDB0008: Reducer method EmptyReducer_WithThreeArgs does not have a ReducerContext parameter. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\gpt-4-1\\llm\\StdbModule.csproj]\r\n",
- "phase": "build_or_publish"
+ "golden_db": "basics-t-004-insert-golden",
+ "llm_db": "basics-t-004-insert-gpt-4-1-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
+ }
+ },
+ "data_parity_insert_user": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "args": [
+ 1,
+ "Alice",
+ 30,
+ true
+ ],
+ "golden_db": "basics-t-004-insert-golden",
+ "golden_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true",
+ "llm_db": "basics-t-004-insert-gpt-4-1-llm",
+ "llm_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true",
+ "query": "SELECT Id, Name, Age, Active FROM users WHERE Id=1",
+ "reducer": "InsertUser",
+ "server": "local"
}
}
},
"vendor": "openai",
- "started_at": "2025-10-21T23:58:20.228678700Z",
- "finished_at": "2025-10-21T23:58:31.045062900Z"
+ "started_at": "2025-10-19T23:14:45.864828Z",
+ "finished_at": "2025-10-19T23:21:34.169691700Z"
},
- "t_021_multi_column_index": {
+ "t_005_update": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_021_multi_column_index",
+ "task": "t_005_update",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-4.1",
- "total_tests": 4,
- "passed_tests": 4,
+ "total_tests": 3,
+ "passed_tests": 3,
"llm_output": null,
- "category": "schema",
+ "category": "basics",
"route_api_model": "gpt-4.1",
- "golden_db": "schema-t-021-multi-column-index-golden",
- "llm_db": "schema-t-021-multi-column-index-gpt-4-1-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\gpt-4-1\\llm",
+ "golden_db": "basics-t-005-update-golden",
+ "llm_db": "basics-t-005-update-gpt-4-1-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_005_update\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_005_update\\csharp\\server\\gpt-4-1\\llm",
"scorer_details": {
- "mcindex_lookup_u7_d1": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM logs WHERE UserId=7 AND Day=1"
- }
- },
- "mcindex_seed_count": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 3,
- "expected": 3,
- "sql": "SELECT COUNT(*) AS n FROM logs"
- }
- },
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-021-multi-column-index-golden",
- "llm_db": "schema-t-021-multi-column-index-gpt-4-1-llm",
+ "golden_db": "basics-t-005-update-golden",
+ "llm_db": "basics-t-005-update-gpt-4-1-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -2098,51 +2011,68 @@
"tables_equal": true
}
},
- "mcindex_lookup_u7_d2": {
+ "data_parity_update_user": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM logs WHERE UserId=7 AND Day=2"
+ "args": [
+ 1,
+ "Alice2",
+ 31,
+ false
+ ],
+ "golden_db": "basics-t-005-update-golden",
+ "golden_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
+ "llm_db": "basics-t-005-update-gpt-4-1-llm",
+ "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
+ "query": "SELECT Id, Name, Age, Active FROM users WHERE Id=1",
+ "reducer": "UpdateUser",
+ "server": "local"
+ }
+ },
+ "seed_users_row": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
}
}
},
"vendor": "openai",
- "started_at": "2025-10-19T23:16:26.984790600Z",
- "finished_at": "2025-10-19T23:22:02.575406100Z"
+ "started_at": "2025-10-19T23:14:45.947481300Z",
+ "finished_at": "2025-10-19T23:20:56.538872800Z"
},
- "t_019_many_to_many": {
+ "t_006_delete": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_019_many_to_many",
+ "task": "t_006_delete",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-4.1",
- "total_tests": 5,
- "passed_tests": 5,
+ "total_tests": 3,
+ "passed_tests": 3,
"llm_output": null,
- "category": "schema",
+ "category": "basics",
"route_api_model": "gpt-4.1",
- "golden_db": "schema-t-019-many-to-many-golden",
- "llm_db": "schema-t-019-many-to-many-gpt-4-1-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\gpt-4-1\\llm",
+ "golden_db": "basics-t-006-delete-golden",
+ "llm_db": "basics-t-006-delete-gpt-4-1-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\gpt-4-1\\llm",
"scorer_details": {
- "m2m_has_1_10": {
+ "delete_user_count_zero": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM memberships WHERE UserId=1 AND GroupId=10"
+ "actual": 0,
+ "expected": 0,
+ "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=1"
}
},
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-019-many-to-many-golden",
- "llm_db": "schema-t-019-many-to-many-gpt-4-1-llm",
+ "golden_db": "basics-t-006-delete-golden",
+ "llm_db": "basics-t-006-delete-gpt-4-1-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -2150,74 +2080,49 @@
"tables_equal": true
}
},
- "m2m_has_2_20": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM memberships WHERE UserId=2 AND GroupId=20"
- }
- },
- "m2m_has_1_20": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM memberships WHERE UserId=1 AND GroupId=20"
- }
- },
- "memberships_three_rows": {
+ "seed_users_row": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 3,
- "expected": 3,
- "sql": "SELECT COUNT(*) AS n FROM memberships"
+ "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
}
}
},
"vendor": "openai",
- "started_at": "2025-10-19T23:14:47.077016900Z",
- "finished_at": "2025-10-19T23:21:23.280888800Z"
+ "started_at": "2025-10-19T23:14:46.030891300Z",
+ "finished_at": "2025-10-19T23:19:58.029303400Z"
},
- "t_014_elementary_columns": {
+ "t_007_crud": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_014_elementary_columns",
+ "task": "t_007_crud",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-4.1",
- "total_tests": 3,
- "passed_tests": 3,
+ "total_tests": 4,
+ "passed_tests": 4,
"llm_output": null,
- "category": "schema",
+ "category": "basics",
"route_api_model": "gpt-4.1",
- "golden_db": "schema-t-014-elementary-columns-golden",
- "llm_db": "schema-t-014-elementary-columns-gpt-4-1-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_014_elementary_columns\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_014_elementary_columns\\csharp\\server\\gpt-4-1\\llm",
+ "golden_db": "basics-t-007-crud-golden",
+ "llm_db": "basics-t-007-crud-gpt-4-1-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\gpt-4-1\\llm",
"scorer_details": {
- "elementary_columns_row_parity": {
+ "crud_total_count_one": {
"pass": true,
"partial": 1.0,
"notes": {
- "args": [],
- "golden_db": "schema-t-014-elementary-columns-golden",
- "golden_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"",
- "llm_db": "schema-t-014-elementary-columns-gpt-4-1-llm",
- "llm_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"",
- "query": "SELECT Id, Count, Total, Price, Ratio, Active, Name FROM primitives WHERE Id=1",
- "reducer": "Seed",
- "server": "local"
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM users"
}
},
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-014-elementary-columns-golden",
- "llm_db": "schema-t-014-elementary-columns-gpt-4-1-llm",
+ "golden_db": "basics-t-007-crud-golden",
+ "llm_db": "basics-t-007-crud-gpt-4-1-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -2225,19 +2130,62 @@
"tables_equal": true
}
},
- "elementary_columns_row_count": {
+ "crud_row_id1_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM primitives WHERE Id=1"
+ "args": [],
+ "golden_db": "basics-t-007-crud-golden",
+ "golden_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
+ "llm_db": "basics-t-007-crud-gpt-4-1-llm",
+ "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
+ "query": "SELECT Id, Name, Age, Active FROM users WHERE Id=1",
+ "reducer": "Crud",
+ "server": "local"
+ }
+ },
+ "crud_row_id2_deleted": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 0,
+ "expected": 0,
+ "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=2"
}
}
},
"vendor": "openai",
- "started_at": "2025-10-19T23:14:46.672484500Z",
- "finished_at": "2025-10-19T23:20:32.512254Z"
+ "started_at": "2025-10-19T23:14:46.112434Z",
+ "finished_at": "2025-10-19T23:21:35.282933Z"
+ },
+ "t_008_index_lookup": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_008_index_lookup",
+ "lang": "csharp",
+ "golden_published": true,
+ "model_name": "GPT-4.1",
+ "total_tests": 3,
+ "passed_tests": 0,
+ "llm_output": null,
+ "category": "basics",
+ "route_api_model": "gpt-4.1",
+ "golden_db": "basics-t-008-index-lookup-golden",
+ "llm_db": "basics-t-008-index-lookup-gpt-4-1-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gpt-4-1\\llm",
+ "scorer_details": {
+ "publish_error": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gpt-4-1\\llm\\Lib.cs(7,27): warning CS8981: The type name 'users' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gpt-4-1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gpt-4-1\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\Module.users.cs(5,16): warning CS8981: The type name 'users' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gpt-4-1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gpt-4-1\\llm\\Lib.cs(17,27): warning CS8981: The type name 'results' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gpt-4-1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gpt-4-1\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\Module.results.cs(5,16): warning CS8981: The type name 'results' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gpt-4-1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gpt-4-1\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'results' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gpt-4-1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gpt-4-1\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(70,24): warning CS8981: The type name 'users' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gpt-4-1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gpt-4-1\\llm\\Lib.cs(30,59): error CS1061: 'Module.users?' does not contain a definition for 'Id' and no accessible extension method 'Id' accepting a first argument of type 'Module.users?' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gpt-4-1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gpt-4-1\\llm\\Lib.cs(30,75): error CS1061: 'Module.users?' does not contain a definition for 'Name' and no accessible extension method 'Name' accepting a first argument of type 'Module.users?' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gpt-4-1\\llm\\StdbModule.csproj]\r\n",
+ "phase": "build_or_publish"
+ }
+ }
+ },
+ "vendor": "openai",
+ "started_at": "2025-10-19T23:14:46.194138800Z",
+ "finished_at": "2025-10-19T23:15:38.531073800Z"
},
"t_009_init": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
@@ -2255,26 +2203,13 @@
"work_dir_golden": "target\\llm-runs\\basics\\t_009_init\\csharp\\server\\golden",
"work_dir_llm": "target\\llm-runs\\basics\\t_009_init\\csharp\\server\\gpt-4-1\\llm",
"scorer_details": {
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "golden_db": "basics-t-009-init-golden",
- "llm_db": "basics-t-009-init-gpt-4-1-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
- }
- },
- "init_seed_bob": {
+ "init_seed_alice": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false"
+ "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true"
}
},
"init_total_two": {
@@ -2286,13 +2221,26 @@
"sql": "SELECT COUNT(*) AS n FROM users"
}
},
- "init_seed_alice": {
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "golden_db": "basics-t-009-init-golden",
+ "llm_db": "basics-t-009-init-gpt-4-1-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
+ }
+ },
+ "init_seed_bob": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true"
+ "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false"
}
}
},
@@ -2300,9 +2248,9 @@
"started_at": "2025-10-19T23:14:46.275224700Z",
"finished_at": "2025-10-19T23:22:03.188112800Z"
},
- "t_002_scheduled_table": {
+ "t_010_connect": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_002_scheduled_table",
+ "task": "t_010_connect",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-4.1",
@@ -2311,17 +2259,17 @@
"llm_output": null,
"category": "basics",
"route_api_model": "gpt-4.1",
- "golden_db": "basics-t-002-scheduled-table-golden",
- "llm_db": "basics-t-002-scheduled-table-gpt-4-1-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_002_scheduled_table\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_002_scheduled_table\\csharp\\server\\gpt-4-1\\llm",
+ "golden_db": "basics-t-010-connect-golden",
+ "llm_db": "basics-t-010-connect-gpt-4-1-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_010_connect\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_010_connect\\csharp\\server\\gpt-4-1\\llm",
"scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-002-scheduled-table-golden",
- "llm_db": "basics-t-002-scheduled-table-gpt-4-1-llm",
+ "golden_db": "basics-t-010-connect-golden",
+ "llm_db": "basics-t-010-connect-gpt-4-1-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -2331,12 +2279,12 @@
}
},
"vendor": "openai",
- "started_at": "2025-10-19T23:14:45.702356900Z",
- "finished_at": "2025-10-19T23:21:28.649313300Z"
+ "started_at": "2025-10-19T23:14:46.353401200Z",
+ "finished_at": "2025-10-19T23:19:06.467659300Z"
},
- "t_005_update": {
+ "t_011_helper_function": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_005_update",
+ "task": "t_011_helper_function",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-4.1",
@@ -2345,202 +2293,290 @@
"llm_output": null,
"category": "basics",
"route_api_model": "gpt-4.1",
- "golden_db": "basics-t-005-update-golden",
- "llm_db": "basics-t-005-update-gpt-4-1-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_005_update\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_005_update\\csharp\\server\\gpt-4-1\\llm",
+ "golden_db": "basics-t-011-helper-function-golden",
+ "llm_db": "basics-t-011-helper-function-gpt-4-1-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\gpt-4-1\\llm",
"scorer_details": {
- "seed_users_row": {
+ "schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
+ "golden_db": "basics-t-011-helper-function-golden",
+ "llm_db": "basics-t-011-helper-function-gpt-4-1-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
}
},
- "data_parity_update_user": {
+ "helper_func_sum_parity": {
"pass": true,
"partial": 1.0,
"notes": {
"args": [
1,
- "Alice2",
- 31,
- false
+ 2,
+ 3
],
- "golden_db": "basics-t-005-update-golden",
- "golden_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
- "llm_db": "basics-t-005-update-gpt-4-1-llm",
- "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
- "query": "SELECT Id, Name, Age, Active FROM users WHERE Id=1",
- "reducer": "UpdateUser",
+ "golden_db": "basics-t-011-helper-function-golden",
+ "golden_out": "Id | Sum ----+----- 1 | 5",
+ "llm_db": "basics-t-011-helper-function-gpt-4-1-llm",
+ "llm_out": "Id | Sum ----+----- 1 | 5",
+ "query": "SELECT Id, Sum FROM results WHERE Id=1",
+ "reducer": "ComputeSum",
"server": "local"
}
},
- "schema_parity": {
+ "helper_func_sum_abs": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-005-update-golden",
- "llm_db": "basics-t-005-update-gpt-4-1-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1 AND Sum=5"
}
}
},
"vendor": "openai",
- "started_at": "2025-10-19T23:14:45.947481300Z",
- "finished_at": "2025-10-19T23:20:56.538872800Z"
+ "started_at": "2025-10-19T23:14:46.430200300Z",
+ "finished_at": "2025-10-19T23:21:07.326153600Z"
},
- "t_004_insert": {
+ "t_012_spacetime_product_type": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_004_insert",
+ "task": "t_012_spacetime_product_type",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-4.1",
- "total_tests": 2,
- "passed_tests": 2,
+ "total_tests": 3,
+ "passed_tests": 3,
"llm_output": null,
- "category": "basics",
+ "category": "schema",
"route_api_model": "gpt-4.1",
- "golden_db": "basics-t-004-insert-golden",
- "llm_db": "basics-t-004-insert-gpt-4-1-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_004_insert\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_004_insert\\csharp\\server\\gpt-4-1\\llm",
+ "golden_db": "schema-t-012-spacetime-product-type-golden",
+ "llm_db": "schema-t-012-spacetime-product-type-gpt-4-1-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\gpt-4-1\\llm",
"scorer_details": {
- "data_parity_insert_user": {
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "golden_db": "schema-t-012-spacetime-product-type-golden",
+ "llm_db": "schema-t-012-spacetime-product-type-gpt-4-1-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
+ }
+ },
+ "product_type_row_count": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1"
+ }
+ },
+ "product_type_row_parity": {
"pass": true,
"partial": 1.0,
"notes": {
"args": [
1,
- "Alice",
- 30,
- true
+ 2,
+ 3
],
- "golden_db": "basics-t-004-insert-golden",
- "golden_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true",
- "llm_db": "basics-t-004-insert-gpt-4-1-llm",
- "llm_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true",
- "query": "SELECT Id, Name, Age, Active FROM users WHERE Id=1",
- "reducer": "InsertUser",
+ "golden_db": "schema-t-012-spacetime-product-type-golden",
+ "golden_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)",
+ "llm_db": "schema-t-012-spacetime-product-type-gpt-4-1-llm",
+ "llm_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)",
+ "query": "SELECT Id, Value FROM results WHERE Id=1",
+ "reducer": "SetScore",
"server": "local"
}
- },
+ }
+ },
+ "vendor": "openai",
+ "started_at": "2025-10-19T23:14:46.506688400Z",
+ "finished_at": "2025-10-19T23:21:33.499319900Z"
+ },
+ "t_013_spacetime_sum_type": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_013_spacetime_sum_type",
+ "lang": "csharp",
+ "golden_published": true,
+ "model_name": "GPT-4.1",
+ "total_tests": 3,
+ "passed_tests": 3,
+ "llm_output": null,
+ "category": "schema",
+ "route_api_model": "gpt-4.1",
+ "golden_db": "schema-t-013-spacetime-sum-type-golden",
+ "llm_db": "schema-t-013-spacetime-sum-type-gpt-4-1-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\gpt-4-1\\llm",
+ "scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-004-insert-golden",
- "llm_db": "basics-t-004-insert-gpt-4-1-llm",
+ "golden_db": "schema-t-013-spacetime-sum-type-golden",
+ "llm_db": "schema-t-013-spacetime-sum-type-gpt-4-1-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
"tables_diff": null,
"tables_equal": true
}
+ },
+ "sum_type_row_count": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1"
+ }
+ },
+ "sum_type_row_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "args": [
+ 1,
+ 10
+ ],
+ "golden_db": "schema-t-013-spacetime-sum-type-golden",
+ "golden_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))",
+ "llm_db": "schema-t-013-spacetime-sum-type-gpt-4-1-llm",
+ "llm_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))",
+ "query": "SELECT Id, Value FROM results WHERE Id=1",
+ "reducer": "SetCircle",
+ "server": "local"
+ }
}
},
"vendor": "openai",
- "started_at": "2025-10-19T23:14:45.864828Z",
- "finished_at": "2025-10-19T23:21:34.169691700Z"
+ "started_at": "2025-10-19T23:14:46.591403300Z",
+ "finished_at": "2025-10-19T23:21:26.868961500Z"
},
- "t_007_crud": {
+ "t_014_elementary_columns": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_007_crud",
+ "task": "t_014_elementary_columns",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-4.1",
- "total_tests": 4,
- "passed_tests": 4,
+ "total_tests": 3,
+ "passed_tests": 3,
"llm_output": null,
- "category": "basics",
+ "category": "schema",
"route_api_model": "gpt-4.1",
- "golden_db": "basics-t-007-crud-golden",
- "llm_db": "basics-t-007-crud-gpt-4-1-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\gpt-4-1\\llm",
+ "golden_db": "schema-t-014-elementary-columns-golden",
+ "llm_db": "schema-t-014-elementary-columns-gpt-4-1-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_014_elementary_columns\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_014_elementary_columns\\csharp\\server\\gpt-4-1\\llm",
"scorer_details": {
- "crud_total_count_one": {
+ "elementary_columns_row_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "args": [],
+ "golden_db": "schema-t-014-elementary-columns-golden",
+ "golden_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"",
+ "llm_db": "schema-t-014-elementary-columns-gpt-4-1-llm",
+ "llm_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"",
+ "query": "SELECT Id, Count, Total, Price, Ratio, Active, Name FROM primitives WHERE Id=1",
+ "reducer": "Seed",
+ "server": "local"
+ }
+ },
+ "elementary_columns_row_count": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM users"
+ "sql": "SELECT COUNT(*) AS n FROM primitives WHERE Id=1"
}
},
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-007-crud-golden",
- "llm_db": "basics-t-007-crud-gpt-4-1-llm",
+ "golden_db": "schema-t-014-elementary-columns-golden",
+ "llm_db": "schema-t-014-elementary-columns-gpt-4-1-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
"tables_diff": null,
"tables_equal": true
}
- },
- "crud_row_id2_deleted": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 0,
- "expected": 0,
- "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=2"
- }
- },
- "crud_row_id1_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "args": [],
- "golden_db": "basics-t-007-crud-golden",
- "golden_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
- "llm_db": "basics-t-007-crud-gpt-4-1-llm",
- "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
- "query": "SELECT Id, Name, Age, Active FROM users WHERE Id=1",
- "reducer": "Crud",
- "server": "local"
- }
}
},
"vendor": "openai",
- "started_at": "2025-10-19T23:14:46.112434Z",
- "finished_at": "2025-10-19T23:21:35.282933Z"
+ "started_at": "2025-10-19T23:14:46.672484500Z",
+ "finished_at": "2025-10-19T23:20:32.512254Z"
},
- "t_008_index_lookup": {
+ "t_015_product_type_columns": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_008_index_lookup",
+ "task": "t_015_product_type_columns",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-4.1",
"total_tests": 3,
- "passed_tests": 0,
+ "passed_tests": 3,
"llm_output": null,
- "category": "basics",
+ "category": "schema",
"route_api_model": "gpt-4.1",
- "golden_db": "basics-t-008-index-lookup-golden",
- "llm_db": "basics-t-008-index-lookup-gpt-4-1-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gpt-4-1\\llm",
+ "golden_db": "schema-t-015-product-type-columns-golden",
+ "llm_db": "schema-t-015-product-type-columns-gpt-4-1-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\gpt-4-1\\llm",
"scorer_details": {
- "publish_error": {
- "pass": false,
- "partial": 0.0,
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
"notes": {
- "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gpt-4-1\\llm\\Lib.cs(7,27): warning CS8981: The type name 'users' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gpt-4-1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gpt-4-1\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\Module.users.cs(5,16): warning CS8981: The type name 'users' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gpt-4-1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gpt-4-1\\llm\\Lib.cs(17,27): warning CS8981: The type name 'results' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gpt-4-1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gpt-4-1\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\Module.results.cs(5,16): warning CS8981: The type name 'results' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gpt-4-1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gpt-4-1\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'results' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gpt-4-1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gpt-4-1\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(70,24): warning CS8981: The type name 'users' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gpt-4-1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gpt-4-1\\llm\\Lib.cs(30,59): error CS1061: 'Module.users?' does not contain a definition for 'Id' and no accessible extension method 'Id' accepting a first argument of type 'Module.users?' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gpt-4-1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gpt-4-1\\llm\\Lib.cs(30,75): error CS1061: 'Module.users?' does not contain a definition for 'Name' and no accessible extension method 'Name' accepting a first argument of type 'Module.users?' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gpt-4-1\\llm\\StdbModule.csproj]\r\n",
- "phase": "build_or_publish"
+ "golden_db": "schema-t-015-product-type-columns-golden",
+ "llm_db": "schema-t-015-product-type-columns-gpt-4-1-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
+ }
+ },
+ "product_type_columns_row_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "args": [],
+ "golden_db": "schema-t-015-product-type-columns-golden",
+ "golden_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)",
+ "llm_db": "schema-t-015-product-type-columns-gpt-4-1-llm",
+ "llm_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)",
+ "query": "SELECT Id, Home, Work, Pos FROM profiles WHERE Id=1",
+ "reducer": "Seed",
+ "server": "local"
+ }
+ },
+ "product_type_columns_row_count": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM profiles WHERE Id=1"
}
}
},
"vendor": "openai",
- "started_at": "2025-10-19T23:14:46.194138800Z",
- "finished_at": "2025-10-19T23:15:38.531073800Z"
+ "started_at": "2025-10-19T23:14:46.754300400Z",
+ "finished_at": "2025-10-19T23:21:32.610201800Z"
},
"t_016_sum_type_columns": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
@@ -2599,51 +2635,37 @@
"started_at": "2025-10-19T23:14:46.832512600Z",
"finished_at": "2025-10-19T23:21:36.178957600Z"
},
- "t_018_constraints": {
+ "t_017_scheduled_columns": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_018_constraints",
+ "task": "t_017_scheduled_columns",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-4.1",
- "total_tests": 3,
- "passed_tests": 3,
+ "total_tests": 2,
+ "passed_tests": 2,
"llm_output": null,
"category": "schema",
"route_api_model": "gpt-4.1",
- "golden_db": "schema-t-018-constraints-golden",
- "llm_db": "schema-t-018-constraints-gpt-4-1-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\gpt-4-1\\llm",
+ "golden_db": "schema-t-017-scheduled-columns-golden",
+ "llm_db": "schema-t-017-scheduled-columns-gpt-4-1-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_017_scheduled_columns\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_017_scheduled_columns\\csharp\\server\\gpt-4-1\\llm",
"scorer_details": {
- "constraints_seed_two_rows": {
+ "scheduled_seeded_one_row": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM accounts WHERE Id=2"
- }
- },
- "constraints_row_parity_after_seed": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "args": [],
- "golden_db": "schema-t-018-constraints-golden",
- "golden_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"",
- "llm_db": "schema-t-018-constraints-gpt-4-1-llm",
- "llm_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"",
- "query": "SELECT Id, Email, Name FROM accounts WHERE Id=1",
- "reducer": "Seed",
- "server": "local"
+ "sql": "SELECT COUNT(*) AS n FROM tick_timer WHERE ScheduledId>=0"
}
},
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-018-constraints-golden",
- "llm_db": "schema-t-018-constraints-gpt-4-1-llm",
+ "golden_db": "schema-t-017-scheduled-columns-golden",
+ "llm_db": "schema-t-017-scheduled-columns-gpt-4-1-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -2653,12 +2675,12 @@
}
},
"vendor": "openai",
- "started_at": "2025-10-19T23:14:46.995375100Z",
- "finished_at": "2025-10-19T23:22:03.642106100Z"
+ "started_at": "2025-10-19T23:14:46.912901700Z",
+ "finished_at": "2025-10-19T23:22:02.767038400Z"
},
- "t_013_spacetime_sum_type": {
+ "t_018_constraints": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_013_spacetime_sum_type",
+ "task": "t_018_constraints",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-4.1",
@@ -2667,34 +2689,31 @@
"llm_output": null,
"category": "schema",
"route_api_model": "gpt-4.1",
- "golden_db": "schema-t-013-spacetime-sum-type-golden",
- "llm_db": "schema-t-013-spacetime-sum-type-gpt-4-1-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\gpt-4-1\\llm",
+ "golden_db": "schema-t-018-constraints-golden",
+ "llm_db": "schema-t-018-constraints-gpt-4-1-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\gpt-4-1\\llm",
"scorer_details": {
- "sum_type_row_count": {
+ "constraints_seed_two_rows": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1"
+ "sql": "SELECT COUNT(*) AS n FROM accounts WHERE Id=2"
}
},
- "sum_type_row_parity": {
+ "constraints_row_parity_after_seed": {
"pass": true,
"partial": 1.0,
"notes": {
- "args": [
- 1,
- 10
- ],
- "golden_db": "schema-t-013-spacetime-sum-type-golden",
- "golden_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))",
- "llm_db": "schema-t-013-spacetime-sum-type-gpt-4-1-llm",
- "llm_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))",
- "query": "SELECT Id, Value FROM results WHERE Id=1",
- "reducer": "SetCircle",
+ "args": [],
+ "golden_db": "schema-t-018-constraints-golden",
+ "golden_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"",
+ "llm_db": "schema-t-018-constraints-gpt-4-1-llm",
+ "llm_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"",
+ "query": "SELECT Id, Email, Name FROM accounts WHERE Id=1",
+ "reducer": "Seed",
"server": "local"
}
},
@@ -2702,8 +2721,8 @@
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-013-spacetime-sum-type-golden",
- "llm_db": "schema-t-013-spacetime-sum-type-gpt-4-1-llm",
+ "golden_db": "schema-t-018-constraints-golden",
+ "llm_db": "schema-t-018-constraints-gpt-4-1-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -2713,31 +2732,67 @@
}
},
"vendor": "openai",
- "started_at": "2025-10-19T23:14:46.591403300Z",
- "finished_at": "2025-10-19T23:21:26.868961500Z"
+ "started_at": "2025-10-19T23:14:46.995375100Z",
+ "finished_at": "2025-10-19T23:22:03.642106100Z"
},
- "t_001_basic_tables": {
+ "t_019_many_to_many": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_001_basic_tables",
+ "task": "t_019_many_to_many",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-4.1",
- "total_tests": 1,
- "passed_tests": 1,
+ "total_tests": 5,
+ "passed_tests": 5,
"llm_output": null,
- "category": "basics",
+ "category": "schema",
"route_api_model": "gpt-4.1",
- "golden_db": "basics-t-001-basic-tables-golden",
- "llm_db": "basics-t-001-basic-tables-gpt-4-1-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\gpt-4-1\\llm",
+ "golden_db": "schema-t-019-many-to-many-golden",
+ "llm_db": "schema-t-019-many-to-many-gpt-4-1-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\gpt-4-1\\llm",
"scorer_details": {
+ "memberships_three_rows": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 3,
+ "expected": 3,
+ "sql": "SELECT COUNT(*) AS n FROM memberships"
+ }
+ },
+ "m2m_has_1_10": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM memberships WHERE UserId=1 AND GroupId=10"
+ }
+ },
+ "m2m_has_2_20": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM memberships WHERE UserId=2 AND GroupId=20"
+ }
+ },
+ "m2m_has_1_20": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM memberships WHERE UserId=1 AND GroupId=20"
+ }
+ },
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-001-basic-tables-golden",
- "llm_db": "basics-t-001-basic-tables-gpt-4-1-llm",
+ "golden_db": "schema-t-019-many-to-many-golden",
+ "llm_db": "schema-t-019-many-to-many-gpt-4-1-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -2747,31 +2802,60 @@
}
},
"vendor": "openai",
- "started_at": "2025-10-19T23:14:45.625566500Z",
- "finished_at": "2025-10-19T23:22:02.903292300Z"
+ "started_at": "2025-10-19T23:14:47.077016900Z",
+ "finished_at": "2025-10-19T23:21:23.280888800Z"
},
- "t_011_helper_function": {
+ "t_020_ecs": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_011_helper_function",
+ "task": "t_020_ecs",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-4.1",
- "total_tests": 3,
- "passed_tests": 3,
+ "total_tests": 5,
+ "passed_tests": 0,
"llm_output": null,
- "category": "basics",
+ "category": "schema",
"route_api_model": "gpt-4.1",
- "golden_db": "basics-t-011-helper-function-golden",
- "llm_db": "basics-t-011-helper-function-gpt-4-1-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\gpt-4-1\\llm",
+ "golden_db": "schema-t-020-ecs-golden",
+ "llm_db": "schema-t-020-ecs-gpt-4-1-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\gpt-4-1\\llm",
+ "scorer_details": {
+ "publish_error": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\gpt-4-1\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'entities' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\gpt-4-1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\gpt-4-1\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(113,24): warning CS8981: The type name 'positions' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\gpt-4-1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\gpt-4-1\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(156,24): warning CS8981: The type name 'velocities' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\gpt-4-1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\gpt-4-1\\llm\\Lib.cs(65,37): error CS1061: 'Module.Velocity?' does not contain a definition for 'VX' and no accessible extension method 'VX' accepting a first argument of type 'Module.Velocity?' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\gpt-4-1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\gpt-4-1\\llm\\Lib.cs(66,37): error CS1061: 'Module.Velocity?' does not contain a definition for 'VY' and no accessible extension method 'VY' accepting a first argument of type 'Module.Velocity?' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\gpt-4-1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\gpt-4-1\\llm\\Lib.cs(71,30): error CS1061: 'Module.NextPosition?' does not contain a definition for 'X' and no accessible extension method 'X' accepting a first argument of type 'Module.NextPosition?' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\gpt-4-1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\gpt-4-1\\llm\\Lib.cs(72,30): error CS1061: 'Module.NextPosition?' does not contain a definition for 'Y' and no accessible extension method 'Y' accepting a first argument of type 'Module.NextPosition?' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\gpt-4-1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\gpt-4-1\\llm\\Lib.cs(73,55): error CS1503: Argument 1: cannot convert from 'Module.NextPosition?' to 'Module.NextPosition' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\gpt-4-1\\llm\\StdbModule.csproj]\r\n",
+ "phase": "build_or_publish"
+ }
+ }
+ },
+ "vendor": "openai",
+ "started_at": "2025-10-19T23:15:38.719205400Z",
+ "finished_at": "2025-10-19T23:16:26.679038600Z"
+ },
+ "t_021_multi_column_index": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_021_multi_column_index",
+ "lang": "csharp",
+ "golden_published": true,
+ "model_name": "GPT-4.1",
+ "total_tests": 4,
+ "passed_tests": 4,
+ "llm_output": null,
+ "category": "schema",
+ "route_api_model": "gpt-4.1",
+ "golden_db": "schema-t-021-multi-column-index-golden",
+ "llm_db": "schema-t-021-multi-column-index-gpt-4-1-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\gpt-4-1\\llm",
"scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-011-helper-function-golden",
- "llm_db": "basics-t-011-helper-function-gpt-4-1-llm",
+ "golden_db": "schema-t-021-multi-column-index-golden",
+ "llm_db": "schema-t-021-multi-column-index-gpt-4-1-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -2779,110 +2863,95 @@
"tables_equal": true
}
},
- "helper_func_sum_abs": {
+ "mcindex_seed_count": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 3,
+ "expected": 3,
+ "sql": "SELECT COUNT(*) AS n FROM logs"
+ }
+ },
+ "mcindex_lookup_u7_d2": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1 AND Sum=5"
+ "sql": "SELECT COUNT(*) AS n FROM logs WHERE UserId=7 AND Day=2"
}
},
- "helper_func_sum_parity": {
+ "mcindex_lookup_u7_d1": {
"pass": true,
"partial": 1.0,
"notes": {
- "args": [
- 1,
- 2,
- 3
- ],
- "golden_db": "basics-t-011-helper-function-golden",
- "golden_out": "Id | Sum ----+----- 1 | 5",
- "llm_db": "basics-t-011-helper-function-gpt-4-1-llm",
- "llm_out": "Id | Sum ----+----- 1 | 5",
- "query": "SELECT Id, Sum FROM results WHERE Id=1",
- "reducer": "ComputeSum",
- "server": "local"
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM logs WHERE UserId=7 AND Day=1"
}
}
},
"vendor": "openai",
- "started_at": "2025-10-19T23:14:46.430200300Z",
- "finished_at": "2025-10-19T23:21:07.326153600Z"
- },
- "t_006_delete": {
+ "started_at": "2025-10-19T23:16:26.984790600Z",
+ "finished_at": "2025-10-19T23:22:02.575406100Z"
+ }
+ }
+ },
+ {
+ "name": "GPT-4o",
+ "route_api_model": "gpt-4o",
+ "tasks": {
+ "t_000_empty_reducers": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_006_delete",
+ "task": "t_000_empty_reducers",
"lang": "csharp",
"golden_published": true,
- "model_name": "GPT-4.1",
- "total_tests": 3,
- "passed_tests": 3,
- "llm_output": null,
+ "model_name": "GPT-4o",
+ "total_tests": 1,
+ "passed_tests": 0,
+ "llm_output": "```csharp\npublic static partial class Module\n{\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_NoArgs(ReducerContext ctx)\n {\n }\n\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_WithInt(ReducerContext ctx, int count)\n {\n }\n\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_WithString(ReducerContext ctx, string name)\n {\n }\n\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_WithTwoArgs(ReducerContext ctx, int count, string name)\n {\n }\n\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_WithThreeArgs(ReducerContext ctx, bool active, float ratio, string label)\n {\n }\n}\n```",
"category": "basics",
- "route_api_model": "gpt-4.1",
- "golden_db": "basics-t-006-delete-golden",
- "llm_db": "basics-t-006-delete-gpt-4-1-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\gpt-4-1\\llm",
+ "route_api_model": "gpt-4o",
+ "golden_db": "basics-t-000-empty-reducers-golden",
+ "llm_db": "basics-t-000-empty-reducers-gpt-4o-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\gpt-4o\\llm",
"scorer_details": {
- "seed_users_row": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
- }
- },
- "delete_user_count_zero": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 0,
- "expected": 0,
- "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=1"
- }
- },
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
+ "publish_error": {
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "golden_db": "basics-t-006-delete-golden",
- "llm_db": "basics-t-006-delete-gpt-4-1-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
+ "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\gpt-4o\\llm\\Lib.cs(5,44): error CS0246: The type or namespace name 'ReducerContext' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\gpt-4o\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\gpt-4o\\llm\\Lib.cs(10,45): error CS0246: The type or namespace name 'ReducerContext' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\gpt-4o\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\gpt-4o\\llm\\Lib.cs(15,48): error CS0246: The type or namespace name 'ReducerContext' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\gpt-4o\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\gpt-4o\\llm\\Lib.cs(20,49): error CS0246: The type or namespace name 'ReducerContext' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\gpt-4o\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\gpt-4o\\llm\\Lib.cs(25,51): error CS0246: The type or namespace name 'ReducerContext' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\gpt-4o\\llm\\StdbModule.csproj]\r\n",
+ "phase": "build_or_publish"
}
}
},
"vendor": "openai",
- "started_at": "2025-10-19T23:14:46.030891300Z",
- "finished_at": "2025-10-19T23:19:58.029303400Z"
+ "started_at": "2025-10-21T23:58:20.810430600Z",
+ "finished_at": "2025-10-21T23:58:40.508615200Z"
},
- "t_010_connect": {
+ "t_001_basic_tables": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_010_connect",
+ "task": "t_001_basic_tables",
"lang": "csharp",
"golden_published": true,
- "model_name": "GPT-4.1",
+ "model_name": "GPT-4o",
"total_tests": 1,
"passed_tests": 1,
"llm_output": null,
"category": "basics",
- "route_api_model": "gpt-4.1",
- "golden_db": "basics-t-010-connect-golden",
- "llm_db": "basics-t-010-connect-gpt-4-1-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_010_connect\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_010_connect\\csharp\\server\\gpt-4-1\\llm",
+ "route_api_model": "gpt-4o",
+ "golden_db": "basics-t-001-basic-tables-golden",
+ "llm_db": "basics-t-001-basic-tables-gpt-4o-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\gpt-4o\\llm",
"scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-010-connect-golden",
- "llm_db": "basics-t-010-connect-gpt-4-1-llm",
+ "golden_db": "basics-t-001-basic-tables-golden",
+ "llm_db": "basics-t-001-basic-tables-gpt-4o-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -2892,43 +2961,42 @@
}
},
"vendor": "openai",
- "started_at": "2025-10-19T23:14:46.353401200Z",
- "finished_at": "2025-10-19T23:19:06.467659300Z"
- }
- }
- },
- {
- "name": "GPT-4o",
- "route_api_model": "gpt-4o",
- "tasks": {
- "t_008_index_lookup": {
+ "started_at": "2025-10-19T23:14:49.018655700Z",
+ "finished_at": "2025-10-19T23:21:36.975184900Z"
+ },
+ "t_002_scheduled_table": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_008_index_lookup",
+ "task": "t_002_scheduled_table",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-4o",
- "total_tests": 3,
- "passed_tests": 0,
+ "total_tests": 1,
+ "passed_tests": 1,
"llm_output": null,
"category": "basics",
"route_api_model": "gpt-4o",
- "golden_db": "basics-t-008-index-lookup-golden",
- "llm_db": "basics-t-008-index-lookup-gpt-4o-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gpt-4o\\llm",
+ "golden_db": "basics-t-002-scheduled-table-golden",
+ "llm_db": "basics-t-002-scheduled-table-gpt-4o-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_002_scheduled_table\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_002_scheduled_table\\csharp\\server\\gpt-4o\\llm",
"scorer_details": {
- "publish_error": {
- "pass": false,
- "partial": 0.0,
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
"notes": {
- "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gpt-4o\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'results' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gpt-4o\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gpt-4o\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(70,24): warning CS8981: The type name 'users' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gpt-4o\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gpt-4o\\llm\\Lib.cs(30,58): error CS1061: 'Module.User?' does not contain a definition for 'Id' and no accessible extension method 'Id' accepting a first argument of type 'Module.User?' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gpt-4o\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gpt-4o\\llm\\Lib.cs(30,74): error CS1061: 'Module.User?' does not contain a definition for 'Name' and no accessible extension method 'Name' accepting a first argument of type 'Module.User?' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gpt-4o\\llm\\StdbModule.csproj]\r\n",
- "phase": "build_or_publish"
+ "golden_db": "basics-t-002-scheduled-table-golden",
+ "llm_db": "basics-t-002-scheduled-table-gpt-4o-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
}
}
},
"vendor": "openai",
- "started_at": "2025-10-19T23:14:49.571594500Z",
- "finished_at": "2025-10-19T23:15:55.256976100Z"
+ "started_at": "2025-10-19T23:14:49.097275400Z",
+ "finished_at": "2025-10-19T23:22:07.069582700Z"
},
"t_003_struct_in_table": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
@@ -2964,6 +3032,59 @@
"started_at": "2025-10-19T23:14:49.177926300Z",
"finished_at": "2025-10-19T23:22:08.984849500Z"
},
+ "t_004_insert": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_004_insert",
+ "lang": "csharp",
+ "golden_published": true,
+ "model_name": "GPT-4o",
+ "total_tests": 2,
+ "passed_tests": 2,
+ "llm_output": null,
+ "category": "basics",
+ "route_api_model": "gpt-4o",
+ "golden_db": "basics-t-004-insert-golden",
+ "llm_db": "basics-t-004-insert-gpt-4o-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_004_insert\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_004_insert\\csharp\\server\\gpt-4o\\llm",
+ "scorer_details": {
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "golden_db": "basics-t-004-insert-golden",
+ "llm_db": "basics-t-004-insert-gpt-4o-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
+ }
+ },
+ "data_parity_insert_user": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "args": [
+ 1,
+ "Alice",
+ 30,
+ true
+ ],
+ "golden_db": "basics-t-004-insert-golden",
+ "golden_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true",
+ "llm_db": "basics-t-004-insert-gpt-4o-llm",
+ "llm_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true",
+ "query": "SELECT Id, Name, Age, Active FROM users WHERE Id=1",
+ "reducer": "InsertUser",
+ "server": "local"
+ }
+ }
+ },
+ "vendor": "openai",
+ "started_at": "2025-10-19T23:14:49.259201Z",
+ "finished_at": "2025-10-19T23:19:31.384858900Z"
+ },
"t_005_update": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
"task": "t_005_update",
@@ -2980,25 +3101,6 @@
"work_dir_golden": "target\\llm-runs\\basics\\t_005_update\\csharp\\server\\golden",
"work_dir_llm": "target\\llm-runs\\basics\\t_005_update\\csharp\\server\\gpt-4o\\llm",
"scorer_details": {
- "data_parity_update_user": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "args": [
- 1,
- "Alice2",
- 31,
- false
- ],
- "golden_db": "basics-t-005-update-golden",
- "golden_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
- "llm_db": "basics-t-005-update-gpt-4o-llm",
- "llm_out": "Id | Name | Age | Active ----+------+-----+--------",
- "query": "SELECT Id, Name, Age, Active FROM users WHERE Id=1",
- "reducer": "UpdateUser",
- "server": "local"
- }
- },
"schema_parity": {
"pass": true,
"partial": 1.0,
@@ -3020,82 +3122,121 @@
"phase": "sql_golden",
"sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
}
+ },
+ "data_parity_update_user": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "args": [
+ 1,
+ "Alice2",
+ 31,
+ false
+ ],
+ "golden_db": "basics-t-005-update-golden",
+ "golden_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
+ "llm_db": "basics-t-005-update-gpt-4o-llm",
+ "llm_out": "Id | Name | Age | Active ----+------+-----+--------",
+ "query": "SELECT Id, Name, Age, Active FROM users WHERE Id=1",
+ "reducer": "UpdateUser",
+ "server": "local"
+ }
}
},
"vendor": "openai",
"started_at": "2025-10-19T23:14:49.338948400Z",
"finished_at": "2025-10-19T23:22:08.272449900Z"
},
- "t_001_basic_tables": {
+ "t_006_delete": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_001_basic_tables",
+ "task": "t_006_delete",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-4o",
- "total_tests": 1,
- "passed_tests": 1,
+ "total_tests": 3,
+ "passed_tests": 3,
"llm_output": null,
"category": "basics",
"route_api_model": "gpt-4o",
- "golden_db": "basics-t-001-basic-tables-golden",
- "llm_db": "basics-t-001-basic-tables-gpt-4o-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\gpt-4o\\llm",
+ "golden_db": "basics-t-006-delete-golden",
+ "llm_db": "basics-t-006-delete-gpt-4o-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\gpt-4o\\llm",
"scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-001-basic-tables-golden",
- "llm_db": "basics-t-001-basic-tables-gpt-4o-llm",
+ "golden_db": "basics-t-006-delete-golden",
+ "llm_db": "basics-t-006-delete-gpt-4o-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
"tables_diff": null,
"tables_equal": true
}
+ },
+ "seed_users_row": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
+ }
+ },
+ "delete_user_count_zero": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 0,
+ "expected": 0,
+ "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=1"
+ }
}
},
"vendor": "openai",
- "started_at": "2025-10-19T23:14:49.018655700Z",
- "finished_at": "2025-10-19T23:21:36.975184900Z"
+ "started_at": "2025-10-19T23:14:49.416091900Z",
+ "finished_at": "2025-10-19T23:22:07.466212900Z"
},
- "t_016_sum_type_columns": {
+ "t_007_crud": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_016_sum_type_columns",
+ "task": "t_007_crud",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-4o",
- "total_tests": 3,
- "passed_tests": 3,
+ "total_tests": 4,
+ "passed_tests": 2,
"llm_output": null,
- "category": "schema",
+ "category": "basics",
"route_api_model": "gpt-4o",
- "golden_db": "schema-t-016-sum-type-columns-golden",
- "llm_db": "schema-t-016-sum-type-columns-gpt-4o-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\gpt-4o\\llm",
+ "golden_db": "basics-t-007-crud-golden",
+ "llm_db": "basics-t-007-crud-gpt-4o-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\gpt-4o\\llm",
"scorer_details": {
- "sum_type_columns_row_parity": {
+ "crud_total_count_one": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "actual": 0,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM users"
+ }
+ },
+ "crud_row_id2_deleted": {
"pass": true,
"partial": 1.0,
"notes": {
- "args": [],
- "golden_db": "schema-t-016-sum-type-columns-golden",
- "golden_out": "Id | A | B ----+--------------------------+--------------------------------------- 1 | (Circle = (Radius = 10)) | (Rectangle = (Width = 4, Height = 6))",
- "llm_db": "schema-t-016-sum-type-columns-gpt-4o-llm",
- "llm_out": "Id | A | B ----+--------------------------+--------------------------------------- 1 | (Circle = (Radius = 10)) | (Rectangle = (Width = 4, Height = 6))",
- "query": "SELECT Id, A, B FROM drawings WHERE Id=1",
- "reducer": "Seed",
- "server": "local"
+ "actual": 0,
+ "expected": 0,
+ "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=2"
}
},
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-016-sum-type-columns-golden",
- "llm_db": "schema-t-016-sum-type-columns-gpt-4o-llm",
+ "golden_db": "basics-t-007-crud-golden",
+ "llm_db": "basics-t-007-crud-gpt-4o-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -3103,109 +3244,142 @@
"tables_equal": true
}
},
- "sum_type_columns_row_count": {
- "pass": true,
- "partial": 1.0,
+ "crud_row_id1_parity": {
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM drawings WHERE Id=1"
+ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: SpacetimeDB.UniqueConstraintViolationException: Value with given unique identifier already exists\n at SpacetimeDB.Internal.FFI.CheckedStatus.Marshaller.ConvertToManaged(Errno )\n at SpacetimeDB.Internal.FFI.datastore_insert_bsatn(TableId , Span`1 , UInt32& )\n at SpacetimeDB.Internal.ITableView`2[[SpacetimeDB.Internal.TableHandles.users, StdbModule, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null],[Module.User, StdbModule, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null]].DoInsert(User )\n at SpacetimeDB.Internal.TableHandles.users.Insert(User )\n at Module.Crud(ReducerContext )\n at ModuleRegistration.Crud.Invoke(BinaryReader , IReducerContext )\n at SpacetimeDB.Internal.Module.__call_reducer__(UInt32 id, UInt64 sender_0, UInt64 sender_1, UInt64 sender_2, UInt64 sender_3, UInt64 conn_id_0, UInt64 conn_id_1, Timestamp timestamp, BytesSource args, BytesSink error)\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:3000/v1/database/c200b8341c6ccfc8fe52f225fa1fbfc9b53d0de44b74ac5b291d8c94ef9e9fab/call/Crud)\n",
+ "phase": "call_reducer_golden"
}
}
},
"vendor": "openai",
- "started_at": "2025-10-19T23:14:50.213667100Z",
- "finished_at": "2025-10-19T23:22:04.938678600Z"
+ "started_at": "2025-10-19T23:14:49.493560800Z",
+ "finished_at": "2025-10-19T23:22:06.350294700Z"
},
- "t_021_multi_column_index": {
+ "t_008_index_lookup": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_021_multi_column_index",
+ "task": "t_008_index_lookup",
+ "lang": "csharp",
+ "golden_published": true,
+ "model_name": "GPT-4o",
+ "total_tests": 3,
+ "passed_tests": 0,
+ "llm_output": null,
+ "category": "basics",
+ "route_api_model": "gpt-4o",
+ "golden_db": "basics-t-008-index-lookup-golden",
+ "llm_db": "basics-t-008-index-lookup-gpt-4o-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gpt-4o\\llm",
+ "scorer_details": {
+ "publish_error": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gpt-4o\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'results' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gpt-4o\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gpt-4o\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(70,24): warning CS8981: The type name 'users' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gpt-4o\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gpt-4o\\llm\\Lib.cs(30,58): error CS1061: 'Module.User?' does not contain a definition for 'Id' and no accessible extension method 'Id' accepting a first argument of type 'Module.User?' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gpt-4o\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gpt-4o\\llm\\Lib.cs(30,74): error CS1061: 'Module.User?' does not contain a definition for 'Name' and no accessible extension method 'Name' accepting a first argument of type 'Module.User?' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gpt-4o\\llm\\StdbModule.csproj]\r\n",
+ "phase": "build_or_publish"
+ }
+ }
+ },
+ "vendor": "openai",
+ "started_at": "2025-10-19T23:14:49.571594500Z",
+ "finished_at": "2025-10-19T23:15:55.256976100Z"
+ },
+ "t_009_init": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_009_init",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-4o",
"total_tests": 4,
"passed_tests": 4,
"llm_output": null,
- "category": "schema",
+ "category": "basics",
"route_api_model": "gpt-4o",
- "golden_db": "schema-t-021-multi-column-index-golden",
- "llm_db": "schema-t-021-multi-column-index-gpt-4o-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\gpt-4o\\llm",
+ "golden_db": "basics-t-009-init-golden",
+ "llm_db": "basics-t-009-init-gpt-4o-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_009_init\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_009_init\\csharp\\server\\gpt-4o\\llm",
"scorer_details": {
- "mcindex_lookup_u7_d1": {
+ "init_seed_alice": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM logs WHERE UserId=7 AND Day=1"
+ "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true"
}
},
- "mcindex_lookup_u7_d2": {
+ "init_total_two": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 2,
+ "expected": 2,
+ "sql": "SELECT COUNT(*) AS n FROM users"
+ }
+ },
+ "init_seed_bob": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM logs WHERE UserId=7 AND Day=2"
+ "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false"
}
},
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-021-multi-column-index-golden",
- "llm_db": "schema-t-021-multi-column-index-gpt-4o-llm",
+ "golden_db": "basics-t-009-init-golden",
+ "llm_db": "basics-t-009-init-gpt-4o-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
"tables_diff": null,
"tables_equal": true
}
- },
- "mcindex_seed_count": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 3,
- "expected": 3,
- "sql": "SELECT COUNT(*) AS n FROM logs"
- }
}
},
"vendor": "openai",
- "started_at": "2025-10-19T23:15:55.257289800Z",
- "finished_at": "2025-10-19T23:22:05.343429300Z"
+ "started_at": "2025-10-19T23:14:49.650550900Z",
+ "finished_at": "2025-10-19T23:21:36.762874500Z"
},
- "t_013_spacetime_sum_type": {
+ "t_010_connect": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_013_spacetime_sum_type",
+ "task": "t_010_connect",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-4o",
- "total_tests": 3,
- "passed_tests": 0,
+ "total_tests": 1,
+ "passed_tests": 1,
"llm_output": null,
- "category": "schema",
+ "category": "basics",
"route_api_model": "gpt-4o",
- "golden_db": "schema-t-013-spacetime-sum-type-golden",
- "llm_db": "schema-t-013-spacetime-sum-type-gpt-4o-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\gpt-4o\\llm",
+ "golden_db": "basics-t-010-connect-golden",
+ "llm_db": "basics-t-010-connect-gpt-4o-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_010_connect\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_010_connect\\csharp\\server\\gpt-4o\\llm",
"scorer_details": {
- "publish_error": {
- "pass": false,
- "partial": 0.0,
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
"notes": {
- "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\gpt-4o\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'results' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\gpt-4o\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\gpt-4o\\llm\\Lib.cs(36,31): error CS0426: The type name 'Circle' does not exist in the type 'Module.Shape' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\gpt-4o\\llm\\StdbModule.csproj]\r\n",
- "phase": "build_or_publish"
+ "golden_db": "basics-t-010-connect-golden",
+ "llm_db": "basics-t-010-connect-gpt-4o-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
}
}
},
"vendor": "openai",
- "started_at": "2025-10-19T23:14:49.965780500Z",
- "finished_at": "2025-10-19T23:15:28.568301500Z"
+ "started_at": "2025-10-19T23:14:49.729349700Z",
+ "finished_at": "2025-10-19T23:22:09.123699500Z"
},
"t_011_helper_function": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
@@ -3223,15 +3397,6 @@
"work_dir_golden": "target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\golden",
"work_dir_llm": "target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\gpt-4o\\llm",
"scorer_details": {
- "helper_func_sum_abs": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1 AND Sum=5"
- }
- },
"helper_func_sum_parity": {
"pass": true,
"partial": 1.0,
@@ -3262,41 +3427,61 @@
"tables_diff": null,
"tables_equal": true
}
+ },
+ "helper_func_sum_abs": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1 AND Sum=5"
+ }
}
},
"vendor": "openai",
"started_at": "2025-10-19T23:14:49.807225800Z",
"finished_at": "2025-10-19T23:22:05.729373200Z"
},
- "t_006_delete": {
+ "t_012_spacetime_product_type": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_006_delete",
+ "task": "t_012_spacetime_product_type",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-4o",
"total_tests": 3,
"passed_tests": 3,
"llm_output": null,
- "category": "basics",
+ "category": "schema",
"route_api_model": "gpt-4o",
- "golden_db": "basics-t-006-delete-golden",
- "llm_db": "basics-t-006-delete-gpt-4o-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\gpt-4o\\llm",
+ "golden_db": "schema-t-012-spacetime-product-type-golden",
+ "llm_db": "schema-t-012-spacetime-product-type-gpt-4o-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\gpt-4o\\llm",
"scorer_details": {
- "seed_users_row": {
+ "product_type_row_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
+ "args": [
+ 1,
+ 2,
+ 3
+ ],
+ "golden_db": "schema-t-012-spacetime-product-type-golden",
+ "golden_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)",
+ "llm_db": "schema-t-012-spacetime-product-type-gpt-4o-llm",
+ "llm_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)",
+ "query": "SELECT Id, Value FROM results WHERE Id=1",
+ "reducer": "SetScore",
+ "server": "local"
}
},
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-006-delete-golden",
- "llm_db": "basics-t-006-delete-gpt-4o-llm",
+ "golden_db": "schema-t-012-spacetime-product-type-golden",
+ "llm_db": "schema-t-012-spacetime-product-type-gpt-4o-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -3304,120 +3489,142 @@
"tables_equal": true
}
},
- "delete_user_count_zero": {
+ "product_type_row_count": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 0,
- "expected": 0,
- "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=1"
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1"
}
}
},
"vendor": "openai",
- "started_at": "2025-10-19T23:14:49.416091900Z",
- "finished_at": "2025-10-19T23:22:07.466212900Z"
+ "started_at": "2025-10-19T23:14:49.884598100Z",
+ "finished_at": "2025-10-19T23:22:08.815334900Z"
},
- "t_007_crud": {
+ "t_013_spacetime_sum_type": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_007_crud",
+ "task": "t_013_spacetime_sum_type",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-4o",
- "total_tests": 4,
- "passed_tests": 2,
+ "total_tests": 3,
+ "passed_tests": 0,
"llm_output": null,
- "category": "basics",
+ "category": "schema",
"route_api_model": "gpt-4o",
- "golden_db": "basics-t-007-crud-golden",
- "llm_db": "basics-t-007-crud-gpt-4o-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\gpt-4o\\llm",
+ "golden_db": "schema-t-013-spacetime-sum-type-golden",
+ "llm_db": "schema-t-013-spacetime-sum-type-gpt-4o-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\gpt-4o\\llm",
"scorer_details": {
- "crud_row_id1_parity": {
+ "publish_error": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: SpacetimeDB.UniqueConstraintViolationException: Value with given unique identifier already exists\n at SpacetimeDB.Internal.FFI.CheckedStatus.Marshaller.ConvertToManaged(Errno )\n at SpacetimeDB.Internal.FFI.datastore_insert_bsatn(TableId , Span`1 , UInt32& )\n at SpacetimeDB.Internal.ITableView`2[[SpacetimeDB.Internal.TableHandles.users, StdbModule, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null],[Module.User, StdbModule, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null]].DoInsert(User )\n at SpacetimeDB.Internal.TableHandles.users.Insert(User )\n at Module.Crud(ReducerContext )\n at ModuleRegistration.Crud.Invoke(BinaryReader , IReducerContext )\n at SpacetimeDB.Internal.Module.__call_reducer__(UInt32 id, UInt64 sender_0, UInt64 sender_1, UInt64 sender_2, UInt64 sender_3, UInt64 conn_id_0, UInt64 conn_id_1, Timestamp timestamp, BytesSource args, BytesSink error)\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:3000/v1/database/c200b8341c6ccfc8fe52f225fa1fbfc9b53d0de44b74ac5b291d8c94ef9e9fab/call/Crud)\n",
- "phase": "call_reducer_golden"
+ "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\gpt-4o\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'results' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\gpt-4o\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\gpt-4o\\llm\\Lib.cs(36,31): error CS0426: The type name 'Circle' does not exist in the type 'Module.Shape' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\gpt-4o\\llm\\StdbModule.csproj]\r\n",
+ "phase": "build_or_publish"
}
- },
- "schema_parity": {
+ }
+ },
+ "vendor": "openai",
+ "started_at": "2025-10-19T23:14:49.965780500Z",
+ "finished_at": "2025-10-19T23:15:28.568301500Z"
+ },
+ "t_014_elementary_columns": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_014_elementary_columns",
+ "lang": "csharp",
+ "golden_published": true,
+ "model_name": "GPT-4o",
+ "total_tests": 3,
+ "passed_tests": 3,
+ "llm_output": null,
+ "category": "schema",
+ "route_api_model": "gpt-4o",
+ "golden_db": "schema-t-014-elementary-columns-golden",
+ "llm_db": "schema-t-014-elementary-columns-gpt-4o-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_014_elementary_columns\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_014_elementary_columns\\csharp\\server\\gpt-4o\\llm",
+ "scorer_details": {
+ "elementary_columns_row_count": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-007-crud-golden",
- "llm_db": "basics-t-007-crud-gpt-4o-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM primitives WHERE Id=1"
}
},
- "crud_row_id2_deleted": {
+ "elementary_columns_row_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 0,
- "expected": 0,
- "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=2"
+ "args": [],
+ "golden_db": "schema-t-014-elementary-columns-golden",
+ "golden_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"",
+ "llm_db": "schema-t-014-elementary-columns-gpt-4o-llm",
+ "llm_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"",
+ "query": "SELECT Id, Count, Total, Price, Ratio, Active, Name FROM primitives WHERE Id=1",
+ "reducer": "Seed",
+ "server": "local"
}
},
- "crud_total_count_one": {
- "pass": false,
- "partial": 0.0,
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
"notes": {
- "actual": 0,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM users"
+ "golden_db": "schema-t-014-elementary-columns-golden",
+ "llm_db": "schema-t-014-elementary-columns-gpt-4o-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
}
}
},
"vendor": "openai",
- "started_at": "2025-10-19T23:14:49.493560800Z",
- "finished_at": "2025-10-19T23:22:06.350294700Z"
+ "started_at": "2025-10-19T23:14:50.045925800Z",
+ "finished_at": "2025-10-19T23:22:06.933706Z"
},
- "t_009_init": {
+ "t_015_product_type_columns": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_009_init",
+ "task": "t_015_product_type_columns",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-4o",
- "total_tests": 4,
- "passed_tests": 4,
+ "total_tests": 3,
+ "passed_tests": 3,
"llm_output": null,
- "category": "basics",
+ "category": "schema",
"route_api_model": "gpt-4o",
- "golden_db": "basics-t-009-init-golden",
- "llm_db": "basics-t-009-init-gpt-4o-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_009_init\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_009_init\\csharp\\server\\gpt-4o\\llm",
+ "golden_db": "schema-t-015-product-type-columns-golden",
+ "llm_db": "schema-t-015-product-type-columns-gpt-4o-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\gpt-4o\\llm",
"scorer_details": {
- "init_total_two": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 2,
- "expected": 2,
- "sql": "SELECT COUNT(*) AS n FROM users"
- }
- },
- "init_seed_bob": {
+ "product_type_columns_row_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false"
+ "args": [],
+ "golden_db": "schema-t-015-product-type-columns-golden",
+ "golden_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)",
+ "llm_db": "schema-t-015-product-type-columns-gpt-4o-llm",
+ "llm_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)",
+ "query": "SELECT Id, Home, Work, Pos FROM profiles WHERE Id=1",
+ "reducer": "Seed",
+ "server": "local"
}
},
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-009-init-golden",
- "llm_db": "basics-t-009-init-gpt-4o-llm",
+ "golden_db": "schema-t-015-product-type-columns-golden",
+ "llm_db": "schema-t-015-product-type-columns-gpt-4o-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -3425,42 +3632,56 @@
"tables_equal": true
}
},
- "init_seed_alice": {
+ "product_type_columns_row_count": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true"
+ "sql": "SELECT COUNT(*) AS n FROM profiles WHERE Id=1"
}
}
},
"vendor": "openai",
- "started_at": "2025-10-19T23:14:49.650550900Z",
- "finished_at": "2025-10-19T23:21:36.762874500Z"
+ "started_at": "2025-10-19T23:14:50.129537100Z",
+ "finished_at": "2025-10-19T23:22:10.004667200Z"
},
- "t_004_insert": {
+ "t_016_sum_type_columns": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_004_insert",
+ "task": "t_016_sum_type_columns",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-4o",
- "total_tests": 2,
- "passed_tests": 2,
+ "total_tests": 3,
+ "passed_tests": 3,
"llm_output": null,
- "category": "basics",
+ "category": "schema",
"route_api_model": "gpt-4o",
- "golden_db": "basics-t-004-insert-golden",
- "llm_db": "basics-t-004-insert-gpt-4o-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_004_insert\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_004_insert\\csharp\\server\\gpt-4o\\llm",
+ "golden_db": "schema-t-016-sum-type-columns-golden",
+ "llm_db": "schema-t-016-sum-type-columns-gpt-4o-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\gpt-4o\\llm",
"scorer_details": {
+ "sum_type_columns_row_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "args": [],
+ "golden_db": "schema-t-016-sum-type-columns-golden",
+ "golden_out": "Id | A | B ----+--------------------------+--------------------------------------- 1 | (Circle = (Radius = 10)) | (Rectangle = (Width = 4, Height = 6))",
+ "llm_db": "schema-t-016-sum-type-columns-gpt-4o-llm",
+ "llm_out": "Id | A | B ----+--------------------------+--------------------------------------- 1 | (Circle = (Radius = 10)) | (Rectangle = (Width = 4, Height = 6))",
+ "query": "SELECT Id, A, B FROM drawings WHERE Id=1",
+ "reducer": "Seed",
+ "server": "local"
+ }
+ },
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-004-insert-golden",
- "llm_db": "basics-t-004-insert-gpt-4o-llm",
+ "golden_db": "schema-t-016-sum-type-columns-golden",
+ "llm_db": "schema-t-016-sum-type-columns-gpt-4o-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -3468,61 +3689,42 @@
"tables_equal": true
}
},
- "data_parity_insert_user": {
+ "sum_type_columns_row_count": {
"pass": true,
"partial": 1.0,
"notes": {
- "args": [
- 1,
- "Alice",
- 30,
- true
- ],
- "golden_db": "basics-t-004-insert-golden",
- "golden_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true",
- "llm_db": "basics-t-004-insert-gpt-4o-llm",
- "llm_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true",
- "query": "SELECT Id, Name, Age, Active FROM users WHERE Id=1",
- "reducer": "InsertUser",
- "server": "local"
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM drawings WHERE Id=1"
}
}
},
"vendor": "openai",
- "started_at": "2025-10-19T23:14:49.259201Z",
- "finished_at": "2025-10-19T23:19:31.384858900Z"
+ "started_at": "2025-10-19T23:14:50.213667100Z",
+ "finished_at": "2025-10-19T23:22:04.938678600Z"
},
- "t_012_spacetime_product_type": {
+ "t_017_scheduled_columns": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_012_spacetime_product_type",
+ "task": "t_017_scheduled_columns",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-4o",
- "total_tests": 3,
- "passed_tests": 3,
+ "total_tests": 2,
+ "passed_tests": 2,
"llm_output": null,
"category": "schema",
"route_api_model": "gpt-4o",
- "golden_db": "schema-t-012-spacetime-product-type-golden",
- "llm_db": "schema-t-012-spacetime-product-type-gpt-4o-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\gpt-4o\\llm",
+ "golden_db": "schema-t-017-scheduled-columns-golden",
+ "llm_db": "schema-t-017-scheduled-columns-gpt-4o-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_017_scheduled_columns\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_017_scheduled_columns\\csharp\\server\\gpt-4o\\llm",
"scorer_details": {
- "product_type_row_count": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1"
- }
- },
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-012-spacetime-product-type-golden",
- "llm_db": "schema-t-012-spacetime-product-type-gpt-4o-llm",
+ "golden_db": "schema-t-017-scheduled-columns-golden",
+ "llm_db": "schema-t-017-scheduled-columns-gpt-4o-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -3530,62 +3732,76 @@
"tables_equal": true
}
},
- "product_type_row_parity": {
+ "scheduled_seeded_one_row": {
"pass": true,
"partial": 1.0,
"notes": {
- "args": [
- 1,
- 2,
- 3
- ],
- "golden_db": "schema-t-012-spacetime-product-type-golden",
- "golden_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)",
- "llm_db": "schema-t-012-spacetime-product-type-gpt-4o-llm",
- "llm_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)",
- "query": "SELECT Id, Value FROM results WHERE Id=1",
- "reducer": "SetScore",
- "server": "local"
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM tick_timer WHERE ScheduledId>=0"
}
}
},
"vendor": "openai",
- "started_at": "2025-10-19T23:14:49.884598100Z",
- "finished_at": "2025-10-19T23:22:08.815334900Z"
+ "started_at": "2025-10-19T23:14:50.301619800Z",
+ "finished_at": "2025-10-19T23:22:05.908430300Z"
},
- "t_002_scheduled_table": {
+ "t_018_constraints": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_002_scheduled_table",
+ "task": "t_018_constraints",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-4o",
- "total_tests": 1,
- "passed_tests": 1,
+ "total_tests": 3,
+ "passed_tests": 3,
"llm_output": null,
- "category": "basics",
+ "category": "schema",
"route_api_model": "gpt-4o",
- "golden_db": "basics-t-002-scheduled-table-golden",
- "llm_db": "basics-t-002-scheduled-table-gpt-4o-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_002_scheduled_table\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_002_scheduled_table\\csharp\\server\\gpt-4o\\llm",
+ "golden_db": "schema-t-018-constraints-golden",
+ "llm_db": "schema-t-018-constraints-gpt-4o-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\gpt-4o\\llm",
"scorer_details": {
+ "constraints_seed_two_rows": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM accounts WHERE Id=2"
+ }
+ },
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-002-scheduled-table-golden",
- "llm_db": "basics-t-002-scheduled-table-gpt-4o-llm",
+ "golden_db": "schema-t-018-constraints-golden",
+ "llm_db": "schema-t-018-constraints-gpt-4o-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
"tables_diff": null,
"tables_equal": true
}
+ },
+ "constraints_row_parity_after_seed": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "args": [],
+ "golden_db": "schema-t-018-constraints-golden",
+ "golden_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"",
+ "llm_db": "schema-t-018-constraints-gpt-4o-llm",
+ "llm_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"",
+ "query": "SELECT Id, Email, Name FROM accounts WHERE Id=1",
+ "reducer": "Seed",
+ "server": "local"
+ }
}
},
"vendor": "openai",
- "started_at": "2025-10-19T23:14:49.097275400Z",
- "finished_at": "2025-10-19T23:22:07.069582700Z"
+ "started_at": "2025-10-19T23:14:50.376242400Z",
+ "finished_at": "2025-10-19T23:22:09.578243500Z"
},
"t_019_many_to_many": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
@@ -3603,15 +3819,6 @@
"work_dir_golden": "target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\golden",
"work_dir_llm": "target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\gpt-4o\\llm",
"scorer_details": {
- "m2m_has_1_10": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM memberships WHERE UserId=1 AND GroupId=10"
- }
- },
"memberships_three_rows": {
"pass": true,
"partial": 1.0,
@@ -3621,13 +3828,13 @@
"sql": "SELECT COUNT(*) AS n FROM memberships"
}
},
- "m2m_has_2_20": {
+ "m2m_has_1_10": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM memberships WHERE UserId=2 AND GroupId=20"
+ "sql": "SELECT COUNT(*) AS n FROM memberships WHERE UserId=1 AND GroupId=10"
}
},
"m2m_has_1_20": {
@@ -3639,6 +3846,15 @@
"sql": "SELECT COUNT(*) AS n FROM memberships WHERE UserId=1 AND GroupId=20"
}
},
+ "m2m_has_2_20": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM memberships WHERE UserId=2 AND GroupId=20"
+ }
+ },
"schema_parity": {
"pass": true,
"partial": 1.0,
@@ -3657,71 +3873,14 @@
"started_at": "2025-10-19T23:14:50.450933100Z",
"finished_at": "2025-10-19T23:22:07.905233Z"
},
- "t_014_elementary_columns": {
+ "t_020_ecs": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_014_elementary_columns",
+ "task": "t_020_ecs",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-4o",
- "total_tests": 3,
- "passed_tests": 3,
- "llm_output": null,
- "category": "schema",
- "route_api_model": "gpt-4o",
- "golden_db": "schema-t-014-elementary-columns-golden",
- "llm_db": "schema-t-014-elementary-columns-gpt-4o-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_014_elementary_columns\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_014_elementary_columns\\csharp\\server\\gpt-4o\\llm",
- "scorer_details": {
- "elementary_columns_row_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "args": [],
- "golden_db": "schema-t-014-elementary-columns-golden",
- "golden_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"",
- "llm_db": "schema-t-014-elementary-columns-gpt-4o-llm",
- "llm_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"",
- "query": "SELECT Id, Count, Total, Price, Ratio, Active, Name FROM primitives WHERE Id=1",
- "reducer": "Seed",
- "server": "local"
- }
- },
- "elementary_columns_row_count": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM primitives WHERE Id=1"
- }
- },
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "golden_db": "schema-t-014-elementary-columns-golden",
- "llm_db": "schema-t-014-elementary-columns-gpt-4o-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
- }
- }
- },
- "vendor": "openai",
- "started_at": "2025-10-19T23:14:50.045925800Z",
- "finished_at": "2025-10-19T23:22:06.933706Z"
- },
- "t_020_ecs": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_020_ecs",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "GPT-4o",
- "total_tests": 5,
- "passed_tests": 0,
+ "total_tests": 5,
+ "passed_tests": 0,
"llm_output": null,
"category": "schema",
"route_api_model": "gpt-4o",
@@ -3743,28 +3902,46 @@
"started_at": "2025-10-19T23:15:28.569819600Z",
"finished_at": "2025-10-19T23:16:14.434383700Z"
},
- "t_017_scheduled_columns": {
+ "t_021_multi_column_index": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_017_scheduled_columns",
+ "task": "t_021_multi_column_index",
"lang": "csharp",
"golden_published": true,
"model_name": "GPT-4o",
- "total_tests": 2,
- "passed_tests": 2,
+ "total_tests": 4,
+ "passed_tests": 4,
"llm_output": null,
"category": "schema",
"route_api_model": "gpt-4o",
- "golden_db": "schema-t-017-scheduled-columns-golden",
- "llm_db": "schema-t-017-scheduled-columns-gpt-4o-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_017_scheduled_columns\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_017_scheduled_columns\\csharp\\server\\gpt-4o\\llm",
+ "golden_db": "schema-t-021-multi-column-index-golden",
+ "llm_db": "schema-t-021-multi-column-index-gpt-4o-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\gpt-4o\\llm",
"scorer_details": {
+ "mcindex_lookup_u7_d1": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM logs WHERE UserId=7 AND Day=1"
+ }
+ },
+ "mcindex_seed_count": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 3,
+ "expected": 3,
+ "sql": "SELECT COUNT(*) AS n FROM logs"
+ }
+ },
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-017-scheduled-columns-golden",
- "llm_db": "schema-t-017-scheduled-columns-gpt-4o-llm",
+ "golden_db": "schema-t-021-multi-column-index-golden",
+ "llm_db": "schema-t-021-multi-column-index-gpt-4o-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -3772,71 +3949,48 @@
"tables_equal": true
}
},
- "scheduled_seeded_one_row": {
+ "mcindex_lookup_u7_d2": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM tick_timer WHERE ScheduledId>=0"
+ "sql": "SELECT COUNT(*) AS n FROM logs WHERE UserId=7 AND Day=2"
}
}
},
"vendor": "openai",
- "started_at": "2025-10-19T23:14:50.301619800Z",
- "finished_at": "2025-10-19T23:22:05.908430300Z"
- },
+ "started_at": "2025-10-19T23:15:55.257289800Z",
+ "finished_at": "2025-10-19T23:22:05.343429300Z"
+ }
+ }
+ },
+ {
+ "name": "Claude 4.5 Sonnet",
+ "route_api_model": "claude-sonnet-4-5",
+ "tasks": {
"t_000_empty_reducers": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
"task": "t_000_empty_reducers",
"lang": "csharp",
"golden_published": true,
- "model_name": "GPT-4o",
+ "model_name": "Claude 4.5 Sonnet",
"total_tests": 1,
- "passed_tests": 0,
- "llm_output": "```csharp\npublic static partial class Module\n{\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_NoArgs(ReducerContext ctx)\n {\n }\n\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_WithInt(ReducerContext ctx, int count)\n {\n }\n\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_WithString(ReducerContext ctx, string name)\n {\n }\n\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_WithTwoArgs(ReducerContext ctx, int count, string name)\n {\n }\n\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_WithThreeArgs(ReducerContext ctx, bool active, float ratio, string label)\n {\n }\n}\n```",
+ "passed_tests": 1,
+ "llm_output": "```csharp\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_NoArgs(ReducerContext ctx)\n {\n }\n\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_WithInt(ReducerContext ctx, int count)\n {\n }\n\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_WithString(ReducerContext ctx, string name)\n {\n }\n\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_WithTwoArgs(ReducerContext ctx, int count, string name)\n {\n }\n\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_WithThreeArgs(ReducerContext ctx, bool active, float ratio, string label)\n {\n }\n}\n```",
"category": "basics",
- "route_api_model": "gpt-4o",
+ "route_api_model": "claude-sonnet-4-5",
"golden_db": "basics-t-000-empty-reducers-golden",
- "llm_db": "basics-t-000-empty-reducers-gpt-4o-llm",
+ "llm_db": "basics-t-000-empty-reducers-claude-4-5-sonnet-llm",
"work_dir_golden": "target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\gpt-4o\\llm",
- "scorer_details": {
- "publish_error": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\gpt-4o\\llm\\Lib.cs(5,44): error CS0246: The type or namespace name 'ReducerContext' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\gpt-4o\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\gpt-4o\\llm\\Lib.cs(10,45): error CS0246: The type or namespace name 'ReducerContext' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\gpt-4o\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\gpt-4o\\llm\\Lib.cs(15,48): error CS0246: The type or namespace name 'ReducerContext' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\gpt-4o\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\gpt-4o\\llm\\Lib.cs(20,49): error CS0246: The type or namespace name 'ReducerContext' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\gpt-4o\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\gpt-4o\\llm\\Lib.cs(25,51): error CS0246: The type or namespace name 'ReducerContext' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\gpt-4o\\llm\\StdbModule.csproj]\r\n",
- "phase": "build_or_publish"
- }
- }
- },
- "vendor": "openai",
- "started_at": "2025-10-21T23:58:20.810430600Z",
- "finished_at": "2025-10-21T23:58:40.508615200Z"
- },
- "t_010_connect": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_010_connect",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "GPT-4o",
- "total_tests": 1,
- "passed_tests": 1,
- "llm_output": null,
- "category": "basics",
- "route_api_model": "gpt-4o",
- "golden_db": "basics-t-010-connect-golden",
- "llm_db": "basics-t-010-connect-gpt-4o-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_010_connect\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_010_connect\\csharp\\server\\gpt-4o\\llm",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\claude-4-5-sonnet\\llm",
"scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-010-connect-golden",
- "llm_db": "basics-t-010-connect-gpt-4o-llm",
+ "golden_db": "basics-t-000-empty-reducers-golden",
+ "llm_db": "basics-t-000-empty-reducers-claude-4-5-sonnet-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -3845,133 +3999,81 @@
}
}
},
- "vendor": "openai",
- "started_at": "2025-10-19T23:14:49.729349700Z",
- "finished_at": "2025-10-19T23:22:09.123699500Z"
+ "vendor": "anthropic",
+ "started_at": "2025-10-22T00:01:34.778974900Z",
+ "finished_at": "2025-10-22T00:02:21.346109400Z"
},
- "t_018_constraints": {
+ "t_001_basic_tables": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_018_constraints",
+ "task": "t_001_basic_tables",
"lang": "csharp",
"golden_published": true,
- "model_name": "GPT-4o",
- "total_tests": 3,
- "passed_tests": 3,
+ "model_name": "Claude 4.5 Sonnet",
+ "total_tests": 1,
+ "passed_tests": 1,
"llm_output": null,
- "category": "schema",
- "route_api_model": "gpt-4o",
- "golden_db": "schema-t-018-constraints-golden",
- "llm_db": "schema-t-018-constraints-gpt-4o-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\gpt-4o\\llm",
+ "category": "basics",
+ "route_api_model": "claude-sonnet-4-5",
+ "golden_db": "basics-t-001-basic-tables-golden",
+ "llm_db": "basics-t-001-basic-tables-claude-4-5-sonnet-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\claude-4-5-sonnet\\llm",
"scorer_details": {
- "constraints_seed_two_rows": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM accounts WHERE Id=2"
- }
- },
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-018-constraints-golden",
- "llm_db": "schema-t-018-constraints-gpt-4o-llm",
+ "golden_db": "basics-t-001-basic-tables-golden",
+ "llm_db": "basics-t-001-basic-tables-claude-4-5-sonnet-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
"tables_diff": null,
"tables_equal": true
}
- },
- "constraints_row_parity_after_seed": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "args": [],
- "golden_db": "schema-t-018-constraints-golden",
- "golden_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"",
- "llm_db": "schema-t-018-constraints-gpt-4o-llm",
- "llm_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"",
- "query": "SELECT Id, Email, Name FROM accounts WHERE Id=1",
- "reducer": "Seed",
- "server": "local"
- }
}
},
- "vendor": "openai",
- "started_at": "2025-10-19T23:14:50.376242400Z",
- "finished_at": "2025-10-19T23:22:09.578243500Z"
+ "vendor": "anthropic",
+ "started_at": "2025-10-20T20:02:02.003779500Z",
+ "finished_at": "2025-10-20T20:03:40.332710900Z"
},
- "t_015_product_type_columns": {
+ "t_002_scheduled_table": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_015_product_type_columns",
+ "task": "t_002_scheduled_table",
"lang": "csharp",
"golden_published": true,
- "model_name": "GPT-4o",
- "total_tests": 3,
- "passed_tests": 3,
+ "model_name": "Claude 4.5 Sonnet",
+ "total_tests": 1,
+ "passed_tests": 1,
"llm_output": null,
- "category": "schema",
- "route_api_model": "gpt-4o",
- "golden_db": "schema-t-015-product-type-columns-golden",
- "llm_db": "schema-t-015-product-type-columns-gpt-4o-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\gpt-4o\\llm",
+ "category": "basics",
+ "route_api_model": "claude-sonnet-4-5",
+ "golden_db": "basics-t-002-scheduled-table-golden",
+ "llm_db": "basics-t-002-scheduled-table-claude-4-5-sonnet-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_002_scheduled_table\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_002_scheduled_table\\csharp\\server\\claude-4-5-sonnet\\llm",
"scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-015-product-type-columns-golden",
- "llm_db": "schema-t-015-product-type-columns-gpt-4o-llm",
+ "golden_db": "basics-t-002-scheduled-table-golden",
+ "llm_db": "basics-t-002-scheduled-table-claude-4-5-sonnet-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
"tables_diff": null,
"tables_equal": true
}
- },
- "product_type_columns_row_count": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM profiles WHERE Id=1"
- }
- },
- "product_type_columns_row_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "args": [],
- "golden_db": "schema-t-015-product-type-columns-golden",
- "golden_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)",
- "llm_db": "schema-t-015-product-type-columns-gpt-4o-llm",
- "llm_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)",
- "query": "SELECT Id, Home, Work, Pos FROM profiles WHERE Id=1",
- "reducer": "Seed",
- "server": "local"
- }
}
},
- "vendor": "openai",
- "started_at": "2025-10-19T23:14:50.129537100Z",
- "finished_at": "2025-10-19T23:22:10.004667200Z"
- }
- }
- },
- {
- "name": "Claude 4.5 Sonnet",
- "route_api_model": "claude-sonnet-4-5",
- "tasks": {
- "t_010_connect": {
+ "vendor": "anthropic",
+ "started_at": "2025-10-20T20:02:02.870487100Z",
+ "finished_at": "2025-10-20T20:03:40.137742600Z"
+ },
+ "t_003_struct_in_table": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_010_connect",
+ "task": "t_003_struct_in_table",
"lang": "csharp",
"golden_published": true,
"model_name": "Claude 4.5 Sonnet",
@@ -3980,17 +4082,17 @@
"llm_output": null,
"category": "basics",
"route_api_model": "claude-sonnet-4-5",
- "golden_db": "basics-t-010-connect-golden",
- "llm_db": "basics-t-010-connect-claude-4-5-sonnet-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_010_connect\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_010_connect\\csharp\\server\\claude-4-5-sonnet\\llm",
+ "golden_db": "basics-t-003-struct-in-table-golden",
+ "llm_db": "basics-t-003-struct-in-table-claude-4-5-sonnet-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_003_struct_in_table\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_003_struct_in_table\\csharp\\server\\claude-4-5-sonnet\\llm",
"scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-010-connect-golden",
- "llm_db": "basics-t-010-connect-claude-4-5-sonnet-llm",
+ "golden_db": "basics-t-003-struct-in-table-golden",
+ "llm_db": "basics-t-003-struct-in-table-claude-4-5-sonnet-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -4000,65 +4102,103 @@
}
},
"vendor": "anthropic",
- "started_at": "2025-10-20T20:02:09.220935700Z",
- "finished_at": "2025-10-20T20:03:42.589474300Z"
+ "started_at": "2025-10-20T20:02:03.728356300Z",
+ "finished_at": "2025-10-20T20:03:39.465168500Z"
},
- "t_000_empty_reducers": {
+ "t_004_insert": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_000_empty_reducers",
+ "task": "t_004_insert",
"lang": "csharp",
"golden_published": true,
"model_name": "Claude 4.5 Sonnet",
- "total_tests": 1,
- "passed_tests": 1,
- "llm_output": "```csharp\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_NoArgs(ReducerContext ctx)\n {\n }\n\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_WithInt(ReducerContext ctx, int count)\n {\n }\n\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_WithString(ReducerContext ctx, string name)\n {\n }\n\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_WithTwoArgs(ReducerContext ctx, int count, string name)\n {\n }\n\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_WithThreeArgs(ReducerContext ctx, bool active, float ratio, string label)\n {\n }\n}\n```",
+ "total_tests": 2,
+ "passed_tests": 2,
+ "llm_output": null,
"category": "basics",
"route_api_model": "claude-sonnet-4-5",
- "golden_db": "basics-t-000-empty-reducers-golden",
- "llm_db": "basics-t-000-empty-reducers-claude-4-5-sonnet-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\claude-4-5-sonnet\\llm",
+ "golden_db": "basics-t-004-insert-golden",
+ "llm_db": "basics-t-004-insert-claude-4-5-sonnet-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_004_insert\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_004_insert\\csharp\\server\\claude-4-5-sonnet\\llm",
"scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-000-empty-reducers-golden",
- "llm_db": "basics-t-000-empty-reducers-claude-4-5-sonnet-llm",
+ "golden_db": "basics-t-004-insert-golden",
+ "llm_db": "basics-t-004-insert-claude-4-5-sonnet-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
"tables_diff": null,
"tables_equal": true
}
+ },
+ "data_parity_insert_user": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "args": [
+ 1,
+ "Alice",
+ 30,
+ true
+ ],
+ "golden_db": "basics-t-004-insert-golden",
+ "golden_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true",
+ "llm_db": "basics-t-004-insert-claude-4-5-sonnet-llm",
+ "llm_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true",
+ "query": "SELECT Id, Name, Age, Active FROM users WHERE Id=1",
+ "reducer": "InsertUser",
+ "server": "local"
+ }
}
},
"vendor": "anthropic",
- "started_at": "2025-10-22T00:01:34.778974900Z",
- "finished_at": "2025-10-22T00:02:21.346109400Z"
+ "started_at": "2025-10-20T20:02:04.535678400Z",
+ "finished_at": "2025-10-20T20:04:43.019219800Z"
},
- "t_012_spacetime_product_type": {
+ "t_005_update": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_012_spacetime_product_type",
+ "task": "t_005_update",
"lang": "csharp",
"golden_published": true,
"model_name": "Claude 4.5 Sonnet",
"total_tests": 3,
"passed_tests": 3,
"llm_output": null,
- "category": "schema",
+ "category": "basics",
"route_api_model": "claude-sonnet-4-5",
- "golden_db": "schema-t-012-spacetime-product-type-golden",
- "llm_db": "schema-t-012-spacetime-product-type-claude-4-5-sonnet-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\claude-4-5-sonnet\\llm",
+ "golden_db": "basics-t-005-update-golden",
+ "llm_db": "basics-t-005-update-claude-4-5-sonnet-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_005_update\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_005_update\\csharp\\server\\claude-4-5-sonnet\\llm",
"scorer_details": {
+ "data_parity_update_user": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "args": [
+ 1,
+ "Alice2",
+ 31,
+ false
+ ],
+ "golden_db": "basics-t-005-update-golden",
+ "golden_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
+ "llm_db": "basics-t-005-update-claude-4-5-sonnet-llm",
+ "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
+ "query": "SELECT Id, Name, Age, Active FROM users WHERE Id=1",
+ "reducer": "UpdateUser",
+ "server": "local"
+ }
+ },
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-012-spacetime-product-type-golden",
- "llm_db": "schema-t-012-spacetime-product-type-claude-4-5-sonnet-llm",
+ "golden_db": "basics-t-005-update-golden",
+ "llm_db": "basics-t-005-update-claude-4-5-sonnet-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -4066,37 +4206,17 @@
"tables_equal": true
}
},
- "product_type_row_count": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1"
- }
- },
- "product_type_row_parity": {
+ "seed_users_row": {
"pass": true,
"partial": 1.0,
"notes": {
- "args": [
- 1,
- 2,
- 3
- ],
- "golden_db": "schema-t-012-spacetime-product-type-golden",
- "golden_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)",
- "llm_db": "schema-t-012-spacetime-product-type-claude-4-5-sonnet-llm",
- "llm_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)",
- "query": "SELECT Id, Value FROM results WHERE Id=1",
- "reducer": "SetScore",
- "server": "local"
+ "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
}
}
},
"vendor": "anthropic",
- "started_at": "2025-10-20T20:32:45.063269700Z",
- "finished_at": "2025-10-20T20:34:57.184216100Z"
+ "started_at": "2025-10-20T20:02:05.344244900Z",
+ "finished_at": "2025-10-20T20:03:40.000516300Z"
},
"t_006_delete": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
@@ -4114,6 +4234,19 @@
"work_dir_golden": "target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\golden",
"work_dir_llm": "target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\claude-4-5-sonnet\\llm",
"scorer_details": {
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "golden_db": "basics-t-006-delete-golden",
+ "llm_db": "basics-t-006-delete-claude-4-5-sonnet-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
+ }
+ },
"delete_user_count_zero": {
"pass": true,
"partial": 1.0,
@@ -4129,100 +4262,150 @@
"notes": {
"sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
}
- },
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "golden_db": "basics-t-006-delete-golden",
- "llm_db": "basics-t-006-delete-claude-4-5-sonnet-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
- }
}
},
"vendor": "anthropic",
"started_at": "2025-10-20T20:02:06.118333200Z",
"finished_at": "2025-10-20T20:03:42.420803600Z"
},
- "t_005_update": {
+ "t_007_crud": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_005_update",
+ "task": "t_007_crud",
"lang": "csharp",
"golden_published": true,
"model_name": "Claude 4.5 Sonnet",
- "total_tests": 3,
- "passed_tests": 3,
+ "total_tests": 4,
+ "passed_tests": 4,
"llm_output": null,
"category": "basics",
"route_api_model": "claude-sonnet-4-5",
- "golden_db": "basics-t-005-update-golden",
- "llm_db": "basics-t-005-update-claude-4-5-sonnet-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_005_update\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_005_update\\csharp\\server\\claude-4-5-sonnet\\llm",
+ "golden_db": "basics-t-007-crud-golden",
+ "llm_db": "basics-t-007-crud-claude-4-5-sonnet-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\claude-4-5-sonnet\\llm",
"scorer_details": {
- "seed_users_row": {
+ "schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
+ "golden_db": "basics-t-007-crud-golden",
+ "llm_db": "basics-t-007-crud-claude-4-5-sonnet-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
}
},
- "data_parity_update_user": {
+ "crud_row_id1_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "args": [
- 1,
- "Alice2",
- 31,
- false
- ],
- "golden_db": "basics-t-005-update-golden",
+ "args": [],
+ "golden_db": "basics-t-007-crud-golden",
"golden_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
- "llm_db": "basics-t-005-update-claude-4-5-sonnet-llm",
+ "llm_db": "basics-t-007-crud-claude-4-5-sonnet-llm",
"llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
"query": "SELECT Id, Name, Age, Active FROM users WHERE Id=1",
- "reducer": "UpdateUser",
+ "reducer": "Crud",
"server": "local"
}
},
- "schema_parity": {
+ "crud_total_count_one": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-005-update-golden",
- "llm_db": "basics-t-005-update-claude-4-5-sonnet-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM users"
+ }
+ },
+ "crud_row_id2_deleted": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 0,
+ "expected": 0,
+ "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=2"
}
}
},
"vendor": "anthropic",
- "started_at": "2025-10-20T20:02:05.344244900Z",
- "finished_at": "2025-10-20T20:03:40.000516300Z"
+ "started_at": "2025-10-20T20:02:06.891303Z",
+ "finished_at": "2025-10-20T20:03:41.980606100Z"
},
- "t_009_init": {
+ "t_008_index_lookup": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_009_init",
+ "task": "t_008_index_lookup",
"lang": "csharp",
"golden_published": true,
"model_name": "Claude 4.5 Sonnet",
- "total_tests": 4,
- "passed_tests": 4,
+ "total_tests": 3,
+ "passed_tests": 3,
"llm_output": null,
"category": "basics",
"route_api_model": "claude-sonnet-4-5",
- "golden_db": "basics-t-009-init-golden",
- "llm_db": "basics-t-009-init-claude-4-5-sonnet-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_009_init\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_009_init\\csharp\\server\\claude-4-5-sonnet\\llm",
+ "golden_db": "basics-t-008-index-lookup-golden",
+ "llm_db": "basics-t-008-index-lookup-claude-4-5-sonnet-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\claude-4-5-sonnet\\llm",
+ "scorer_details": {
+ "index_lookup_projection_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "args": [
+ 1
+ ],
+ "golden_db": "basics-t-008-index-lookup-golden",
+ "golden_out": "Id | Name ----+--------- 1 | \"Alice\"",
+ "llm_db": "basics-t-008-index-lookup-claude-4-5-sonnet-llm",
+ "llm_out": "Id | Name ----+--------- 1 | \"Alice\"",
+ "query": "SELECT Id, Name FROM results WHERE Id=1",
+ "reducer": "LookupUserName",
+ "server": "local"
+ }
+ },
+ "seed_user_row": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
+ }
+ },
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "golden_db": "basics-t-008-index-lookup-golden",
+ "llm_db": "basics-t-008-index-lookup-claude-4-5-sonnet-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
+ }
+ }
+ },
+ "vendor": "anthropic",
+ "started_at": "2025-10-20T20:02:07.665233500Z",
+ "finished_at": "2025-10-20T20:03:41.416098Z"
+ },
+ "t_009_init": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_009_init",
+ "lang": "csharp",
+ "golden_published": true,
+ "model_name": "Claude 4.5 Sonnet",
+ "total_tests": 4,
+ "passed_tests": 4,
+ "llm_output": null,
+ "category": "basics",
+ "route_api_model": "claude-sonnet-4-5",
+ "golden_db": "basics-t-009-init-golden",
+ "llm_db": "basics-t-009-init-claude-4-5-sonnet-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_009_init\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_009_init\\csharp\\server\\claude-4-5-sonnet\\llm",
"scorer_details": {
"init_total_two": {
"pass": true,
@@ -4233,13 +4416,17 @@
"sql": "SELECT COUNT(*) AS n FROM users"
}
},
- "init_seed_bob": {
+ "schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false"
+ "golden_db": "basics-t-009-init-golden",
+ "llm_db": "basics-t-009-init-claude-4-5-sonnet-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
}
},
"init_seed_alice": {
@@ -4251,12 +4438,42 @@
"sql": "SELECT COUNT(*) AS n FROM users WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true"
}
},
+ "init_seed_bob": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false"
+ }
+ }
+ },
+ "vendor": "anthropic",
+ "started_at": "2025-10-20T20:02:08.444043800Z",
+ "finished_at": "2025-10-20T20:04:19.040660700Z"
+ },
+ "t_010_connect": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_010_connect",
+ "lang": "csharp",
+ "golden_published": true,
+ "model_name": "Claude 4.5 Sonnet",
+ "total_tests": 1,
+ "passed_tests": 1,
+ "llm_output": null,
+ "category": "basics",
+ "route_api_model": "claude-sonnet-4-5",
+ "golden_db": "basics-t-010-connect-golden",
+ "llm_db": "basics-t-010-connect-claude-4-5-sonnet-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_010_connect\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_010_connect\\csharp\\server\\claude-4-5-sonnet\\llm",
+ "scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-009-init-golden",
- "llm_db": "basics-t-009-init-claude-4-5-sonnet-llm",
+ "golden_db": "basics-t-010-connect-golden",
+ "llm_db": "basics-t-010-connect-claude-4-5-sonnet-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -4266,12 +4483,12 @@
}
},
"vendor": "anthropic",
- "started_at": "2025-10-20T20:02:08.444043800Z",
- "finished_at": "2025-10-20T20:04:19.040660700Z"
+ "started_at": "2025-10-20T20:02:09.220935700Z",
+ "finished_at": "2025-10-20T20:03:42.589474300Z"
},
- "t_008_index_lookup": {
+ "t_011_helper_function": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_008_index_lookup",
+ "task": "t_011_helper_function",
"lang": "csharp",
"golden_published": true,
"model_name": "Claude 4.5 Sonnet",
@@ -4280,31 +4497,35 @@
"llm_output": null,
"category": "basics",
"route_api_model": "claude-sonnet-4-5",
- "golden_db": "basics-t-008-index-lookup-golden",
- "llm_db": "basics-t-008-index-lookup-claude-4-5-sonnet-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\claude-4-5-sonnet\\llm",
+ "golden_db": "basics-t-011-helper-function-golden",
+ "llm_db": "basics-t-011-helper-function-claude-4-5-sonnet-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\claude-4-5-sonnet\\llm",
"scorer_details": {
- "seed_user_row": {
+ "helper_func_sum_abs": {
"pass": true,
"partial": 1.0,
"notes": {
- "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1 AND Sum=5"
}
},
- "index_lookup_projection_parity": {
+ "helper_func_sum_parity": {
"pass": true,
"partial": 1.0,
"notes": {
"args": [
- 1
+ 1,
+ 2,
+ 3
],
- "golden_db": "basics-t-008-index-lookup-golden",
- "golden_out": "Id | Name ----+--------- 1 | \"Alice\"",
- "llm_db": "basics-t-008-index-lookup-claude-4-5-sonnet-llm",
- "llm_out": "Id | Name ----+--------- 1 | \"Alice\"",
- "query": "SELECT Id, Name FROM results WHERE Id=1",
- "reducer": "LookupUserName",
+ "golden_db": "basics-t-011-helper-function-golden",
+ "golden_out": "Id | Sum ----+----- 1 | 5",
+ "llm_db": "basics-t-011-helper-function-claude-4-5-sonnet-llm",
+ "llm_out": "Id | Sum ----+----- 1 | 5",
+ "query": "SELECT Id, Sum FROM results WHERE Id=1",
+ "reducer": "ComputeSum",
"server": "local"
}
},
@@ -4312,8 +4533,8 @@
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-008-index-lookup-golden",
- "llm_db": "basics-t-008-index-lookup-claude-4-5-sonnet-llm",
+ "golden_db": "basics-t-011-helper-function-golden",
+ "llm_db": "basics-t-011-helper-function-claude-4-5-sonnet-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -4323,74 +4544,98 @@
}
},
"vendor": "anthropic",
- "started_at": "2025-10-20T20:02:07.665233500Z",
- "finished_at": "2025-10-20T20:03:41.416098Z"
+ "started_at": "2025-10-20T20:02:10.006523700Z",
+ "finished_at": "2025-10-20T20:04:19.536787900Z"
},
- "t_007_crud": {
+ "t_012_spacetime_product_type": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_007_crud",
+ "task": "t_012_spacetime_product_type",
"lang": "csharp",
"golden_published": true,
"model_name": "Claude 4.5 Sonnet",
- "total_tests": 4,
- "passed_tests": 4,
+ "total_tests": 3,
+ "passed_tests": 3,
"llm_output": null,
- "category": "basics",
+ "category": "schema",
"route_api_model": "claude-sonnet-4-5",
- "golden_db": "basics-t-007-crud-golden",
- "llm_db": "basics-t-007-crud-claude-4-5-sonnet-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\claude-4-5-sonnet\\llm",
+ "golden_db": "schema-t-012-spacetime-product-type-golden",
+ "llm_db": "schema-t-012-spacetime-product-type-claude-4-5-sonnet-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\claude-4-5-sonnet\\llm",
"scorer_details": {
- "crud_row_id1_parity": {
+ "product_type_row_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "args": [],
- "golden_db": "basics-t-007-crud-golden",
- "golden_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
- "llm_db": "basics-t-007-crud-claude-4-5-sonnet-llm",
- "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
- "query": "SELECT Id, Name, Age, Active FROM users WHERE Id=1",
- "reducer": "Crud",
+ "args": [
+ 1,
+ 2,
+ 3
+ ],
+ "golden_db": "schema-t-012-spacetime-product-type-golden",
+ "golden_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)",
+ "llm_db": "schema-t-012-spacetime-product-type-claude-4-5-sonnet-llm",
+ "llm_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)",
+ "query": "SELECT Id, Value FROM results WHERE Id=1",
+ "reducer": "SetScore",
"server": "local"
}
},
+ "product_type_row_count": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1"
+ }
+ },
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-007-crud-golden",
- "llm_db": "basics-t-007-crud-claude-4-5-sonnet-llm",
+ "golden_db": "schema-t-012-spacetime-product-type-golden",
+ "llm_db": "schema-t-012-spacetime-product-type-claude-4-5-sonnet-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
"tables_diff": null,
"tables_equal": true
}
- },
- "crud_row_id2_deleted": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 0,
- "expected": 0,
- "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=2"
- }
- },
- "crud_total_count_one": {
- "pass": true,
- "partial": 1.0,
+ }
+ },
+ "vendor": "anthropic",
+ "started_at": "2025-10-20T20:32:45.063269700Z",
+ "finished_at": "2025-10-20T20:34:57.184216100Z"
+ },
+ "t_013_spacetime_sum_type": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_013_spacetime_sum_type",
+ "lang": "csharp",
+ "golden_published": true,
+ "model_name": "Claude 4.5 Sonnet",
+ "total_tests": 3,
+ "passed_tests": 0,
+ "llm_output": null,
+ "category": "schema",
+ "route_api_model": "claude-sonnet-4-5",
+ "golden_db": "schema-t-013-spacetime-sum-type-golden",
+ "llm_db": "schema-t-013-spacetime-sum-type-claude-4-5-sonnet-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\claude-4-5-sonnet\\llm",
+ "scorer_details": {
+ "publish_error": {
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM users"
+ "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\claude-4-5-sonnet\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.BSATN.Codegen\\SpacetimeDB.Codegen.Type\\Shape.cs(6,26): error CS8910: The primary constructor conflicts with the synthesized copy constructor. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\claude-4-5-sonnet\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\claude-4-5-sonnet\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.BSATN.Codegen\\SpacetimeDB.Codegen.Type\\Shape.cs(12,26): error CS8910: The primary constructor conflicts with the synthesized copy constructor. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\claude-4-5-sonnet\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\claude-4-5-sonnet\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'results' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\claude-4-5-sonnet\\llm\\StdbModule.csproj]\r\n",
+ "phase": "build_or_publish"
}
}
},
"vendor": "anthropic",
- "started_at": "2025-10-20T20:02:06.891303Z",
- "finished_at": "2025-10-20T20:03:41.980606100Z"
+ "started_at": "2025-10-20T20:32:45.889461900Z",
+ "finished_at": "2025-10-20T20:33:04.383640200Z"
},
"t_014_elementary_columns": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
@@ -4421,15 +4666,6 @@
"tables_equal": true
}
},
- "elementary_columns_row_count": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM primitives WHERE Id=1"
- }
- },
"elementary_columns_row_parity": {
"pass": true,
"partial": 1.0,
@@ -4443,45 +4679,77 @@
"reducer": "Seed",
"server": "local"
}
+ },
+ "elementary_columns_row_count": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM primitives WHERE Id=1"
+ }
}
},
"vendor": "anthropic",
"started_at": "2025-10-20T20:32:46.666579300Z",
"finished_at": "2025-10-20T20:34:02.615664900Z"
},
- "t_003_struct_in_table": {
+ "t_015_product_type_columns": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_003_struct_in_table",
+ "task": "t_015_product_type_columns",
"lang": "csharp",
"golden_published": true,
"model_name": "Claude 4.5 Sonnet",
- "total_tests": 1,
- "passed_tests": 1,
+ "total_tests": 3,
+ "passed_tests": 3,
"llm_output": null,
- "category": "basics",
+ "category": "schema",
"route_api_model": "claude-sonnet-4-5",
- "golden_db": "basics-t-003-struct-in-table-golden",
- "llm_db": "basics-t-003-struct-in-table-claude-4-5-sonnet-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_003_struct_in_table\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_003_struct_in_table\\csharp\\server\\claude-4-5-sonnet\\llm",
+ "golden_db": "schema-t-015-product-type-columns-golden",
+ "llm_db": "schema-t-015-product-type-columns-claude-4-5-sonnet-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\claude-4-5-sonnet\\llm",
"scorer_details": {
+ "product_type_columns_row_count": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM profiles WHERE Id=1"
+ }
+ },
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-003-struct-in-table-golden",
- "llm_db": "basics-t-003-struct-in-table-claude-4-5-sonnet-llm",
+ "golden_db": "schema-t-015-product-type-columns-golden",
+ "llm_db": "schema-t-015-product-type-columns-claude-4-5-sonnet-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
"tables_diff": null,
"tables_equal": true
}
+ },
+ "product_type_columns_row_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "args": [],
+ "golden_db": "schema-t-015-product-type-columns-golden",
+ "golden_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)",
+ "llm_db": "schema-t-015-product-type-columns-claude-4-5-sonnet-llm",
+ "llm_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)",
+ "query": "SELECT Id, Home, Work, Pos FROM profiles WHERE Id=1",
+ "reducer": "Seed",
+ "server": "local"
+ }
}
},
"vendor": "anthropic",
- "started_at": "2025-10-20T20:02:03.728356300Z",
- "finished_at": "2025-10-20T20:03:39.465168500Z"
+ "started_at": "2025-10-20T20:32:47.433184500Z",
+ "finished_at": "2025-10-20T20:34:06.772724400Z"
},
"t_016_sum_type_columns": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
@@ -4499,6 +4767,28 @@
"work_dir_golden": "target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\golden",
"work_dir_llm": "target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\claude-4-5-sonnet\\llm",
"scorer_details": {
+ "sum_type_columns_row_count": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM drawings WHERE Id=1"
+ }
+ },
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "golden_db": "schema-t-016-sum-type-columns-golden",
+ "llm_db": "schema-t-016-sum-type-columns-claude-4-5-sonnet-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
+ }
+ },
"sum_type_columns_row_parity": {
"pass": true,
"partial": 1.0,
@@ -4512,22 +4802,43 @@
"reducer": "Seed",
"server": "local"
}
- },
- "sum_type_columns_row_count": {
+ }
+ },
+ "vendor": "anthropic",
+ "started_at": "2025-10-20T20:32:48.200769700Z",
+ "finished_at": "2025-10-20T20:34:57.939715700Z"
+ },
+ "t_017_scheduled_columns": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_017_scheduled_columns",
+ "lang": "csharp",
+ "golden_published": true,
+ "model_name": "Claude 4.5 Sonnet",
+ "total_tests": 2,
+ "passed_tests": 2,
+ "llm_output": null,
+ "category": "schema",
+ "route_api_model": "claude-sonnet-4-5",
+ "golden_db": "schema-t-017-scheduled-columns-golden",
+ "llm_db": "schema-t-017-scheduled-columns-claude-4-5-sonnet-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_017_scheduled_columns\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_017_scheduled_columns\\csharp\\server\\claude-4-5-sonnet\\llm",
+ "scorer_details": {
+ "scheduled_seeded_one_row": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM drawings WHERE Id=1"
+ "sql": "SELECT COUNT(*) AS n FROM tick_timer WHERE ScheduledId>=0"
}
},
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-016-sum-type-columns-golden",
- "llm_db": "schema-t-016-sum-type-columns-claude-4-5-sonnet-llm",
+ "golden_db": "schema-t-017-scheduled-columns-golden",
+ "llm_db": "schema-t-017-scheduled-columns-claude-4-5-sonnet-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -4537,8 +4848,59 @@
}
},
"vendor": "anthropic",
- "started_at": "2025-10-20T20:32:48.200769700Z",
- "finished_at": "2025-10-20T20:34:57.939715700Z"
+ "started_at": "2025-10-20T20:32:49.000529800Z",
+ "finished_at": "2025-10-20T20:34:07.005785Z"
+ },
+ "t_018_constraints": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_018_constraints",
+ "lang": "csharp",
+ "golden_published": true,
+ "model_name": "Claude 4.5 Sonnet",
+ "total_tests": 3,
+ "passed_tests": 2,
+ "llm_output": null,
+ "category": "schema",
+ "route_api_model": "claude-sonnet-4-5",
+ "golden_db": "schema-t-018-constraints-golden",
+ "llm_db": "schema-t-018-constraints-claude-4-5-sonnet-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\claude-4-5-sonnet\\llm",
+ "scorer_details": {
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "golden_db": "schema-t-018-constraints-golden",
+ "llm_db": "schema-t-018-constraints-claude-4-5-sonnet-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
+ }
+ },
+ "constraints_row_parity_after_seed": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: can't directly call special Init lifecycle reducer\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:3000/v1/database/c200988c7e20b2058d3109402652f801c7191877ccc7e94f877c525721feaab5/call/Seed)\n",
+ "phase": "call_reducer_llm"
+ }
+ },
+ "constraints_seed_two_rows": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM accounts WHERE Id=2"
+ }
+ }
+ },
+ "vendor": "anthropic",
+ "started_at": "2025-10-20T20:32:49.781739300Z",
+ "finished_at": "2025-10-20T20:34:07.384235700Z"
},
"t_019_many_to_many": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
@@ -4556,22 +4918,13 @@
"work_dir_golden": "target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\golden",
"work_dir_llm": "target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\claude-4-5-sonnet\\llm",
"scorer_details": {
- "m2m_has_1_10": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM memberships WHERE UserId=1 AND GroupId=10"
- }
- },
- "m2m_has_2_20": {
+ "m2m_has_1_20": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM memberships WHERE UserId=2 AND GroupId=20"
+ "sql": "SELECT COUNT(*) AS n FROM memberships WHERE UserId=1 AND GroupId=20"
}
},
"schema_parity": {
@@ -4587,22 +4940,31 @@
"tables_equal": true
}
},
- "memberships_three_rows": {
+ "m2m_has_1_10": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 3,
- "expected": 3,
- "sql": "SELECT COUNT(*) AS n FROM memberships"
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM memberships WHERE UserId=1 AND GroupId=10"
}
},
- "m2m_has_1_20": {
+ "m2m_has_2_20": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM memberships WHERE UserId=1 AND GroupId=20"
+ "sql": "SELECT COUNT(*) AS n FROM memberships WHERE UserId=2 AND GroupId=20"
+ }
+ },
+ "memberships_three_rows": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 3,
+ "expected": 3,
+ "sql": "SELECT COUNT(*) AS n FROM memberships"
}
}
},
@@ -4626,22 +4988,26 @@
"work_dir_golden": "target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\golden",
"work_dir_llm": "target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\claude-4-5-sonnet\\llm",
"scorer_details": {
- "ecs_next_pos_entity1": {
+ "schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM next_positions WHERE EntityId=1 AND X=1 AND Y=0"
+ "golden_db": "schema-t-020-ecs-golden",
+ "llm_db": "schema-t-020-ecs-claude-4-5-sonnet-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
}
},
- "ecs_step_next_positions_count": {
+ "ecs_next_pos_entity1": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 2,
- "expected": 2,
- "sql": "SELECT COUNT(*) AS n FROM next_positions"
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM next_positions WHERE EntityId=1 AND X=1 AND Y=0"
}
},
"ecs_seed_positions_count": {
@@ -4653,17 +5019,13 @@
"sql": "SELECT COUNT(*) AS n FROM positions"
}
},
- "schema_parity": {
+ "ecs_step_next_positions_count": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-020-ecs-golden",
- "llm_db": "schema-t-020-ecs-claude-4-5-sonnet-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
+ "actual": 2,
+ "expected": 2,
+ "sql": "SELECT COUNT(*) AS n FROM next_positions"
}
},
"ecs_next_pos_entity2": {
@@ -4680,51 +5042,63 @@
"started_at": "2025-10-20T20:32:51.324485600Z",
"finished_at": "2025-10-20T20:34:10.701136300Z"
},
- "t_015_product_type_columns": {
+ "t_021_multi_column_index": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_015_product_type_columns",
+ "task": "t_021_multi_column_index",
"lang": "csharp",
"golden_published": true,
"model_name": "Claude 4.5 Sonnet",
- "total_tests": 3,
- "passed_tests": 3,
+ "total_tests": 4,
+ "passed_tests": 0,
"llm_output": null,
"category": "schema",
"route_api_model": "claude-sonnet-4-5",
- "golden_db": "schema-t-015-product-type-columns-golden",
- "llm_db": "schema-t-015-product-type-columns-claude-4-5-sonnet-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\claude-4-5-sonnet\\llm",
+ "golden_db": "schema-t-021-multi-column-index-golden",
+ "llm_db": "schema-t-021-multi-column-index-claude-4-5-sonnet-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\claude-4-5-sonnet\\llm",
"scorer_details": {
- "product_type_columns_row_count": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM profiles WHERE Id=1"
- }
- },
- "product_type_columns_row_parity": {
- "pass": true,
- "partial": 1.0,
+ "publish_error": {
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "args": [],
- "golden_db": "schema-t-015-product-type-columns-golden",
- "golden_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)",
- "llm_db": "schema-t-015-product-type-columns-claude-4-5-sonnet-llm",
- "llm_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)",
- "query": "SELECT Id, Home, Work, Pos FROM profiles WHERE Id=1",
- "reducer": "Seed",
- "server": "local"
+ "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\claude-4-5-sonnet\\llm\\Lib.cs(7,6): error STDB0004: Index attribute doesn't specify columns. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\claude-4-5-sonnet\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\claude-4-5-sonnet\\llm\\Lib.cs(7,52): error CS0246: The type or namespace name 'BTree' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\claude-4-5-sonnet\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\claude-4-5-sonnet\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'logs' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\claude-4-5-sonnet\\llm\\StdbModule.csproj]\r\n",
+ "phase": "build_or_publish"
}
- },
+ }
+ },
+ "vendor": "anthropic",
+ "started_at": "2025-10-20T20:32:52.080580500Z",
+ "finished_at": "2025-10-20T20:33:04.428357700Z"
+ }
+ }
+ },
+ {
+ "name": "Claude 4 Sonnet",
+ "route_api_model": "claude-sonnet-4",
+ "tasks": {
+ "t_000_empty_reducers": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_000_empty_reducers",
+ "lang": "csharp",
+ "golden_published": true,
+ "model_name": "Claude 4 Sonnet",
+ "total_tests": 1,
+ "passed_tests": 1,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Reducer]\n public static void EmptyReducer_NoArgs(ReducerContext ctx)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithInt(ReducerContext ctx, int count)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithString(ReducerContext ctx, string name)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithTwoArgs(ReducerContext ctx, int count, string name)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithThreeArgs(ReducerContext ctx, bool active, float ratio, string label)\n {\n }\n}",
+ "category": "basics",
+ "route_api_model": "claude-sonnet-4",
+ "golden_db": "basics-t-000-empty-reducers-golden",
+ "llm_db": "basics-t-000-empty-reducers-claude-4-sonnet-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\claude-4-sonnet\\llm",
+ "scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-015-product-type-columns-golden",
- "llm_db": "schema-t-015-product-type-columns-claude-4-5-sonnet-llm",
+ "golden_db": "basics-t-000-empty-reducers-golden",
+ "llm_db": "basics-t-000-empty-reducers-claude-4-sonnet-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -4734,210 +5108,99 @@
}
},
"vendor": "anthropic",
- "started_at": "2025-10-20T20:32:47.433184500Z",
- "finished_at": "2025-10-20T20:34:06.772724400Z"
+ "started_at": "2025-10-22T00:05:14.429525200Z",
+ "finished_at": "2025-10-22T00:05:59.968429900Z"
},
- "t_011_helper_function": {
+ "t_001_basic_tables": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_011_helper_function",
+ "task": "t_001_basic_tables",
"lang": "csharp",
"golden_published": true,
- "model_name": "Claude 4.5 Sonnet",
- "total_tests": 3,
- "passed_tests": 3,
+ "model_name": "Claude 4 Sonnet",
+ "total_tests": 1,
+ "passed_tests": 1,
"llm_output": null,
"category": "basics",
- "route_api_model": "claude-sonnet-4-5",
- "golden_db": "basics-t-011-helper-function-golden",
- "llm_db": "basics-t-011-helper-function-claude-4-5-sonnet-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\claude-4-5-sonnet\\llm",
+ "route_api_model": "claude-sonnet-4",
+ "golden_db": "basics-t-001-basic-tables-golden",
+ "llm_db": "basics-t-001-basic-tables-claude-4-sonnet-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\claude-4-sonnet\\llm",
"scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-011-helper-function-golden",
- "llm_db": "basics-t-011-helper-function-claude-4-5-sonnet-llm",
+ "golden_db": "basics-t-001-basic-tables-golden",
+ "llm_db": "basics-t-001-basic-tables-claude-4-sonnet-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
"tables_diff": null,
"tables_equal": true
}
- },
- "helper_func_sum_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "args": [
- 1,
- 2,
- 3
- ],
- "golden_db": "basics-t-011-helper-function-golden",
- "golden_out": "Id | Sum ----+----- 1 | 5",
- "llm_db": "basics-t-011-helper-function-claude-4-5-sonnet-llm",
- "llm_out": "Id | Sum ----+----- 1 | 5",
- "query": "SELECT Id, Sum FROM results WHERE Id=1",
- "reducer": "ComputeSum",
- "server": "local"
- }
- },
- "helper_func_sum_abs": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1 AND Sum=5"
- }
- }
- },
- "vendor": "anthropic",
- "started_at": "2025-10-20T20:02:10.006523700Z",
- "finished_at": "2025-10-20T20:04:19.536787900Z"
- },
- "t_021_multi_column_index": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_021_multi_column_index",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "Claude 4.5 Sonnet",
- "total_tests": 4,
- "passed_tests": 0,
- "llm_output": null,
- "category": "schema",
- "route_api_model": "claude-sonnet-4-5",
- "golden_db": "schema-t-021-multi-column-index-golden",
- "llm_db": "schema-t-021-multi-column-index-claude-4-5-sonnet-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\claude-4-5-sonnet\\llm",
- "scorer_details": {
- "publish_error": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\claude-4-5-sonnet\\llm\\Lib.cs(7,6): error STDB0004: Index attribute doesn't specify columns. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\claude-4-5-sonnet\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\claude-4-5-sonnet\\llm\\Lib.cs(7,52): error CS0246: The type or namespace name 'BTree' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\claude-4-5-sonnet\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\claude-4-5-sonnet\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'logs' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\claude-4-5-sonnet\\llm\\StdbModule.csproj]\r\n",
- "phase": "build_or_publish"
- }
}
},
"vendor": "anthropic",
- "started_at": "2025-10-20T20:32:52.080580500Z",
- "finished_at": "2025-10-20T20:33:04.428357700Z"
+ "started_at": "2025-10-20T20:37:00.924706700Z",
+ "finished_at": "2025-10-20T20:39:19.137937700Z"
},
- "t_017_scheduled_columns": {
+ "t_002_scheduled_table": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_017_scheduled_columns",
+ "task": "t_002_scheduled_table",
"lang": "csharp",
"golden_published": true,
- "model_name": "Claude 4.5 Sonnet",
- "total_tests": 2,
- "passed_tests": 2,
+ "model_name": "Claude 4 Sonnet",
+ "total_tests": 1,
+ "passed_tests": 1,
"llm_output": null,
- "category": "schema",
- "route_api_model": "claude-sonnet-4-5",
- "golden_db": "schema-t-017-scheduled-columns-golden",
- "llm_db": "schema-t-017-scheduled-columns-claude-4-5-sonnet-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_017_scheduled_columns\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_017_scheduled_columns\\csharp\\server\\claude-4-5-sonnet\\llm",
+ "category": "basics",
+ "route_api_model": "claude-sonnet-4",
+ "golden_db": "basics-t-002-scheduled-table-golden",
+ "llm_db": "basics-t-002-scheduled-table-claude-4-sonnet-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_002_scheduled_table\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_002_scheduled_table\\csharp\\server\\claude-4-sonnet\\llm",
"scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-017-scheduled-columns-golden",
- "llm_db": "schema-t-017-scheduled-columns-claude-4-5-sonnet-llm",
+ "golden_db": "basics-t-002-scheduled-table-golden",
+ "llm_db": "basics-t-002-scheduled-table-claude-4-sonnet-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
"tables_diff": null,
"tables_equal": true
}
- },
- "scheduled_seeded_one_row": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM tick_timer WHERE ScheduledId>=0"
- }
- }
- },
- "vendor": "anthropic",
- "started_at": "2025-10-20T20:32:49.000529800Z",
- "finished_at": "2025-10-20T20:34:07.005785Z"
- },
- "t_013_spacetime_sum_type": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_013_spacetime_sum_type",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "Claude 4.5 Sonnet",
- "total_tests": 3,
- "passed_tests": 0,
- "llm_output": null,
- "category": "schema",
- "route_api_model": "claude-sonnet-4-5",
- "golden_db": "schema-t-013-spacetime-sum-type-golden",
- "llm_db": "schema-t-013-spacetime-sum-type-claude-4-5-sonnet-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\claude-4-5-sonnet\\llm",
- "scorer_details": {
- "publish_error": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\claude-4-5-sonnet\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.BSATN.Codegen\\SpacetimeDB.Codegen.Type\\Shape.cs(6,26): error CS8910: The primary constructor conflicts with the synthesized copy constructor. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\claude-4-5-sonnet\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\claude-4-5-sonnet\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.BSATN.Codegen\\SpacetimeDB.Codegen.Type\\Shape.cs(12,26): error CS8910: The primary constructor conflicts with the synthesized copy constructor. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\claude-4-5-sonnet\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\claude-4-5-sonnet\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'results' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\claude-4-5-sonnet\\llm\\StdbModule.csproj]\r\n",
- "phase": "build_or_publish"
- }
}
},
"vendor": "anthropic",
- "started_at": "2025-10-20T20:32:45.889461900Z",
- "finished_at": "2025-10-20T20:33:04.383640200Z"
+ "started_at": "2025-10-20T20:37:01.709500800Z",
+ "finished_at": "2025-10-20T20:38:41.001791800Z"
},
- "t_018_constraints": {
+ "t_003_struct_in_table": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_018_constraints",
+ "task": "t_003_struct_in_table",
"lang": "csharp",
"golden_published": true,
- "model_name": "Claude 4.5 Sonnet",
- "total_tests": 3,
- "passed_tests": 2,
+ "model_name": "Claude 4 Sonnet",
+ "total_tests": 1,
+ "passed_tests": 1,
"llm_output": null,
- "category": "schema",
- "route_api_model": "claude-sonnet-4-5",
- "golden_db": "schema-t-018-constraints-golden",
- "llm_db": "schema-t-018-constraints-claude-4-5-sonnet-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\claude-4-5-sonnet\\llm",
+ "category": "basics",
+ "route_api_model": "claude-sonnet-4",
+ "golden_db": "basics-t-003-struct-in-table-golden",
+ "llm_db": "basics-t-003-struct-in-table-claude-4-sonnet-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_003_struct_in_table\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_003_struct_in_table\\csharp\\server\\claude-4-sonnet\\llm",
"scorer_details": {
- "constraints_row_parity_after_seed": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: can't directly call special Init lifecycle reducer\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:3000/v1/database/c200988c7e20b2058d3109402652f801c7191877ccc7e94f877c525721feaab5/call/Seed)\n",
- "phase": "call_reducer_llm"
- }
- },
- "constraints_seed_two_rows": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM accounts WHERE Id=2"
- }
- },
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-018-constraints-golden",
- "llm_db": "schema-t-018-constraints-claude-4-5-sonnet-llm",
+ "golden_db": "basics-t-003-struct-in-table-golden",
+ "llm_db": "basics-t-003-struct-in-table-claude-4-sonnet-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -4947,38 +5210,25 @@
}
},
"vendor": "anthropic",
- "started_at": "2025-10-20T20:32:49.781739300Z",
- "finished_at": "2025-10-20T20:34:07.384235700Z"
+ "started_at": "2025-10-20T20:37:02.482414500Z",
+ "finished_at": "2025-10-20T20:38:42.440390900Z"
},
"t_004_insert": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
"task": "t_004_insert",
"lang": "csharp",
"golden_published": true,
- "model_name": "Claude 4.5 Sonnet",
+ "model_name": "Claude 4 Sonnet",
"total_tests": 2,
"passed_tests": 2,
"llm_output": null,
"category": "basics",
- "route_api_model": "claude-sonnet-4-5",
+ "route_api_model": "claude-sonnet-4",
"golden_db": "basics-t-004-insert-golden",
- "llm_db": "basics-t-004-insert-claude-4-5-sonnet-llm",
+ "llm_db": "basics-t-004-insert-claude-4-sonnet-llm",
"work_dir_golden": "target\\llm-runs\\basics\\t_004_insert\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_004_insert\\csharp\\server\\claude-4-5-sonnet\\llm",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_004_insert\\csharp\\server\\claude-4-sonnet\\llm",
"scorer_details": {
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "golden_db": "basics-t-004-insert-golden",
- "llm_db": "basics-t-004-insert-claude-4-5-sonnet-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
- }
- },
"data_parity_insert_user": {
"pass": true,
"partial": 1.0,
@@ -4991,40 +5241,19 @@
],
"golden_db": "basics-t-004-insert-golden",
"golden_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true",
- "llm_db": "basics-t-004-insert-claude-4-5-sonnet-llm",
+ "llm_db": "basics-t-004-insert-claude-4-sonnet-llm",
"llm_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true",
"query": "SELECT Id, Name, Age, Active FROM users WHERE Id=1",
"reducer": "InsertUser",
"server": "local"
}
- }
- },
- "vendor": "anthropic",
- "started_at": "2025-10-20T20:02:04.535678400Z",
- "finished_at": "2025-10-20T20:04:43.019219800Z"
- },
- "t_002_scheduled_table": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_002_scheduled_table",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "Claude 4.5 Sonnet",
- "total_tests": 1,
- "passed_tests": 1,
- "llm_output": null,
- "category": "basics",
- "route_api_model": "claude-sonnet-4-5",
- "golden_db": "basics-t-002-scheduled-table-golden",
- "llm_db": "basics-t-002-scheduled-table-claude-4-5-sonnet-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_002_scheduled_table\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_002_scheduled_table\\csharp\\server\\claude-4-5-sonnet\\llm",
- "scorer_details": {
+ },
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-002-scheduled-table-golden",
- "llm_db": "basics-t-002-scheduled-table-claude-4-5-sonnet-llm",
+ "golden_db": "basics-t-004-insert-golden",
+ "llm_db": "basics-t-004-insert-claude-4-sonnet-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -5034,79 +5263,41 @@
}
},
"vendor": "anthropic",
- "started_at": "2025-10-20T20:02:02.870487100Z",
- "finished_at": "2025-10-20T20:03:40.137742600Z"
+ "started_at": "2025-10-20T20:37:03.262533900Z",
+ "finished_at": "2025-10-20T20:38:41.453040200Z"
},
- "t_001_basic_tables": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_001_basic_tables",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "Claude 4.5 Sonnet",
- "total_tests": 1,
- "passed_tests": 1,
- "llm_output": null,
- "category": "basics",
- "route_api_model": "claude-sonnet-4-5",
- "golden_db": "basics-t-001-basic-tables-golden",
- "llm_db": "basics-t-001-basic-tables-claude-4-5-sonnet-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\claude-4-5-sonnet\\llm",
- "scorer_details": {
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "golden_db": "basics-t-001-basic-tables-golden",
- "llm_db": "basics-t-001-basic-tables-claude-4-5-sonnet-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
- }
- }
- },
- "vendor": "anthropic",
- "started_at": "2025-10-20T20:02:02.003779500Z",
- "finished_at": "2025-10-20T20:03:40.332710900Z"
- }
- }
- },
- {
- "name": "Claude 4 Sonnet",
- "route_api_model": "claude-sonnet-4",
- "tasks": {
- "t_013_spacetime_sum_type": {
+ "t_005_update": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_013_spacetime_sum_type",
+ "task": "t_005_update",
"lang": "csharp",
"golden_published": true,
"model_name": "Claude 4 Sonnet",
"total_tests": 3,
"passed_tests": 3,
"llm_output": null,
- "category": "schema",
+ "category": "basics",
"route_api_model": "claude-sonnet-4",
- "golden_db": "schema-t-013-spacetime-sum-type-golden",
- "llm_db": "schema-t-013-spacetime-sum-type-claude-4-sonnet-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\claude-4-sonnet\\llm",
+ "golden_db": "basics-t-005-update-golden",
+ "llm_db": "basics-t-005-update-claude-4-sonnet-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_005_update\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_005_update\\csharp\\server\\claude-4-sonnet\\llm",
"scorer_details": {
- "sum_type_row_parity": {
+ "data_parity_update_user": {
"pass": true,
"partial": 1.0,
"notes": {
"args": [
1,
- 10
+ "Alice2",
+ 31,
+ false
],
- "golden_db": "schema-t-013-spacetime-sum-type-golden",
- "golden_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))",
- "llm_db": "schema-t-013-spacetime-sum-type-claude-4-sonnet-llm",
- "llm_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))",
- "query": "SELECT Id, Value FROM results WHERE Id=1",
- "reducer": "SetCircle",
+ "golden_db": "basics-t-005-update-golden",
+ "golden_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
+ "llm_db": "basics-t-005-update-claude-4-sonnet-llm",
+ "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
+ "query": "SELECT Id, Name, Age, Active FROM users WHERE Id=1",
+ "reducer": "UpdateUser",
"server": "local"
}
},
@@ -5114,8 +5305,8 @@
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-013-spacetime-sum-type-golden",
- "llm_db": "schema-t-013-spacetime-sum-type-claude-4-sonnet-llm",
+ "golden_db": "basics-t-005-update-golden",
+ "llm_db": "basics-t-005-update-claude-4-sonnet-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -5123,91 +5314,137 @@
"tables_equal": true
}
},
- "sum_type_row_count": {
+ "seed_users_row": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1"
+ "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
}
}
},
"vendor": "anthropic",
- "started_at": "2025-10-20T20:41:36.683550700Z",
- "finished_at": "2025-10-20T20:43:44.763925200Z"
+ "started_at": "2025-10-20T20:37:04.018006400Z",
+ "finished_at": "2025-10-20T20:39:19.762751800Z"
},
- "t_003_struct_in_table": {
+ "t_006_delete": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_003_struct_in_table",
+ "task": "t_006_delete",
"lang": "csharp",
"golden_published": true,
"model_name": "Claude 4 Sonnet",
- "total_tests": 1,
- "passed_tests": 1,
+ "total_tests": 3,
+ "passed_tests": 3,
"llm_output": null,
"category": "basics",
"route_api_model": "claude-sonnet-4",
- "golden_db": "basics-t-003-struct-in-table-golden",
- "llm_db": "basics-t-003-struct-in-table-claude-4-sonnet-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_003_struct_in_table\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_003_struct_in_table\\csharp\\server\\claude-4-sonnet\\llm",
+ "golden_db": "basics-t-006-delete-golden",
+ "llm_db": "basics-t-006-delete-claude-4-sonnet-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\claude-4-sonnet\\llm",
"scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-003-struct-in-table-golden",
- "llm_db": "basics-t-003-struct-in-table-claude-4-sonnet-llm",
+ "golden_db": "basics-t-006-delete-golden",
+ "llm_db": "basics-t-006-delete-claude-4-sonnet-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
"tables_diff": null,
"tables_equal": true
}
+ },
+ "delete_user_count_zero": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 0,
+ "expected": 0,
+ "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=1"
+ }
+ },
+ "seed_users_row": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
+ }
}
},
"vendor": "anthropic",
- "started_at": "2025-10-20T20:37:02.482414500Z",
- "finished_at": "2025-10-20T20:38:42.440390900Z"
+ "started_at": "2025-10-20T20:37:04.793257600Z",
+ "finished_at": "2025-10-20T20:38:40.837307900Z"
},
- "t_000_empty_reducers": {
+ "t_007_crud": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_000_empty_reducers",
+ "task": "t_007_crud",
"lang": "csharp",
"golden_published": true,
"model_name": "Claude 4 Sonnet",
- "total_tests": 1,
- "passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Reducer]\n public static void EmptyReducer_NoArgs(ReducerContext ctx)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithInt(ReducerContext ctx, int count)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithString(ReducerContext ctx, string name)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithTwoArgs(ReducerContext ctx, int count, string name)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithThreeArgs(ReducerContext ctx, bool active, float ratio, string label)\n {\n }\n}",
+ "total_tests": 4,
+ "passed_tests": 4,
+ "llm_output": null,
"category": "basics",
"route_api_model": "claude-sonnet-4",
- "golden_db": "basics-t-000-empty-reducers-golden",
- "llm_db": "basics-t-000-empty-reducers-claude-4-sonnet-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\claude-4-sonnet\\llm",
+ "golden_db": "basics-t-007-crud-golden",
+ "llm_db": "basics-t-007-crud-claude-4-sonnet-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\claude-4-sonnet\\llm",
"scorer_details": {
+ "crud_total_count_one": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM users"
+ }
+ },
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-000-empty-reducers-golden",
- "llm_db": "basics-t-000-empty-reducers-claude-4-sonnet-llm",
+ "golden_db": "basics-t-007-crud-golden",
+ "llm_db": "basics-t-007-crud-claude-4-sonnet-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
"tables_diff": null,
"tables_equal": true
}
+ },
+ "crud_row_id1_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "args": [],
+ "golden_db": "basics-t-007-crud-golden",
+ "golden_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
+ "llm_db": "basics-t-007-crud-claude-4-sonnet-llm",
+ "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
+ "query": "SELECT Id, Name, Age, Active FROM users WHERE Id=1",
+ "reducer": "Crud",
+ "server": "local"
+ }
+ },
+ "crud_row_id2_deleted": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 0,
+ "expected": 0,
+ "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=2"
+ }
}
},
"vendor": "anthropic",
- "started_at": "2025-10-22T00:05:14.429525200Z",
- "finished_at": "2025-10-22T00:05:59.968429900Z"
+ "started_at": "2025-10-20T20:37:05.570212Z",
+ "finished_at": "2025-10-20T20:38:40.427331500Z"
},
- "t_011_helper_function": {
+ "t_008_index_lookup": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_011_helper_function",
+ "task": "t_008_index_lookup",
"lang": "csharp",
"golden_published": true,
"model_name": "Claude 4 Sonnet",
@@ -5216,55 +5453,112 @@
"llm_output": null,
"category": "basics",
"route_api_model": "claude-sonnet-4",
- "golden_db": "basics-t-011-helper-function-golden",
- "llm_db": "basics-t-011-helper-function-claude-4-sonnet-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\claude-4-sonnet\\llm",
+ "golden_db": "basics-t-008-index-lookup-golden",
+ "llm_db": "basics-t-008-index-lookup-claude-4-sonnet-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\claude-4-sonnet\\llm",
"scorer_details": {
- "helper_func_sum_parity": {
+ "schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "args": [
- 1,
- 2,
- 3
+ "golden_db": "basics-t-008-index-lookup-golden",
+ "llm_db": "basics-t-008-index-lookup-claude-4-sonnet-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
+ }
+ },
+ "seed_user_row": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
+ }
+ },
+ "index_lookup_projection_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "args": [
+ 1
],
- "golden_db": "basics-t-011-helper-function-golden",
- "golden_out": "Id | Sum ----+----- 1 | 5",
- "llm_db": "basics-t-011-helper-function-claude-4-sonnet-llm",
- "llm_out": "Id | Sum ----+----- 1 | 5",
- "query": "SELECT Id, Sum FROM results WHERE Id=1",
- "reducer": "ComputeSum",
+ "golden_db": "basics-t-008-index-lookup-golden",
+ "golden_out": "Id | Name ----+--------- 1 | \"Alice\"",
+ "llm_db": "basics-t-008-index-lookup-claude-4-sonnet-llm",
+ "llm_out": "Id | Name ----+--------- 1 | \"Alice\"",
+ "query": "SELECT Id, Name FROM results WHERE Id=1",
+ "reducer": "LookupUserName",
"server": "local"
}
+ }
+ },
+ "vendor": "anthropic",
+ "started_at": "2025-10-20T20:37:06.488290400Z",
+ "finished_at": "2025-10-20T20:38:38.579135Z"
+ },
+ "t_009_init": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_009_init",
+ "lang": "csharp",
+ "golden_published": true,
+ "model_name": "Claude 4 Sonnet",
+ "total_tests": 4,
+ "passed_tests": 4,
+ "llm_output": null,
+ "category": "basics",
+ "route_api_model": "claude-sonnet-4",
+ "golden_db": "basics-t-009-init-golden",
+ "llm_db": "basics-t-009-init-claude-4-sonnet-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_009_init\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_009_init\\csharp\\server\\claude-4-sonnet\\llm",
+ "scorer_details": {
+ "init_total_two": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 2,
+ "expected": 2,
+ "sql": "SELECT COUNT(*) AS n FROM users"
+ }
},
- "schema_parity": {
+ "init_seed_bob": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-011-helper-function-golden",
- "llm_db": "basics-t-011-helper-function-claude-4-sonnet-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false"
}
},
- "helper_func_sum_abs": {
+ "init_seed_alice": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1 AND Sum=5"
+ "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true"
+ }
+ },
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "golden_db": "basics-t-009-init-golden",
+ "llm_db": "basics-t-009-init-claude-4-sonnet-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
}
}
},
"vendor": "anthropic",
- "started_at": "2025-10-20T20:37:08.816503300Z",
- "finished_at": "2025-10-20T20:38:42.043948600Z"
+ "started_at": "2025-10-20T20:37:07.325249700Z",
+ "finished_at": "2025-10-20T20:39:38.387523900Z"
},
"t_010_connect": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
@@ -5300,195 +5594,276 @@
"started_at": "2025-10-20T20:37:08.070334900Z",
"finished_at": "2025-10-20T20:38:42.209936800Z"
},
- "t_020_ecs": {
+ "t_011_helper_function": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_020_ecs",
+ "task": "t_011_helper_function",
"lang": "csharp",
"golden_published": true,
"model_name": "Claude 4 Sonnet",
- "total_tests": 5,
- "passed_tests": 5,
+ "total_tests": 3,
+ "passed_tests": 3,
"llm_output": null,
- "category": "schema",
+ "category": "basics",
"route_api_model": "claude-sonnet-4",
- "golden_db": "schema-t-020-ecs-golden",
- "llm_db": "schema-t-020-ecs-claude-4-sonnet-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\claude-4-sonnet\\llm",
+ "golden_db": "basics-t-011-helper-function-golden",
+ "llm_db": "basics-t-011-helper-function-claude-4-sonnet-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\claude-4-sonnet\\llm",
"scorer_details": {
- "ecs_seed_positions_count": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 2,
- "expected": 2,
- "sql": "SELECT COUNT(*) AS n FROM positions"
- }
- },
- "ecs_step_next_positions_count": {
+ "helper_func_sum_abs": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 2,
- "expected": 2,
- "sql": "SELECT COUNT(*) AS n FROM next_positions"
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1 AND Sum=5"
}
},
- "ecs_next_pos_entity2": {
+ "helper_func_sum_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM next_positions WHERE EntityId=2 AND X=8 AND Y=3"
+ "args": [
+ 1,
+ 2,
+ 3
+ ],
+ "golden_db": "basics-t-011-helper-function-golden",
+ "golden_out": "Id | Sum ----+----- 1 | 5",
+ "llm_db": "basics-t-011-helper-function-claude-4-sonnet-llm",
+ "llm_out": "Id | Sum ----+----- 1 | 5",
+ "query": "SELECT Id, Sum FROM results WHERE Id=1",
+ "reducer": "ComputeSum",
+ "server": "local"
}
},
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-020-ecs-golden",
- "llm_db": "schema-t-020-ecs-claude-4-sonnet-llm",
+ "golden_db": "basics-t-011-helper-function-golden",
+ "llm_db": "basics-t-011-helper-function-claude-4-sonnet-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
"tables_diff": null,
"tables_equal": true
}
- },
- "ecs_next_pos_entity1": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM next_positions WHERE EntityId=1 AND X=1 AND Y=0"
- }
}
},
"vendor": "anthropic",
- "started_at": "2025-10-20T20:41:43.118766400Z",
- "finished_at": "2025-10-20T20:43:00.563259600Z"
+ "started_at": "2025-10-20T20:37:08.816503300Z",
+ "finished_at": "2025-10-20T20:38:42.043948600Z"
},
- "t_009_init": {
+ "t_012_spacetime_product_type": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_009_init",
+ "task": "t_012_spacetime_product_type",
"lang": "csharp",
"golden_published": true,
"model_name": "Claude 4 Sonnet",
- "total_tests": 4,
- "passed_tests": 4,
+ "total_tests": 3,
+ "passed_tests": 3,
"llm_output": null,
- "category": "basics",
+ "category": "schema",
"route_api_model": "claude-sonnet-4",
- "golden_db": "basics-t-009-init-golden",
- "llm_db": "basics-t-009-init-claude-4-sonnet-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_009_init\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_009_init\\csharp\\server\\claude-4-sonnet\\llm",
+ "golden_db": "schema-t-012-spacetime-product-type-golden",
+ "llm_db": "schema-t-012-spacetime-product-type-claude-4-sonnet-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\claude-4-sonnet\\llm",
"scorer_details": {
- "init_seed_alice": {
+ "product_type_row_count": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true"
+ "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1"
+ }
+ },
+ "product_type_row_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "args": [
+ 1,
+ 2,
+ 3
+ ],
+ "golden_db": "schema-t-012-spacetime-product-type-golden",
+ "golden_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)",
+ "llm_db": "schema-t-012-spacetime-product-type-claude-4-sonnet-llm",
+ "llm_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)",
+ "query": "SELECT Id, Value FROM results WHERE Id=1",
+ "reducer": "SetScore",
+ "server": "local"
}
},
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-009-init-golden",
- "llm_db": "basics-t-009-init-claude-4-sonnet-llm",
+ "golden_db": "schema-t-012-spacetime-product-type-golden",
+ "llm_db": "schema-t-012-spacetime-product-type-claude-4-sonnet-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
"tables_diff": null,
"tables_equal": true
}
+ }
+ },
+ "vendor": "anthropic",
+ "started_at": "2025-10-20T20:41:35.815483900Z",
+ "finished_at": "2025-10-20T20:42:59.202139300Z"
+ },
+ "t_013_spacetime_sum_type": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_013_spacetime_sum_type",
+ "lang": "csharp",
+ "golden_published": true,
+ "model_name": "Claude 4 Sonnet",
+ "total_tests": 3,
+ "passed_tests": 3,
+ "llm_output": null,
+ "category": "schema",
+ "route_api_model": "claude-sonnet-4",
+ "golden_db": "schema-t-013-spacetime-sum-type-golden",
+ "llm_db": "schema-t-013-spacetime-sum-type-claude-4-sonnet-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\claude-4-sonnet\\llm",
+ "scorer_details": {
+ "sum_type_row_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "args": [
+ 1,
+ 10
+ ],
+ "golden_db": "schema-t-013-spacetime-sum-type-golden",
+ "golden_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))",
+ "llm_db": "schema-t-013-spacetime-sum-type-claude-4-sonnet-llm",
+ "llm_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))",
+ "query": "SELECT Id, Value FROM results WHERE Id=1",
+ "reducer": "SetCircle",
+ "server": "local"
+ }
},
- "init_total_two": {
+ "schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 2,
- "expected": 2,
- "sql": "SELECT COUNT(*) AS n FROM users"
+ "golden_db": "schema-t-013-spacetime-sum-type-golden",
+ "llm_db": "schema-t-013-spacetime-sum-type-claude-4-sonnet-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
}
},
- "init_seed_bob": {
+ "sum_type_row_count": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false"
+ "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1"
}
}
},
"vendor": "anthropic",
- "started_at": "2025-10-20T20:37:07.325249700Z",
- "finished_at": "2025-10-20T20:39:38.387523900Z"
+ "started_at": "2025-10-20T20:41:36.683550700Z",
+ "finished_at": "2025-10-20T20:43:44.763925200Z"
},
- "t_019_many_to_many": {
+ "t_014_elementary_columns": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_019_many_to_many",
+ "task": "t_014_elementary_columns",
"lang": "csharp",
"golden_published": true,
"model_name": "Claude 4 Sonnet",
- "total_tests": 5,
- "passed_tests": 0,
+ "total_tests": 3,
+ "passed_tests": 3,
"llm_output": null,
"category": "schema",
"route_api_model": "claude-sonnet-4",
- "golden_db": "schema-t-019-many-to-many-golden",
- "llm_db": "schema-t-019-many-to-many-claude-4-sonnet-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\claude-4-sonnet\\llm",
+ "golden_db": "schema-t-014-elementary-columns-golden",
+ "llm_db": "schema-t-014-elementary-columns-claude-4-sonnet-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_014_elementary_columns\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_014_elementary_columns\\csharp\\server\\claude-4-sonnet\\llm",
"scorer_details": {
- "publish_error": {
- "pass": false,
- "partial": 0.0,
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
"notes": {
- "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\claude-4-sonnet\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(113,24): warning CS8981: The type name 'users' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\claude-4-sonnet\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\claude-4-sonnet\\llm\\Lib.cs(23,6): error CS0104: 'Index' is an ambiguous reference between 'SpacetimeDB.Index' and 'System.Index' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\claude-4-sonnet\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\claude-4-sonnet\\llm\\Lib.cs(24,6): error CS0104: 'Index' is an ambiguous reference between 'SpacetimeDB.Index' and 'System.Index' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\claude-4-sonnet\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\claude-4-sonnet\\llm\\Lib.cs(23,62): error CS9176: There is no target type for the collection expression. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\claude-4-sonnet\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\claude-4-sonnet\\llm\\Lib.cs(24,63): error CS9176: There is no target type for the collection expression. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\claude-4-sonnet\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\claude-4-sonnet\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'groups' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\claude-4-sonnet\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\claude-4-sonnet\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(70,24): warning CS8981: The type name 'memberships' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\claude-4-sonnet\\llm\\StdbModule.csproj]\r\n",
- "phase": "build_or_publish"
+ "golden_db": "schema-t-014-elementary-columns-golden",
+ "llm_db": "schema-t-014-elementary-columns-claude-4-sonnet-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
+ }
+ },
+ "elementary_columns_row_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "args": [],
+ "golden_db": "schema-t-014-elementary-columns-golden",
+ "golden_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"",
+ "llm_db": "schema-t-014-elementary-columns-claude-4-sonnet-llm",
+ "llm_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"",
+ "query": "SELECT Id, Count, Total, Price, Ratio, Active, Name FROM primitives WHERE Id=1",
+ "reducer": "Seed",
+ "server": "local"
+ }
+ },
+ "elementary_columns_row_count": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM primitives WHERE Id=1"
}
}
},
"vendor": "anthropic",
- "started_at": "2025-10-20T20:41:42.211270400Z",
- "finished_at": "2025-10-20T20:41:57.069556400Z"
+ "started_at": "2025-10-20T20:41:37.532050300Z",
+ "finished_at": "2025-10-20T20:43:45.160088600Z"
},
- "t_008_index_lookup": {
+ "t_015_product_type_columns": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_008_index_lookup",
+ "task": "t_015_product_type_columns",
"lang": "csharp",
"golden_published": true,
"model_name": "Claude 4 Sonnet",
"total_tests": 3,
"passed_tests": 3,
"llm_output": null,
- "category": "basics",
+ "category": "schema",
"route_api_model": "claude-sonnet-4",
- "golden_db": "basics-t-008-index-lookup-golden",
- "llm_db": "basics-t-008-index-lookup-claude-4-sonnet-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\claude-4-sonnet\\llm",
+ "golden_db": "schema-t-015-product-type-columns-golden",
+ "llm_db": "schema-t-015-product-type-columns-claude-4-sonnet-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\claude-4-sonnet\\llm",
"scorer_details": {
- "seed_user_row": {
+ "product_type_columns_row_count": {
"pass": true,
"partial": 1.0,
"notes": {
- "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM profiles WHERE Id=1"
}
},
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-008-index-lookup-golden",
- "llm_db": "basics-t-008-index-lookup-claude-4-sonnet-llm",
+ "golden_db": "schema-t-015-product-type-columns-golden",
+ "llm_db": "schema-t-015-product-type-columns-claude-4-sonnet-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -5496,49 +5871,70 @@
"tables_equal": true
}
},
- "index_lookup_projection_parity": {
+ "product_type_columns_row_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "args": [
- 1
- ],
- "golden_db": "basics-t-008-index-lookup-golden",
- "golden_out": "Id | Name ----+--------- 1 | \"Alice\"",
- "llm_db": "basics-t-008-index-lookup-claude-4-sonnet-llm",
- "llm_out": "Id | Name ----+--------- 1 | \"Alice\"",
- "query": "SELECT Id, Name FROM results WHERE Id=1",
- "reducer": "LookupUserName",
+ "args": [],
+ "golden_db": "schema-t-015-product-type-columns-golden",
+ "golden_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)",
+ "llm_db": "schema-t-015-product-type-columns-claude-4-sonnet-llm",
+ "llm_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)",
+ "query": "SELECT Id, Home, Work, Pos FROM profiles WHERE Id=1",
+ "reducer": "Seed",
"server": "local"
}
}
},
"vendor": "anthropic",
- "started_at": "2025-10-20T20:37:06.488290400Z",
- "finished_at": "2025-10-20T20:38:38.579135Z"
+ "started_at": "2025-10-20T20:41:38.389554300Z",
+ "finished_at": "2025-10-20T20:42:57.975866600Z"
},
- "t_002_scheduled_table": {
+ "t_016_sum_type_columns": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_002_scheduled_table",
+ "task": "t_016_sum_type_columns",
"lang": "csharp",
"golden_published": true,
"model_name": "Claude 4 Sonnet",
- "total_tests": 1,
- "passed_tests": 1,
+ "total_tests": 3,
+ "passed_tests": 3,
"llm_output": null,
- "category": "basics",
+ "category": "schema",
"route_api_model": "claude-sonnet-4",
- "golden_db": "basics-t-002-scheduled-table-golden",
- "llm_db": "basics-t-002-scheduled-table-claude-4-sonnet-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_002_scheduled_table\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_002_scheduled_table\\csharp\\server\\claude-4-sonnet\\llm",
+ "golden_db": "schema-t-016-sum-type-columns-golden",
+ "llm_db": "schema-t-016-sum-type-columns-claude-4-sonnet-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\claude-4-sonnet\\llm",
"scorer_details": {
+ "sum_type_columns_row_count": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM drawings WHERE Id=1"
+ }
+ },
+ "sum_type_columns_row_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "args": [],
+ "golden_db": "schema-t-016-sum-type-columns-golden",
+ "golden_out": "Id | A | B ----+--------------------------+--------------------------------------- 1 | (Circle = (Radius = 10)) | (Rectangle = (Width = 4, Height = 6))",
+ "llm_db": "schema-t-016-sum-type-columns-claude-4-sonnet-llm",
+ "llm_out": "Id | A | B ----+--------------------------+--------------------------------------- 1 | (Circle = (Radius = 10)) | (Rectangle = (Width = 4, Height = 6))",
+ "query": "SELECT Id, A, B FROM drawings WHERE Id=1",
+ "reducer": "Seed",
+ "server": "local"
+ }
+ },
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-002-scheduled-table-golden",
- "llm_db": "basics-t-002-scheduled-table-claude-4-sonnet-llm",
+ "golden_db": "schema-t-016-sum-type-columns-golden",
+ "llm_db": "schema-t-016-sum-type-columns-claude-4-sonnet-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -5548,42 +5944,51 @@
}
},
"vendor": "anthropic",
- "started_at": "2025-10-20T20:37:01.709500800Z",
- "finished_at": "2025-10-20T20:38:41.001791800Z"
+ "started_at": "2025-10-20T20:41:39.279440300Z",
+ "finished_at": "2025-10-20T20:42:58.500369200Z"
},
- "t_001_basic_tables": {
+ "t_017_scheduled_columns": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_001_basic_tables",
+ "task": "t_017_scheduled_columns",
"lang": "csharp",
"golden_published": true,
"model_name": "Claude 4 Sonnet",
- "total_tests": 1,
- "passed_tests": 1,
+ "total_tests": 2,
+ "passed_tests": 2,
"llm_output": null,
- "category": "basics",
+ "category": "schema",
"route_api_model": "claude-sonnet-4",
- "golden_db": "basics-t-001-basic-tables-golden",
- "llm_db": "basics-t-001-basic-tables-claude-4-sonnet-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\claude-4-sonnet\\llm",
+ "golden_db": "schema-t-017-scheduled-columns-golden",
+ "llm_db": "schema-t-017-scheduled-columns-claude-4-sonnet-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_017_scheduled_columns\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_017_scheduled_columns\\csharp\\server\\claude-4-sonnet\\llm",
"scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-001-basic-tables-golden",
- "llm_db": "basics-t-001-basic-tables-claude-4-sonnet-llm",
+ "golden_db": "schema-t-017-scheduled-columns-golden",
+ "llm_db": "schema-t-017-scheduled-columns-claude-4-sonnet-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
"tables_diff": null,
"tables_equal": true
}
+ },
+ "scheduled_seeded_one_row": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM tick_timer WHERE ScheduledId>=0"
+ }
}
},
"vendor": "anthropic",
- "started_at": "2025-10-20T20:37:00.924706700Z",
- "finished_at": "2025-10-20T20:39:19.137937700Z"
+ "started_at": "2025-10-20T20:41:40.309622600Z",
+ "finished_at": "2025-10-20T20:42:58.726183Z"
},
"t_018_constraints": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
@@ -5614,51 +6019,93 @@
"started_at": "2025-10-20T20:41:41.251734100Z",
"finished_at": "2025-10-20T20:41:55.229582400Z"
},
- "t_014_elementary_columns": {
+ "t_019_many_to_many": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_014_elementary_columns",
+ "task": "t_019_many_to_many",
"lang": "csharp",
"golden_published": true,
"model_name": "Claude 4 Sonnet",
- "total_tests": 3,
- "passed_tests": 3,
+ "total_tests": 5,
+ "passed_tests": 0,
"llm_output": null,
"category": "schema",
"route_api_model": "claude-sonnet-4",
- "golden_db": "schema-t-014-elementary-columns-golden",
- "llm_db": "schema-t-014-elementary-columns-claude-4-sonnet-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_014_elementary_columns\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_014_elementary_columns\\csharp\\server\\claude-4-sonnet\\llm",
+ "golden_db": "schema-t-019-many-to-many-golden",
+ "llm_db": "schema-t-019-many-to-many-claude-4-sonnet-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\claude-4-sonnet\\llm",
"scorer_details": {
- "elementary_columns_row_parity": {
+ "publish_error": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\claude-4-sonnet\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(113,24): warning CS8981: The type name 'users' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\claude-4-sonnet\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\claude-4-sonnet\\llm\\Lib.cs(23,6): error CS0104: 'Index' is an ambiguous reference between 'SpacetimeDB.Index' and 'System.Index' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\claude-4-sonnet\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\claude-4-sonnet\\llm\\Lib.cs(24,6): error CS0104: 'Index' is an ambiguous reference between 'SpacetimeDB.Index' and 'System.Index' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\claude-4-sonnet\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\claude-4-sonnet\\llm\\Lib.cs(23,62): error CS9176: There is no target type for the collection expression. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\claude-4-sonnet\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\claude-4-sonnet\\llm\\Lib.cs(24,63): error CS9176: There is no target type for the collection expression. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\claude-4-sonnet\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\claude-4-sonnet\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'groups' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\claude-4-sonnet\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\claude-4-sonnet\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(70,24): warning CS8981: The type name 'memberships' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\claude-4-sonnet\\llm\\StdbModule.csproj]\r\n",
+ "phase": "build_or_publish"
+ }
+ }
+ },
+ "vendor": "anthropic",
+ "started_at": "2025-10-20T20:41:42.211270400Z",
+ "finished_at": "2025-10-20T20:41:57.069556400Z"
+ },
+ "t_020_ecs": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_020_ecs",
+ "lang": "csharp",
+ "golden_published": true,
+ "model_name": "Claude 4 Sonnet",
+ "total_tests": 5,
+ "passed_tests": 5,
+ "llm_output": null,
+ "category": "schema",
+ "route_api_model": "claude-sonnet-4",
+ "golden_db": "schema-t-020-ecs-golden",
+ "llm_db": "schema-t-020-ecs-claude-4-sonnet-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\claude-4-sonnet\\llm",
+ "scorer_details": {
+ "ecs_next_pos_entity1": {
"pass": true,
"partial": 1.0,
"notes": {
- "args": [],
- "golden_db": "schema-t-014-elementary-columns-golden",
- "golden_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"",
- "llm_db": "schema-t-014-elementary-columns-claude-4-sonnet-llm",
- "llm_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"",
- "query": "SELECT Id, Count, Total, Price, Ratio, Active, Name FROM primitives WHERE Id=1",
- "reducer": "Seed",
- "server": "local"
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM next_positions WHERE EntityId=1 AND X=1 AND Y=0"
}
},
- "elementary_columns_row_count": {
+ "ecs_seed_positions_count": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 2,
+ "expected": 2,
+ "sql": "SELECT COUNT(*) AS n FROM positions"
+ }
+ },
+ "ecs_step_next_positions_count": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 2,
+ "expected": 2,
+ "sql": "SELECT COUNT(*) AS n FROM next_positions"
+ }
+ },
+ "ecs_next_pos_entity2": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM primitives WHERE Id=1"
+ "sql": "SELECT COUNT(*) AS n FROM next_positions WHERE EntityId=2 AND X=8 AND Y=3"
}
},
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-014-elementary-columns-golden",
- "llm_db": "schema-t-014-elementary-columns-claude-4-sonnet-llm",
+ "golden_db": "schema-t-020-ecs-golden",
+ "llm_db": "schema-t-020-ecs-claude-4-sonnet-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -5668,127 +6115,248 @@
}
},
"vendor": "anthropic",
- "started_at": "2025-10-20T20:41:37.532050300Z",
- "finished_at": "2025-10-20T20:43:45.160088600Z"
+ "started_at": "2025-10-20T20:41:43.118766400Z",
+ "finished_at": "2025-10-20T20:43:00.563259600Z"
},
- "t_004_insert": {
+ "t_021_multi_column_index": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_004_insert",
+ "task": "t_021_multi_column_index",
"lang": "csharp",
"golden_published": true,
"model_name": "Claude 4 Sonnet",
- "total_tests": 2,
- "passed_tests": 2,
+ "total_tests": 4,
+ "passed_tests": 0,
"llm_output": null,
- "category": "basics",
+ "category": "schema",
"route_api_model": "claude-sonnet-4",
- "golden_db": "basics-t-004-insert-golden",
- "llm_db": "basics-t-004-insert-claude-4-sonnet-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_004_insert\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_004_insert\\csharp\\server\\claude-4-sonnet\\llm",
+ "golden_db": "schema-t-021-multi-column-index-golden",
+ "llm_db": "schema-t-021-multi-column-index-claude-4-sonnet-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\claude-4-sonnet\\llm",
+ "scorer_details": {
+ "publish_error": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\claude-4-sonnet\\llm\\Lib.cs(7,6): error CS0104: 'Index' is an ambiguous reference between 'SpacetimeDB.Index' and 'System.Index' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\claude-4-sonnet\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\claude-4-sonnet\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'logs' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\claude-4-sonnet\\llm\\StdbModule.csproj]\r\n",
+ "phase": "build_or_publish"
+ }
+ }
+ },
+ "vendor": "anthropic",
+ "started_at": "2025-10-20T20:41:43.928783Z",
+ "finished_at": "2025-10-20T20:41:55.279653800Z"
+ }
+ }
+ },
+ {
+ "name": "Grok 4",
+ "route_api_model": "grok-4",
+ "tasks": {
+ "t_000_empty_reducers": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_000_empty_reducers",
+ "lang": "csharp",
+ "golden_published": true,
+ "model_name": "Grok 4",
+ "total_tests": 1,
+ "passed_tests": 1,
+ "llm_output": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [SpacetimeDB.Reducer]\r\n public static void EmptyReducer_NoArgs(ReducerContext ctx)\r\n {\r\n }\r\n\r\n [SpacetimeDB.Reducer]\r\n public static void EmptyReducer_WithInt(ReducerContext ctx, int count)\r\n {\r\n }\r\n\r\n [SpacetimeDB.Reducer]\r\n public static void EmptyReducer_WithString(ReducerContext ctx, string name)\r\n {\r\n }\r\n\r\n [SpacetimeDB.Reducer]\r\n public static void EmptyReducer_WithTwoArgs(ReducerContext ctx, int count, string name)\r\n {\r\n }\r\n\r\n [SpacetimeDB.Reducer]\r\n public static void EmptyReducer_WithThreeArgs(ReducerContext ctx, bool active, float ratio, string label)\r\n {\r\n }\r\n}",
+ "category": "basics",
+ "route_api_model": "grok-4",
+ "golden_db": "basics-t-000-empty-reducers-golden",
+ "llm_db": "basics-t-000-empty-reducers-grok-4-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-4\\llm",
"scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-004-insert-golden",
- "llm_db": "basics-t-004-insert-claude-4-sonnet-llm",
+ "golden_db": "basics-t-000-empty-reducers-golden",
+ "llm_db": "basics-t-000-empty-reducers-grok-4-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
"tables_diff": null,
"tables_equal": true
}
- },
- "data_parity_insert_user": {
+ }
+ },
+ "vendor": "xai",
+ "started_at": "2025-10-21T23:58:47.413051600Z",
+ "finished_at": "2025-10-22T00:00:10.958973700Z"
+ },
+ "t_001_basic_tables": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_001_basic_tables",
+ "lang": "csharp",
+ "golden_published": true,
+ "model_name": "Grok 4",
+ "total_tests": 1,
+ "passed_tests": 1,
+ "llm_output": null,
+ "category": "basics",
+ "route_api_model": "grok-4",
+ "golden_db": "basics-t-001-basic-tables-golden",
+ "llm_db": "basics-t-001-basic-tables-grok-4-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\grok-4\\llm",
+ "scorer_details": {
+ "schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "args": [
- 1,
- "Alice",
- 30,
- true
- ],
- "golden_db": "basics-t-004-insert-golden",
- "golden_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true",
- "llm_db": "basics-t-004-insert-claude-4-sonnet-llm",
- "llm_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true",
- "query": "SELECT Id, Name, Age, Active FROM users WHERE Id=1",
- "reducer": "InsertUser",
- "server": "local"
+ "golden_db": "basics-t-001-basic-tables-golden",
+ "llm_db": "basics-t-001-basic-tables-grok-4-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
}
}
},
- "vendor": "anthropic",
- "started_at": "2025-10-20T20:37:03.262533900Z",
- "finished_at": "2025-10-20T20:38:41.453040200Z"
+ "vendor": "xai",
+ "started_at": "2025-10-21T01:13:52.120961600Z",
+ "finished_at": "2025-10-21T01:15:34.090282600Z"
},
- "t_006_delete": {
+ "t_002_scheduled_table": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_006_delete",
+ "task": "t_002_scheduled_table",
"lang": "csharp",
"golden_published": true,
- "model_name": "Claude 4 Sonnet",
- "total_tests": 3,
- "passed_tests": 3,
+ "model_name": "Grok 4",
+ "total_tests": 1,
+ "passed_tests": 1,
"llm_output": null,
"category": "basics",
- "route_api_model": "claude-sonnet-4",
- "golden_db": "basics-t-006-delete-golden",
- "llm_db": "basics-t-006-delete-claude-4-sonnet-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\claude-4-sonnet\\llm",
+ "route_api_model": "grok-4",
+ "golden_db": "basics-t-002-scheduled-table-golden",
+ "llm_db": "basics-t-002-scheduled-table-grok-4-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_002_scheduled_table\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_002_scheduled_table\\csharp\\server\\grok-4\\llm",
"scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-006-delete-golden",
- "llm_db": "basics-t-006-delete-claude-4-sonnet-llm",
+ "golden_db": "basics-t-002-scheduled-table-golden",
+ "llm_db": "basics-t-002-scheduled-table-grok-4-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
"tables_diff": null,
"tables_equal": true
}
- },
- "seed_users_row": {
+ }
+ },
+ "vendor": "xai",
+ "started_at": "2025-10-21T01:13:52.623391700Z",
+ "finished_at": "2025-10-21T01:15:13.728428500Z"
+ },
+ "t_003_struct_in_table": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_003_struct_in_table",
+ "lang": "csharp",
+ "golden_published": true,
+ "model_name": "Grok 4",
+ "total_tests": 1,
+ "passed_tests": 1,
+ "llm_output": null,
+ "category": "basics",
+ "route_api_model": "grok-4",
+ "golden_db": "basics-t-003-struct-in-table-golden",
+ "llm_db": "basics-t-003-struct-in-table-grok-4-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_003_struct_in_table\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_003_struct_in_table\\csharp\\server\\grok-4\\llm",
+ "scorer_details": {
+ "schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
+ "golden_db": "basics-t-003-struct-in-table-golden",
+ "llm_db": "basics-t-003-struct-in-table-grok-4-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
+ }
+ }
+ },
+ "vendor": "xai",
+ "started_at": "2025-10-21T01:13:53.151682100Z",
+ "finished_at": "2025-10-21T01:15:02.170466Z"
+ },
+ "t_004_insert": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_004_insert",
+ "lang": "csharp",
+ "golden_published": true,
+ "model_name": "Grok 4",
+ "total_tests": 2,
+ "passed_tests": 2,
+ "llm_output": null,
+ "category": "basics",
+ "route_api_model": "grok-4",
+ "golden_db": "basics-t-004-insert-golden",
+ "llm_db": "basics-t-004-insert-grok-4-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_004_insert\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_004_insert\\csharp\\server\\grok-4\\llm",
+ "scorer_details": {
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "golden_db": "basics-t-004-insert-golden",
+ "llm_db": "basics-t-004-insert-grok-4-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
}
},
- "delete_user_count_zero": {
+ "data_parity_insert_user": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 0,
- "expected": 0,
- "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=1"
+ "args": [
+ 1,
+ "Alice",
+ 30,
+ true
+ ],
+ "golden_db": "basics-t-004-insert-golden",
+ "golden_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true",
+ "llm_db": "basics-t-004-insert-grok-4-llm",
+ "llm_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true",
+ "query": "SELECT Id, Name, Age, Active FROM users WHERE Id=1",
+ "reducer": "InsertUser",
+ "server": "local"
}
}
},
- "vendor": "anthropic",
- "started_at": "2025-10-20T20:37:04.793257600Z",
- "finished_at": "2025-10-20T20:38:40.837307900Z"
+ "vendor": "xai",
+ "started_at": "2025-10-21T01:13:53.656401800Z",
+ "finished_at": "2025-10-21T01:15:06.998060800Z"
},
"t_005_update": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
"task": "t_005_update",
"lang": "csharp",
"golden_published": true,
- "model_name": "Claude 4 Sonnet",
+ "model_name": "Grok 4",
"total_tests": 3,
"passed_tests": 3,
"llm_output": null,
"category": "basics",
- "route_api_model": "claude-sonnet-4",
+ "route_api_model": "grok-4",
"golden_db": "basics-t-005-update-golden",
- "llm_db": "basics-t-005-update-claude-4-sonnet-llm",
+ "llm_db": "basics-t-005-update-grok-4-llm",
"work_dir_golden": "target\\llm-runs\\basics\\t_005_update\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_005_update\\csharp\\server\\claude-4-sonnet\\llm",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_005_update\\csharp\\server\\grok-4\\llm",
"scorer_details": {
"seed_users_row": {
"pass": true,
@@ -5809,7 +6377,7 @@
],
"golden_db": "basics-t-005-update-golden",
"golden_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
- "llm_db": "basics-t-005-update-claude-4-sonnet-llm",
+ "llm_db": "basics-t-005-update-grok-4-llm",
"llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
"query": "SELECT Id, Name, Age, Active FROM users WHERE Id=1",
"reducer": "UpdateUser",
@@ -5821,7 +6389,7 @@
"partial": 1.0,
"notes": {
"golden_db": "basics-t-005-update-golden",
- "llm_db": "basics-t-005-update-claude-4-sonnet-llm",
+ "llm_db": "basics-t-005-update-grok-4-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -5830,32 +6398,32 @@
}
}
},
- "vendor": "anthropic",
- "started_at": "2025-10-20T20:37:04.018006400Z",
- "finished_at": "2025-10-20T20:39:19.762751800Z"
+ "vendor": "xai",
+ "started_at": "2025-10-21T01:13:54.183350100Z",
+ "finished_at": "2025-10-21T01:15:34.745090800Z"
},
- "t_015_product_type_columns": {
+ "t_006_delete": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_015_product_type_columns",
+ "task": "t_006_delete",
"lang": "csharp",
"golden_published": true,
- "model_name": "Claude 4 Sonnet",
+ "model_name": "Grok 4",
"total_tests": 3,
"passed_tests": 3,
"llm_output": null,
- "category": "schema",
- "route_api_model": "claude-sonnet-4",
- "golden_db": "schema-t-015-product-type-columns-golden",
- "llm_db": "schema-t-015-product-type-columns-claude-4-sonnet-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\claude-4-sonnet\\llm",
+ "category": "basics",
+ "route_api_model": "grok-4",
+ "golden_db": "basics-t-006-delete-golden",
+ "llm_db": "basics-t-006-delete-grok-4-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\grok-4\\llm",
"scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-015-product-type-columns-golden",
- "llm_db": "schema-t-015-product-type-columns-claude-4-sonnet-llm",
+ "golden_db": "basics-t-006-delete-golden",
+ "llm_db": "basics-t-006-delete-grok-4-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -5863,259 +6431,197 @@
"tables_equal": true
}
},
- "product_type_columns_row_parity": {
+ "delete_user_count_zero": {
"pass": true,
"partial": 1.0,
"notes": {
- "args": [],
- "golden_db": "schema-t-015-product-type-columns-golden",
- "golden_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)",
- "llm_db": "schema-t-015-product-type-columns-claude-4-sonnet-llm",
- "llm_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)",
- "query": "SELECT Id, Home, Work, Pos FROM profiles WHERE Id=1",
- "reducer": "Seed",
- "server": "local"
+ "actual": 0,
+ "expected": 0,
+ "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=1"
}
},
- "product_type_columns_row_count": {
+ "seed_users_row": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM profiles WHERE Id=1"
+ "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
}
}
},
- "vendor": "anthropic",
- "started_at": "2025-10-20T20:41:38.389554300Z",
- "finished_at": "2025-10-20T20:42:57.975866600Z"
+ "vendor": "xai",
+ "started_at": "2025-10-21T01:13:54.697276100Z",
+ "finished_at": "2025-10-21T01:15:29.604568200Z"
},
"t_007_crud": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
"task": "t_007_crud",
"lang": "csharp",
"golden_published": true,
- "model_name": "Claude 4 Sonnet",
+ "model_name": "Grok 4",
"total_tests": 4,
- "passed_tests": 4,
+ "passed_tests": 0,
"llm_output": null,
"category": "basics",
- "route_api_model": "claude-sonnet-4",
+ "route_api_model": "grok-4",
"golden_db": "basics-t-007-crud-golden",
- "llm_db": "basics-t-007-crud-claude-4-sonnet-llm",
+ "llm_db": "basics-t-007-crud-grok-4-llm",
"work_dir_golden": "target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\claude-4-sonnet\\llm",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\grok-4\\llm",
"scorer_details": {
- "crud_total_count_one": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM users"
- }
- },
- "crud_row_id2_deleted": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 0,
- "expected": 0,
- "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=2"
- }
- },
- "crud_row_id1_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "args": [],
- "golden_db": "basics-t-007-crud-golden",
- "golden_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
- "llm_db": "basics-t-007-crud-claude-4-sonnet-llm",
- "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
- "query": "SELECT Id, Name, Age, Active FROM users WHERE Id=1",
- "reducer": "Crud",
- "server": "local"
- }
- },
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
+ "publish_error": {
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "golden_db": "basics-t-007-crud-golden",
- "llm_db": "basics-t-007-crud-claude-4-sonnet-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
+ "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\grok-4\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'users' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\grok-4\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\grok-4\\llm\\Lib.cs(23,14): error CS1061: 'Module.Users?' does not contain a definition for 'Name' and no accessible extension method 'Name' accepting a first argument of type 'Module.Users?' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\grok-4\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\grok-4\\llm\\Lib.cs(24,14): error CS1061: 'Module.Users?' does not contain a definition for 'Age' and no accessible extension method 'Age' accepting a first argument of type 'Module.Users?' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\grok-4\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\grok-4\\llm\\Lib.cs(25,14): error CS1061: 'Module.Users?' does not contain a definition for 'Active' and no accessible extension method 'Active' accepting a first argument of type 'Module.Users?' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\grok-4\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\grok-4\\llm\\Lib.cs(26,32): error CS1503: Argument 1: cannot convert from 'Module.Users?' to 'Module.Users' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\grok-4\\llm\\StdbModule.csproj]\r\n",
+ "phase": "build_or_publish"
}
}
},
- "vendor": "anthropic",
- "started_at": "2025-10-20T20:37:05.570212Z",
- "finished_at": "2025-10-20T20:38:40.427331500Z"
+ "vendor": "xai",
+ "started_at": "2025-10-21T01:13:55.219200200Z",
+ "finished_at": "2025-10-21T01:14:52.587564400Z"
},
- "t_021_multi_column_index": {
+ "t_008_index_lookup": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_021_multi_column_index",
+ "task": "t_008_index_lookup",
"lang": "csharp",
"golden_published": true,
- "model_name": "Claude 4 Sonnet",
- "total_tests": 4,
+ "model_name": "Grok 4",
+ "total_tests": 3,
"passed_tests": 0,
"llm_output": null,
- "category": "schema",
- "route_api_model": "claude-sonnet-4",
- "golden_db": "schema-t-021-multi-column-index-golden",
- "llm_db": "schema-t-021-multi-column-index-claude-4-sonnet-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\claude-4-sonnet\\llm",
+ "category": "basics",
+ "route_api_model": "grok-4",
+ "golden_db": "basics-t-008-index-lookup-golden",
+ "llm_db": "basics-t-008-index-lookup-grok-4-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\grok-4\\llm",
"scorer_details": {
"publish_error": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\claude-4-sonnet\\llm\\Lib.cs(7,6): error CS0104: 'Index' is an ambiguous reference between 'SpacetimeDB.Index' and 'System.Index' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\claude-4-sonnet\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\claude-4-sonnet\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'logs' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\claude-4-sonnet\\llm\\StdbModule.csproj]\r\n",
+ "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\grok-4\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'results' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\grok-4\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\grok-4\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(70,24): warning CS8981: The type name 'users' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\grok-4\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\grok-4\\llm\\Lib.cs(30,58): error CS1061: 'Module.User?' does not contain a definition for 'Id' and no accessible extension method 'Id' accepting a first argument of type 'Module.User?' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\grok-4\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\grok-4\\llm\\Lib.cs(30,74): error CS1061: 'Module.User?' does not contain a definition for 'Name' and no accessible extension method 'Name' accepting a first argument of type 'Module.User?' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\grok-4\\llm\\StdbModule.csproj]\r\n",
"phase": "build_or_publish"
}
}
},
- "vendor": "anthropic",
- "started_at": "2025-10-20T20:41:43.928783Z",
- "finished_at": "2025-10-20T20:41:55.279653800Z"
+ "vendor": "xai",
+ "started_at": "2025-10-21T01:13:55.716717500Z",
+ "finished_at": "2025-10-21T01:14:19.157507700Z"
},
- "t_016_sum_type_columns": {
+ "t_009_init": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_016_sum_type_columns",
+ "task": "t_009_init",
"lang": "csharp",
"golden_published": true,
- "model_name": "Claude 4 Sonnet",
- "total_tests": 3,
- "passed_tests": 3,
+ "model_name": "Grok 4",
+ "total_tests": 4,
+ "passed_tests": 4,
"llm_output": null,
- "category": "schema",
- "route_api_model": "claude-sonnet-4",
- "golden_db": "schema-t-016-sum-type-columns-golden",
- "llm_db": "schema-t-016-sum-type-columns-claude-4-sonnet-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\claude-4-sonnet\\llm",
+ "category": "basics",
+ "route_api_model": "grok-4",
+ "golden_db": "basics-t-009-init-golden",
+ "llm_db": "basics-t-009-init-grok-4-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_009_init\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_009_init\\csharp\\server\\grok-4\\llm",
"scorer_details": {
- "schema_parity": {
+ "init_seed_bob": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-016-sum-type-columns-golden",
- "llm_db": "schema-t-016-sum-type-columns-claude-4-sonnet-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false"
}
},
- "sum_type_columns_row_parity": {
+ "init_total_two": {
"pass": true,
"partial": 1.0,
"notes": {
- "args": [],
- "golden_db": "schema-t-016-sum-type-columns-golden",
- "golden_out": "Id | A | B ----+--------------------------+--------------------------------------- 1 | (Circle = (Radius = 10)) | (Rectangle = (Width = 4, Height = 6))",
- "llm_db": "schema-t-016-sum-type-columns-claude-4-sonnet-llm",
- "llm_out": "Id | A | B ----+--------------------------+--------------------------------------- 1 | (Circle = (Radius = 10)) | (Rectangle = (Width = 4, Height = 6))",
- "query": "SELECT Id, A, B FROM drawings WHERE Id=1",
- "reducer": "Seed",
- "server": "local"
+ "actual": 2,
+ "expected": 2,
+ "sql": "SELECT COUNT(*) AS n FROM users"
}
},
- "sum_type_columns_row_count": {
+ "init_seed_alice": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM drawings WHERE Id=1"
+ "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true"
+ }
+ },
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "golden_db": "basics-t-009-init-golden",
+ "llm_db": "basics-t-009-init-grok-4-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
}
}
},
- "vendor": "anthropic",
- "started_at": "2025-10-20T20:41:39.279440300Z",
- "finished_at": "2025-10-20T20:42:58.500369200Z"
+ "vendor": "xai",
+ "started_at": "2025-10-21T01:13:56.243496200Z",
+ "finished_at": "2025-10-21T01:15:45.919553400Z"
},
- "t_017_scheduled_columns": {
+ "t_010_connect": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_017_scheduled_columns",
+ "task": "t_010_connect",
"lang": "csharp",
"golden_published": true,
- "model_name": "Claude 4 Sonnet",
- "total_tests": 2,
- "passed_tests": 2,
+ "model_name": "Grok 4",
+ "total_tests": 1,
+ "passed_tests": 1,
"llm_output": null,
- "category": "schema",
- "route_api_model": "claude-sonnet-4",
- "golden_db": "schema-t-017-scheduled-columns-golden",
- "llm_db": "schema-t-017-scheduled-columns-claude-4-sonnet-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_017_scheduled_columns\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_017_scheduled_columns\\csharp\\server\\claude-4-sonnet\\llm",
+ "category": "basics",
+ "route_api_model": "grok-4",
+ "golden_db": "basics-t-010-connect-golden",
+ "llm_db": "basics-t-010-connect-grok-4-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_010_connect\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_010_connect\\csharp\\server\\grok-4\\llm",
"scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-017-scheduled-columns-golden",
- "llm_db": "schema-t-017-scheduled-columns-claude-4-sonnet-llm",
+ "golden_db": "basics-t-010-connect-golden",
+ "llm_db": "basics-t-010-connect-grok-4-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
"tables_diff": null,
"tables_equal": true
}
- },
- "scheduled_seeded_one_row": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM tick_timer WHERE ScheduledId>=0"
- }
}
},
- "vendor": "anthropic",
- "started_at": "2025-10-20T20:41:40.309622600Z",
- "finished_at": "2025-10-20T20:42:58.726183Z"
+ "vendor": "xai",
+ "started_at": "2025-10-21T01:13:56.737109800Z",
+ "finished_at": "2025-10-21T01:15:13.897979300Z"
},
- "t_012_spacetime_product_type": {
+ "t_011_helper_function": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_012_spacetime_product_type",
+ "task": "t_011_helper_function",
"lang": "csharp",
"golden_published": true,
- "model_name": "Claude 4 Sonnet",
+ "model_name": "Grok 4",
"total_tests": 3,
"passed_tests": 3,
"llm_output": null,
- "category": "schema",
- "route_api_model": "claude-sonnet-4",
- "golden_db": "schema-t-012-spacetime-product-type-golden",
- "llm_db": "schema-t-012-spacetime-product-type-claude-4-sonnet-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\claude-4-sonnet\\llm",
+ "category": "basics",
+ "route_api_model": "grok-4",
+ "golden_db": "basics-t-011-helper-function-golden",
+ "llm_db": "basics-t-011-helper-function-grok-4-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\grok-4\\llm",
"scorer_details": {
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "golden_db": "schema-t-012-spacetime-product-type-golden",
- "llm_db": "schema-t-012-spacetime-product-type-claude-4-sonnet-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
- }
- },
- "product_type_row_parity": {
+ "helper_func_sum_parity": {
"pass": true,
"partial": 1.0,
"notes": {
@@ -6124,38 +6630,74 @@
2,
3
],
- "golden_db": "schema-t-012-spacetime-product-type-golden",
- "golden_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)",
- "llm_db": "schema-t-012-spacetime-product-type-claude-4-sonnet-llm",
- "llm_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)",
- "query": "SELECT Id, Value FROM results WHERE Id=1",
- "reducer": "SetScore",
+ "golden_db": "basics-t-011-helper-function-golden",
+ "golden_out": "Id | Sum ----+----- 1 | 5",
+ "llm_db": "basics-t-011-helper-function-grok-4-llm",
+ "llm_out": "Id | Sum ----+----- 1 | 5",
+ "query": "SELECT Id, Sum FROM results WHERE Id=1",
+ "reducer": "ComputeSum",
"server": "local"
}
},
- "product_type_row_count": {
+ "helper_func_sum_abs": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1"
+ "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1 AND Sum=5"
+ }
+ },
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "golden_db": "basics-t-011-helper-function-golden",
+ "llm_db": "basics-t-011-helper-function-grok-4-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
}
}
},
- "vendor": "anthropic",
- "started_at": "2025-10-20T20:41:35.815483900Z",
- "finished_at": "2025-10-20T20:42:59.202139300Z"
- }
- }
- },
- {
- "name": "Grok 4",
- "route_api_model": "grok-4",
- "tasks": {
- "t_016_sum_type_columns": {
+ "vendor": "xai",
+ "started_at": "2025-10-21T01:13:57.232654500Z",
+ "finished_at": "2025-10-21T01:15:30.130814500Z"
+ },
+ "t_012_spacetime_product_type": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_016_sum_type_columns",
+ "task": "t_012_spacetime_product_type",
+ "lang": "csharp",
+ "golden_published": true,
+ "model_name": "Grok 4",
+ "total_tests": 3,
+ "passed_tests": 0,
+ "llm_output": null,
+ "category": "schema",
+ "route_api_model": "grok-4",
+ "golden_db": "schema-t-012-spacetime-product-type-golden",
+ "llm_db": "schema-t-012-spacetime-product-type-grok-4-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\grok-4\\llm",
+ "scorer_details": {
+ "publish_error": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\grok-4\\llm\\Lib.cs(20,33): error CS0246: The type or namespace name 'ReducerContext' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\grok-4\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\grok-4\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'results' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\grok-4\\llm\\StdbModule.csproj]\r\n",
+ "phase": "build_or_publish"
+ }
+ }
+ },
+ "vendor": "xai",
+ "started_at": "2025-10-21T00:54:56.532675700Z",
+ "finished_at": "2025-10-21T00:55:33.256614400Z"
+ },
+ "t_013_spacetime_sum_type": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_013_spacetime_sum_type",
"lang": "csharp",
"golden_published": true,
"model_name": "Grok 4",
@@ -6164,17 +6706,17 @@
"llm_output": null,
"category": "schema",
"route_api_model": "grok-4",
- "golden_db": "schema-t-016-sum-type-columns-golden",
- "llm_db": "schema-t-016-sum-type-columns-grok-4-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\grok-4\\llm",
+ "golden_db": "schema-t-013-spacetime-sum-type-golden",
+ "llm_db": "schema-t-013-spacetime-sum-type-grok-4-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\grok-4\\llm",
"scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-016-sum-type-columns-golden",
- "llm_db": "schema-t-016-sum-type-columns-grok-4-llm",
+ "golden_db": "schema-t-013-spacetime-sum-type-golden",
+ "llm_db": "schema-t-013-spacetime-sum-type-grok-4-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -6182,37 +6724,40 @@
"tables_equal": true
}
},
- "sum_type_columns_row_parity": {
+ "sum_type_row_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "args": [],
- "golden_db": "schema-t-016-sum-type-columns-golden",
- "golden_out": "Id | A | B ----+--------------------------+--------------------------------------- 1 | (Circle = (Radius = 10)) | (Rectangle = (Width = 4, Height = 6))",
- "llm_db": "schema-t-016-sum-type-columns-grok-4-llm",
- "llm_out": "Id | A | B ----+--------------------------+--------------------------------------- 1 | (Circle = (Radius = 10)) | (Rectangle = (Width = 4, Height = 6))",
- "query": "SELECT Id, A, B FROM drawings WHERE Id=1",
- "reducer": "Seed",
+ "args": [
+ 1,
+ 10
+ ],
+ "golden_db": "schema-t-013-spacetime-sum-type-golden",
+ "golden_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))",
+ "llm_db": "schema-t-013-spacetime-sum-type-grok-4-llm",
+ "llm_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))",
+ "query": "SELECT Id, Value FROM results WHERE Id=1",
+ "reducer": "SetCircle",
"server": "local"
}
},
- "sum_type_columns_row_count": {
+ "sum_type_row_count": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM drawings WHERE Id=1"
+ "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1"
}
}
},
"vendor": "xai",
- "started_at": "2025-10-21T00:54:58.625115500Z",
- "finished_at": "2025-10-21T00:56:39.932389400Z"
+ "started_at": "2025-10-21T00:54:57.115158200Z",
+ "finished_at": "2025-10-21T00:56:49.795312700Z"
},
- "t_018_constraints": {
+ "t_014_elementary_columns": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_018_constraints",
+ "task": "t_014_elementary_columns",
"lang": "csharp",
"golden_published": true,
"model_name": "Grok 4",
@@ -6221,40 +6766,40 @@
"llm_output": null,
"category": "schema",
"route_api_model": "grok-4",
- "golden_db": "schema-t-018-constraints-golden",
- "llm_db": "schema-t-018-constraints-grok-4-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\grok-4\\llm",
+ "golden_db": "schema-t-014-elementary-columns-golden",
+ "llm_db": "schema-t-014-elementary-columns-grok-4-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_014_elementary_columns\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_014_elementary_columns\\csharp\\server\\grok-4\\llm",
"scorer_details": {
- "constraints_row_parity_after_seed": {
+ "elementary_columns_row_parity": {
"pass": true,
"partial": 1.0,
"notes": {
"args": [],
- "golden_db": "schema-t-018-constraints-golden",
- "golden_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"",
- "llm_db": "schema-t-018-constraints-grok-4-llm",
- "llm_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"",
- "query": "SELECT Id, Email, Name FROM accounts WHERE Id=1",
+ "golden_db": "schema-t-014-elementary-columns-golden",
+ "golden_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"",
+ "llm_db": "schema-t-014-elementary-columns-grok-4-llm",
+ "llm_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"",
+ "query": "SELECT Id, Count, Total, Price, Ratio, Active, Name FROM primitives WHERE Id=1",
"reducer": "Seed",
"server": "local"
}
},
- "constraints_seed_two_rows": {
+ "elementary_columns_row_count": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM accounts WHERE Id=2"
+ "sql": "SELECT COUNT(*) AS n FROM primitives WHERE Id=1"
}
},
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-018-constraints-golden",
- "llm_db": "schema-t-018-constraints-grok-4-llm",
+ "golden_db": "schema-t-014-elementary-columns-golden",
+ "llm_db": "schema-t-014-elementary-columns-grok-4-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -6264,12 +6809,12 @@
}
},
"vendor": "xai",
- "started_at": "2025-10-21T00:54:59.711737900Z",
- "finished_at": "2025-10-21T00:56:21.840737100Z"
+ "started_at": "2025-10-21T00:54:57.616635900Z",
+ "finished_at": "2025-10-21T00:56:32.173953200Z"
},
- "t_014_elementary_columns": {
+ "t_015_product_type_columns": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_014_elementary_columns",
+ "task": "t_015_product_type_columns",
"lang": "csharp",
"golden_published": true,
"model_name": "Grok 4",
@@ -6278,17 +6823,17 @@
"llm_output": null,
"category": "schema",
"route_api_model": "grok-4",
- "golden_db": "schema-t-014-elementary-columns-golden",
- "llm_db": "schema-t-014-elementary-columns-grok-4-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_014_elementary_columns\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_014_elementary_columns\\csharp\\server\\grok-4\\llm",
+ "golden_db": "schema-t-015-product-type-columns-golden",
+ "llm_db": "schema-t-015-product-type-columns-grok-4-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\grok-4\\llm",
"scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-014-elementary-columns-golden",
- "llm_db": "schema-t-014-elementary-columns-grok-4-llm",
+ "golden_db": "schema-t-015-product-type-columns-golden",
+ "llm_db": "schema-t-015-product-type-columns-grok-4-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -6296,37 +6841,37 @@
"tables_equal": true
}
},
- "elementary_columns_row_parity": {
+ "product_type_columns_row_count": {
"pass": true,
"partial": 1.0,
"notes": {
- "args": [],
- "golden_db": "schema-t-014-elementary-columns-golden",
- "golden_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"",
- "llm_db": "schema-t-014-elementary-columns-grok-4-llm",
- "llm_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"",
- "query": "SELECT Id, Count, Total, Price, Ratio, Active, Name FROM primitives WHERE Id=1",
- "reducer": "Seed",
- "server": "local"
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM profiles WHERE Id=1"
}
},
- "elementary_columns_row_count": {
+ "product_type_columns_row_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM primitives WHERE Id=1"
+ "args": [],
+ "golden_db": "schema-t-015-product-type-columns-golden",
+ "golden_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)",
+ "llm_db": "schema-t-015-product-type-columns-grok-4-llm",
+ "llm_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)",
+ "query": "SELECT Id, Home, Work, Pos FROM profiles WHERE Id=1",
+ "reducer": "Seed",
+ "server": "local"
}
}
},
"vendor": "xai",
- "started_at": "2025-10-21T00:54:57.616635900Z",
- "finished_at": "2025-10-21T00:56:32.173953200Z"
+ "started_at": "2025-10-21T00:54:58.119730400Z",
+ "finished_at": "2025-10-21T00:56:24.083824100Z"
},
- "t_013_spacetime_sum_type": {
+ "t_016_sum_type_columns": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_013_spacetime_sum_type",
+ "task": "t_016_sum_type_columns",
"lang": "csharp",
"golden_published": true,
"model_name": "Grok 4",
@@ -6335,17 +6880,31 @@
"llm_output": null,
"category": "schema",
"route_api_model": "grok-4",
- "golden_db": "schema-t-013-spacetime-sum-type-golden",
- "llm_db": "schema-t-013-spacetime-sum-type-grok-4-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\grok-4\\llm",
+ "golden_db": "schema-t-016-sum-type-columns-golden",
+ "llm_db": "schema-t-016-sum-type-columns-grok-4-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\grok-4\\llm",
"scorer_details": {
+ "sum_type_columns_row_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "args": [],
+ "golden_db": "schema-t-016-sum-type-columns-golden",
+ "golden_out": "Id | A | B ----+--------------------------+--------------------------------------- 1 | (Circle = (Radius = 10)) | (Rectangle = (Width = 4, Height = 6))",
+ "llm_db": "schema-t-016-sum-type-columns-grok-4-llm",
+ "llm_out": "Id | A | B ----+--------------------------+--------------------------------------- 1 | (Circle = (Radius = 10)) | (Rectangle = (Width = 4, Height = 6))",
+ "query": "SELECT Id, A, B FROM drawings WHERE Id=1",
+ "reducer": "Seed",
+ "server": "local"
+ }
+ },
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-013-spacetime-sum-type-golden",
- "llm_db": "schema-t-013-spacetime-sum-type-grok-4-llm",
+ "golden_db": "schema-t-016-sum-type-columns-golden",
+ "llm_db": "schema-t-016-sum-type-columns-grok-4-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -6353,106 +6912,119 @@
"tables_equal": true
}
},
- "sum_type_row_count": {
+ "sum_type_columns_row_count": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1"
+ "sql": "SELECT COUNT(*) AS n FROM drawings WHERE Id=1"
+ }
+ }
+ },
+ "vendor": "xai",
+ "started_at": "2025-10-21T00:54:58.625115500Z",
+ "finished_at": "2025-10-21T00:56:39.932389400Z"
+ },
+ "t_017_scheduled_columns": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_017_scheduled_columns",
+ "lang": "csharp",
+ "golden_published": true,
+ "model_name": "Grok 4",
+ "total_tests": 2,
+ "passed_tests": 2,
+ "llm_output": null,
+ "category": "schema",
+ "route_api_model": "grok-4",
+ "golden_db": "schema-t-017-scheduled-columns-golden",
+ "llm_db": "schema-t-017-scheduled-columns-grok-4-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_017_scheduled_columns\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_017_scheduled_columns\\csharp\\server\\grok-4\\llm",
+ "scorer_details": {
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "golden_db": "schema-t-017-scheduled-columns-golden",
+ "llm_db": "schema-t-017-scheduled-columns-grok-4-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
}
},
- "sum_type_row_parity": {
+ "scheduled_seeded_one_row": {
"pass": true,
"partial": 1.0,
"notes": {
- "args": [
- 1,
- 10
- ],
- "golden_db": "schema-t-013-spacetime-sum-type-golden",
- "golden_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))",
- "llm_db": "schema-t-013-spacetime-sum-type-grok-4-llm",
- "llm_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))",
- "query": "SELECT Id, Value FROM results WHERE Id=1",
- "reducer": "SetCircle",
- "server": "local"
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM tick_timer WHERE ScheduledId>=0"
}
}
},
"vendor": "xai",
- "started_at": "2025-10-21T00:54:57.115158200Z",
- "finished_at": "2025-10-21T00:56:49.795312700Z"
+ "started_at": "2025-10-21T00:54:59.124579600Z",
+ "finished_at": "2025-10-21T00:56:32.470601800Z"
},
- "t_020_ecs": {
+ "t_018_constraints": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_020_ecs",
+ "task": "t_018_constraints",
"lang": "csharp",
"golden_published": true,
"model_name": "Grok 4",
- "total_tests": 5,
- "passed_tests": 5,
+ "total_tests": 3,
+ "passed_tests": 3,
"llm_output": null,
"category": "schema",
"route_api_model": "grok-4",
- "golden_db": "schema-t-020-ecs-golden",
- "llm_db": "schema-t-020-ecs-grok-4-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\grok-4\\llm",
+ "golden_db": "schema-t-018-constraints-golden",
+ "llm_db": "schema-t-018-constraints-grok-4-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\grok-4\\llm",
"scorer_details": {
- "ecs_next_pos_entity1": {
+ "constraints_seed_two_rows": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM next_positions WHERE EntityId=1 AND X=1 AND Y=0"
- }
- },
- "ecs_step_next_positions_count": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 2,
- "expected": 2,
- "sql": "SELECT COUNT(*) AS n FROM next_positions"
+ "sql": "SELECT COUNT(*) AS n FROM accounts WHERE Id=2"
}
},
- "ecs_seed_positions_count": {
+ "constraints_row_parity_after_seed": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 2,
- "expected": 2,
- "sql": "SELECT COUNT(*) AS n FROM positions"
+ "args": [],
+ "golden_db": "schema-t-018-constraints-golden",
+ "golden_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"",
+ "llm_db": "schema-t-018-constraints-grok-4-llm",
+ "llm_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"",
+ "query": "SELECT Id, Email, Name FROM accounts WHERE Id=1",
+ "reducer": "Seed",
+ "server": "local"
}
},
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-020-ecs-golden",
- "llm_db": "schema-t-020-ecs-grok-4-llm",
+ "golden_db": "schema-t-018-constraints-golden",
+ "llm_db": "schema-t-018-constraints-grok-4-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
"tables_diff": null,
"tables_equal": true
}
- },
- "ecs_next_pos_entity2": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM next_positions WHERE EntityId=2 AND X=8 AND Y=3"
- }
}
},
"vendor": "xai",
- "started_at": "2025-10-21T00:55:00.807120900Z",
- "finished_at": "2025-10-21T00:56:43.533400800Z"
+ "started_at": "2025-10-21T00:54:59.711737900Z",
+ "finished_at": "2025-10-21T00:56:21.840737100Z"
},
"t_019_many_to_many": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
@@ -6479,22 +7051,22 @@
"sql": "SELECT COUNT(*) AS n FROM memberships WHERE UserId=1 AND GroupId=20"
}
},
- "m2m_has_1_10": {
+ "memberships_three_rows": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM memberships WHERE UserId=1 AND GroupId=10"
+ "actual": 3,
+ "expected": 3,
+ "sql": "SELECT COUNT(*) AS n FROM memberships"
}
},
- "memberships_three_rows": {
+ "m2m_has_1_10": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 3,
- "expected": 3,
- "sql": "SELECT COUNT(*) AS n FROM memberships"
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM memberships WHERE UserId=1 AND GroupId=10"
}
},
"m2m_has_2_20": {
@@ -6524,96 +7096,194 @@
"started_at": "2025-10-21T00:55:00.274643700Z",
"finished_at": "2025-10-21T00:56:50.220880200Z"
},
- "t_002_scheduled_table": {
+ "t_020_ecs": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_002_scheduled_table",
+ "task": "t_020_ecs",
"lang": "csharp",
"golden_published": true,
"model_name": "Grok 4",
- "total_tests": 1,
- "passed_tests": 1,
+ "total_tests": 5,
+ "passed_tests": 5,
"llm_output": null,
- "category": "basics",
+ "category": "schema",
"route_api_model": "grok-4",
- "golden_db": "basics-t-002-scheduled-table-golden",
- "llm_db": "basics-t-002-scheduled-table-grok-4-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_002_scheduled_table\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_002_scheduled_table\\csharp\\server\\grok-4\\llm",
+ "golden_db": "schema-t-020-ecs-golden",
+ "llm_db": "schema-t-020-ecs-grok-4-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\grok-4\\llm",
"scorer_details": {
+ "ecs_next_pos_entity1": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM next_positions WHERE EntityId=1 AND X=1 AND Y=0"
+ }
+ },
+ "ecs_step_next_positions_count": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 2,
+ "expected": 2,
+ "sql": "SELECT COUNT(*) AS n FROM next_positions"
+ }
+ },
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-002-scheduled-table-golden",
- "llm_db": "basics-t-002-scheduled-table-grok-4-llm",
+ "golden_db": "schema-t-020-ecs-golden",
+ "llm_db": "schema-t-020-ecs-grok-4-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
"tables_diff": null,
"tables_equal": true
}
+ },
+ "ecs_seed_positions_count": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 2,
+ "expected": 2,
+ "sql": "SELECT COUNT(*) AS n FROM positions"
+ }
+ },
+ "ecs_next_pos_entity2": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM next_positions WHERE EntityId=2 AND X=8 AND Y=3"
+ }
}
},
"vendor": "xai",
- "started_at": "2025-10-21T01:13:52.623391700Z",
- "finished_at": "2025-10-21T01:15:13.728428500Z"
+ "started_at": "2025-10-21T00:55:00.807120900Z",
+ "finished_at": "2025-10-21T00:56:43.533400800Z"
},
- "t_001_basic_tables": {
+ "t_021_multi_column_index": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_001_basic_tables",
+ "task": "t_021_multi_column_index",
"lang": "csharp",
"golden_published": true,
"model_name": "Grok 4",
- "total_tests": 1,
- "passed_tests": 1,
+ "total_tests": 4,
+ "passed_tests": 4,
"llm_output": null,
- "category": "basics",
+ "category": "schema",
"route_api_model": "grok-4",
- "golden_db": "basics-t-001-basic-tables-golden",
- "llm_db": "basics-t-001-basic-tables-grok-4-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\grok-4\\llm",
+ "golden_db": "schema-t-021-multi-column-index-golden",
+ "llm_db": "schema-t-021-multi-column-index-grok-4-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\grok-4\\llm",
"scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-001-basic-tables-golden",
- "llm_db": "basics-t-001-basic-tables-grok-4-llm",
+ "golden_db": "schema-t-021-multi-column-index-golden",
+ "llm_db": "schema-t-021-multi-column-index-grok-4-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
"tables_diff": null,
"tables_equal": true
}
- }
- },
- "vendor": "xai",
- "started_at": "2025-10-21T01:13:52.120961600Z",
- "finished_at": "2025-10-21T01:15:34.090282600Z"
- },
- "t_000_empty_reducers": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_000_empty_reducers",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "Grok 4",
- "total_tests": 1,
- "passed_tests": 1,
- "llm_output": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [SpacetimeDB.Reducer]\r\n public static void EmptyReducer_NoArgs(ReducerContext ctx)\r\n {\r\n }\r\n\r\n [SpacetimeDB.Reducer]\r\n public static void EmptyReducer_WithInt(ReducerContext ctx, int count)\r\n {\r\n }\r\n\r\n [SpacetimeDB.Reducer]\r\n public static void EmptyReducer_WithString(ReducerContext ctx, string name)\r\n {\r\n }\r\n\r\n [SpacetimeDB.Reducer]\r\n public static void EmptyReducer_WithTwoArgs(ReducerContext ctx, int count, string name)\r\n {\r\n }\r\n\r\n [SpacetimeDB.Reducer]\r\n public static void EmptyReducer_WithThreeArgs(ReducerContext ctx, bool active, float ratio, string label)\r\n {\r\n }\r\n}",
- "category": "basics",
- "route_api_model": "grok-4",
- "golden_db": "basics-t-000-empty-reducers-golden",
- "llm_db": "basics-t-000-empty-reducers-grok-4-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-4\\llm",
- "scorer_details": {
+ },
+ "mcindex_seed_count": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 3,
+ "expected": 3,
+ "sql": "SELECT COUNT(*) AS n FROM logs"
+ }
+ },
+ "mcindex_lookup_u7_d1": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM logs WHERE UserId=7 AND Day=1"
+ }
+ },
+ "mcindex_lookup_u7_d2": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM logs WHERE UserId=7 AND Day=2"
+ }
+ }
+ },
+ "vendor": "xai",
+ "started_at": "2025-10-21T00:55:01.339990400Z",
+ "finished_at": "2025-10-21T00:56:41.639356300Z"
+ }
+ }
+ },
+ {
+ "name": "Grok 3 Mini (Beta)",
+ "route_api_model": "grok-3-mini",
+ "tasks": {
+ "t_000_empty_reducers": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_000_empty_reducers",
+ "lang": "csharp",
+ "golden_published": true,
+ "model_name": "Grok 3 Mini (Beta)",
+ "total_tests": 1,
+ "passed_tests": 0,
+ "llm_output": "using Spacetimedb;\n\npublic static partial class Module\n{\n [Reducer]\n public static void EmptyReducer_NoArgs(ReducerContext ctx)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithInt(ReducerContext ctx, int count)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithString(ReducerContext ctx, string name)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithTwoArgs(ReducerContext ctx, int count, string name)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithThreeArgs(ReducerContext ctx, bool active, float ratio, string label)\n {\n }\n}",
+ "category": "basics",
+ "route_api_model": "grok-3-mini",
+ "golden_db": "basics-t-000-empty-reducers-golden",
+ "llm_db": "basics-t-000-empty-reducers-grok-3-mini-beta-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm",
+ "scorer_details": {
+ "publish_error": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(2,7): error CS0246: The type or namespace name 'Spacetimedb' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(7,44): error CS0246: The type or namespace name 'ReducerContext' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(12,45): error CS0246: The type or namespace name 'ReducerContext' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(17,48): error CS0246: The type or namespace name 'ReducerContext' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(22,49): error CS0246: The type or namespace name 'ReducerContext' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(27,51): error CS0246: The type or namespace name 'ReducerContext' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(6,6): error CS0246: The type or namespace name 'ReducerAttribute' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(6,6): error CS0246: The type or namespace name 'Reducer' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(11,6): error CS0246: The type or namespace name 'ReducerAttribute' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(11,6): error CS0246: The type or namespace name 'Reducer' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(16,6): error CS0246: The type or namespace name 'ReducerAttribute' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(16,6): error CS0246: The type or namespace name 'Reducer' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(21,6): error CS0246: The type or namespace name 'ReducerAttribute' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(21,6): error CS0246: The type or namespace name 'Reducer' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(26,6): error CS0246: The type or namespace name 'ReducerAttribute' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(26,6): error CS0246: The type or namespace name 'Reducer' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\n",
+ "phase": "build_or_publish"
+ }
+ }
+ },
+ "vendor": "xai",
+ "started_at": "2025-10-21T23:59:35.577464800Z",
+ "finished_at": "2025-10-22T00:00:02.000437300Z"
+ },
+ "t_001_basic_tables": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_001_basic_tables",
+ "lang": "csharp",
+ "golden_published": true,
+ "model_name": "Grok 3 Mini (Beta)",
+ "total_tests": 1,
+ "passed_tests": 1,
+ "llm_output": null,
+ "category": "basics",
+ "route_api_model": "grok-3-mini",
+ "golden_db": "basics-t-001-basic-tables-golden",
+ "llm_db": "basics-t-001-basic-tables-grok-3-mini-beta-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\grok-3-mini-beta\\llm",
+ "scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-000-empty-reducers-golden",
- "llm_db": "basics-t-000-empty-reducers-grok-4-llm",
+ "golden_db": "basics-t-001-basic-tables-golden",
+ "llm_db": "basics-t-001-basic-tables-grok-3-mini-beta-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -6623,60 +7293,60 @@
}
},
"vendor": "xai",
- "started_at": "2025-10-21T23:58:47.413051600Z",
- "finished_at": "2025-10-22T00:00:10.958973700Z"
+ "started_at": "2025-10-21T01:20:22.848063200Z",
+ "finished_at": "2025-10-21T01:21:42.592621100Z"
},
- "t_012_spacetime_product_type": {
+ "t_002_scheduled_table": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_012_spacetime_product_type",
+ "task": "t_002_scheduled_table",
"lang": "csharp",
"golden_published": true,
- "model_name": "Grok 4",
- "total_tests": 3,
+ "model_name": "Grok 3 Mini (Beta)",
+ "total_tests": 1,
"passed_tests": 0,
"llm_output": null,
- "category": "schema",
- "route_api_model": "grok-4",
- "golden_db": "schema-t-012-spacetime-product-type-golden",
- "llm_db": "schema-t-012-spacetime-product-type-grok-4-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\grok-4\\llm",
+ "category": "basics",
+ "route_api_model": "grok-3-mini",
+ "golden_db": "basics-t-002-scheduled-table-golden",
+ "llm_db": "basics-t-002-scheduled-table-grok-3-mini-beta-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_002_scheduled_table\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_002_scheduled_table\\csharp\\server\\grok-3-mini-beta\\llm",
"scorer_details": {
"publish_error": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\grok-4\\llm\\Lib.cs(20,33): error CS0246: The type or namespace name 'ReducerContext' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\grok-4\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\grok-4\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'results' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\grok-4\\llm\\StdbModule.csproj]\r\n",
+ "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_002_scheduled_table\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(3,19): error CS0234: The type or namespace name 'Types' does not exist in the namespace 'SpacetimeDB' (are you missing an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_002_scheduled_table\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_002_scheduled_table\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(5,60): error CS0103: The name 'Tick' does not exist in the current context [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_002_scheduled_table\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\n",
"phase": "build_or_publish"
}
}
},
"vendor": "xai",
- "started_at": "2025-10-21T00:54:56.532675700Z",
- "finished_at": "2025-10-21T00:55:33.256614400Z"
+ "started_at": "2025-10-21T01:20:23.355931300Z",
+ "finished_at": "2025-10-21T01:20:46.972683Z"
},
"t_003_struct_in_table": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
"task": "t_003_struct_in_table",
"lang": "csharp",
"golden_published": true,
- "model_name": "Grok 4",
+ "model_name": "Grok 3 Mini (Beta)",
"total_tests": 1,
"passed_tests": 1,
"llm_output": null,
"category": "basics",
- "route_api_model": "grok-4",
+ "route_api_model": "grok-3-mini",
"golden_db": "basics-t-003-struct-in-table-golden",
- "llm_db": "basics-t-003-struct-in-table-grok-4-llm",
+ "llm_db": "basics-t-003-struct-in-table-grok-3-mini-beta-llm",
"work_dir_golden": "target\\llm-runs\\basics\\t_003_struct_in_table\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_003_struct_in_table\\csharp\\server\\grok-4\\llm",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_003_struct_in_table\\csharp\\server\\grok-3-mini-beta\\llm",
"scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
"golden_db": "basics-t-003-struct-in-table-golden",
- "llm_db": "basics-t-003-struct-in-table-grok-4-llm",
+ "llm_db": "basics-t-003-struct-in-table-grok-3-mini-beta-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -6686,45 +7356,31 @@
}
},
"vendor": "xai",
- "started_at": "2025-10-21T01:13:53.151682100Z",
- "finished_at": "2025-10-21T01:15:02.170466Z"
+ "started_at": "2025-10-21T01:20:23.854245800Z",
+ "finished_at": "2025-10-21T01:21:38.441196600Z"
},
- "t_015_product_type_columns": {
+ "t_004_insert": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_015_product_type_columns",
+ "task": "t_004_insert",
"lang": "csharp",
"golden_published": true,
- "model_name": "Grok 4",
- "total_tests": 3,
- "passed_tests": 3,
+ "model_name": "Grok 3 Mini (Beta)",
+ "total_tests": 2,
+ "passed_tests": 2,
"llm_output": null,
- "category": "schema",
- "route_api_model": "grok-4",
- "golden_db": "schema-t-015-product-type-columns-golden",
- "llm_db": "schema-t-015-product-type-columns-grok-4-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\grok-4\\llm",
+ "category": "basics",
+ "route_api_model": "grok-3-mini",
+ "golden_db": "basics-t-004-insert-golden",
+ "llm_db": "basics-t-004-insert-grok-3-mini-beta-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_004_insert\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_004_insert\\csharp\\server\\grok-3-mini-beta\\llm",
"scorer_details": {
- "product_type_columns_row_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "args": [],
- "golden_db": "schema-t-015-product-type-columns-golden",
- "golden_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)",
- "llm_db": "schema-t-015-product-type-columns-grok-4-llm",
- "llm_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)",
- "query": "SELECT Id, Home, Work, Pos FROM profiles WHERE Id=1",
- "reducer": "Seed",
- "server": "local"
- }
- },
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-015-product-type-columns-golden",
- "llm_db": "schema-t-015-product-type-columns-grok-4-llm",
+ "golden_db": "basics-t-004-insert-golden",
+ "llm_db": "basics-t-004-insert-grok-3-mini-beta-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -6732,71 +7388,229 @@
"tables_equal": true
}
},
- "product_type_columns_row_count": {
+ "data_parity_insert_user": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM profiles WHERE Id=1"
+ "args": [
+ 1,
+ "Alice",
+ 30,
+ true
+ ],
+ "golden_db": "basics-t-004-insert-golden",
+ "golden_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true",
+ "llm_db": "basics-t-004-insert-grok-3-mini-beta-llm",
+ "llm_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true",
+ "query": "SELECT Id, Name, Age, Active FROM users WHERE Id=1",
+ "reducer": "InsertUser",
+ "server": "local"
}
}
},
"vendor": "xai",
- "started_at": "2025-10-21T00:54:58.119730400Z",
- "finished_at": "2025-10-21T00:56:24.083824100Z"
+ "started_at": "2025-10-21T01:20:24.355687300Z",
+ "finished_at": "2025-10-21T01:21:39.029068800Z"
+ },
+ "t_005_update": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_005_update",
+ "lang": "csharp",
+ "golden_published": true,
+ "model_name": "Grok 3 Mini (Beta)",
+ "total_tests": 3,
+ "passed_tests": 0,
+ "llm_output": null,
+ "category": "basics",
+ "route_api_model": "grok-3-mini",
+ "golden_db": "basics-t-005-update-golden",
+ "llm_db": "basics-t-005-update-grok-3-mini-beta-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_005_update\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_005_update\\csharp\\server\\grok-3-mini-beta\\llm",
+ "scorer_details": {
+ "publish_error": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_005_update\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(14,1): error CS8803: Top-level statements must precede namespace and type declarations. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_005_update\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_005_update\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(15,1): error CS0106: The modifier 'public' is not valid for this item [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_005_update\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\n",
+ "phase": "build_or_publish"
+ }
+ }
+ },
+ "vendor": "xai",
+ "started_at": "2025-10-21T01:20:24.865269700Z",
+ "finished_at": "2025-10-21T01:20:43.203415300Z"
+ },
+ "t_006_delete": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_006_delete",
+ "lang": "csharp",
+ "golden_published": true,
+ "model_name": "Grok 3 Mini (Beta)",
+ "total_tests": 3,
+ "passed_tests": 0,
+ "llm_output": null,
+ "category": "basics",
+ "route_api_model": "grok-3-mini",
+ "golden_db": "basics-t-006-delete-golden",
+ "llm_db": "basics-t-006-delete-grok-3-mini-beta-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\grok-3-mini-beta\\llm",
+ "scorer_details": {
+ "publish_error": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\grok-3-mini-beta\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(56,46): error CS0050: Inconsistent accessibility: return type 'IEnumerable' is less accessible than method 'users.Iter()' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\grok-3-mini-beta\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(57,33): error CS0050: Inconsistent accessibility: return type 'Module.Users' is less accessible than method 'users.Insert(Module.Users)' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\grok-3-mini-beta\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(57,33): error CS0051: Inconsistent accessibility: parameter type 'Module.Users' is less accessible than method 'users.Insert(Module.Users)' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\grok-3-mini-beta\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(58,17): error CS0051: Inconsistent accessibility: parameter type 'Module.Users' is less accessible than method 'users.Delete(Module.Users)' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\grok-3-mini-beta\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(60,25): error CS0060: Inconsistent accessibility: base class 'UniqueIndex' is less accessible than class 'users.IdUniqueIndex' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\grok-3-mini-beta\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(65,34): error CS0050: Inconsistent accessibility: return type 'Module.Users?' is less accessible than method 'users.IdUniqueIndex.Find(int)' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\grok-3-mini-beta\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(66,33): error CS0050: Inconsistent accessibility: return type 'Module.Users' is less accessible than method 'users.IdUniqueIndex.Update(Module.Users)' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\grok-3-mini-beta\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(66,33): error CS0051: Inconsistent accessibility: parameter type 'Module.Users' is less accessible than method 'users.IdUniqueIndex.Update(Module.Users)' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\grok-3-mini-beta\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'users' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\n",
+ "phase": "build_or_publish"
+ }
+ }
+ },
+ "vendor": "xai",
+ "started_at": "2025-10-21T01:20:25.377070500Z",
+ "finished_at": "2025-10-21T01:20:43.300117900Z"
+ },
+ "t_007_crud": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_007_crud",
+ "lang": "csharp",
+ "golden_published": true,
+ "model_name": "Grok 3 Mini (Beta)",
+ "total_tests": 4,
+ "passed_tests": 0,
+ "llm_output": null,
+ "category": "basics",
+ "route_api_model": "grok-3-mini",
+ "golden_db": "basics-t-007-crud-golden",
+ "llm_db": "basics-t-007-crud-grok-3-mini-beta-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\grok-3-mini-beta\\llm",
+ "scorer_details": {
+ "publish_error": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(14,1): error CS8803: Top-level statements must precede namespace and type declarations. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(15,1): error CS0106: The modifier 'public' is not valid for this item [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\n",
+ "phase": "build_or_publish"
+ }
+ }
+ },
+ "vendor": "xai",
+ "started_at": "2025-10-21T01:20:25.895347300Z",
+ "finished_at": "2025-10-21T01:20:44.275623400Z"
},
"t_008_index_lookup": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
"task": "t_008_index_lookup",
"lang": "csharp",
"golden_published": true,
- "model_name": "Grok 4",
+ "model_name": "Grok 3 Mini (Beta)",
"total_tests": 3,
"passed_tests": 0,
"llm_output": null,
"category": "basics",
- "route_api_model": "grok-4",
+ "route_api_model": "grok-3-mini",
"golden_db": "basics-t-008-index-lookup-golden",
- "llm_db": "basics-t-008-index-lookup-grok-4-llm",
+ "llm_db": "basics-t-008-index-lookup-grok-3-mini-beta-llm",
"work_dir_golden": "target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\grok-4\\llm",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\grok-3-mini-beta\\llm",
"scorer_details": {
"publish_error": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\grok-4\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'results' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\grok-4\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\grok-4\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(70,24): warning CS8981: The type name 'users' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\grok-4\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\grok-4\\llm\\Lib.cs(30,58): error CS1061: 'Module.User?' does not contain a definition for 'Id' and no accessible extension method 'Id' accepting a first argument of type 'Module.User?' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\grok-4\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\grok-4\\llm\\Lib.cs(30,74): error CS1061: 'Module.User?' does not contain a definition for 'Name' and no accessible extension method 'Name' accepting a first argument of type 'Module.User?' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\grok-4\\llm\\StdbModule.csproj]\r\n",
+ "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(22,1): error CS8803: Top-level statements must precede namespace and type declarations. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(23,1): error CS0106: The modifier 'public' is not valid for this item [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\n",
"phase": "build_or_publish"
}
}
},
"vendor": "xai",
- "started_at": "2025-10-21T01:13:55.716717500Z",
- "finished_at": "2025-10-21T01:14:19.157507700Z"
+ "started_at": "2025-10-21T01:20:26.421931300Z",
+ "finished_at": "2025-10-21T01:20:43.941929700Z"
+ },
+ "t_009_init": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_009_init",
+ "lang": "csharp",
+ "golden_published": true,
+ "model_name": "Grok 3 Mini (Beta)",
+ "total_tests": 4,
+ "passed_tests": 4,
+ "llm_output": null,
+ "category": "basics",
+ "route_api_model": "grok-3-mini",
+ "golden_db": "basics-t-009-init-golden",
+ "llm_db": "basics-t-009-init-grok-3-mini-beta-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_009_init\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_009_init\\csharp\\server\\grok-3-mini-beta\\llm",
+ "scorer_details": {
+ "init_seed_alice": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true"
+ }
+ },
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "golden_db": "basics-t-009-init-golden",
+ "llm_db": "basics-t-009-init-grok-3-mini-beta-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
+ }
+ },
+ "init_seed_bob": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false"
+ }
+ },
+ "init_total_two": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 2,
+ "expected": 2,
+ "sql": "SELECT COUNT(*) AS n FROM users"
+ }
+ }
+ },
+ "vendor": "xai",
+ "started_at": "2025-10-21T01:20:26.945205600Z",
+ "finished_at": "2025-10-21T01:21:40.007191100Z"
},
"t_010_connect": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
"task": "t_010_connect",
"lang": "csharp",
"golden_published": true,
- "model_name": "Grok 4",
+ "model_name": "Grok 3 Mini (Beta)",
"total_tests": 1,
"passed_tests": 1,
"llm_output": null,
"category": "basics",
- "route_api_model": "grok-4",
+ "route_api_model": "grok-3-mini",
"golden_db": "basics-t-010-connect-golden",
- "llm_db": "basics-t-010-connect-grok-4-llm",
+ "llm_db": "basics-t-010-connect-grok-3-mini-beta-llm",
"work_dir_golden": "target\\llm-runs\\basics\\t_010_connect\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_010_connect\\csharp\\server\\grok-4\\llm",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_010_connect\\csharp\\server\\grok-3-mini-beta\\llm",
"scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
"golden_db": "basics-t-010-connect-golden",
- "llm_db": "basics-t-010-connect-grok-4-llm",
+ "llm_db": "basics-t-010-connect-grok-3-mini-beta-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -6806,60 +7620,60 @@
}
},
"vendor": "xai",
- "started_at": "2025-10-21T01:13:56.737109800Z",
- "finished_at": "2025-10-21T01:15:13.897979300Z"
+ "started_at": "2025-10-21T01:20:27.445891100Z",
+ "finished_at": "2025-10-21T01:21:42.754747300Z"
},
- "t_007_crud": {
+ "t_011_helper_function": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_007_crud",
+ "task": "t_011_helper_function",
"lang": "csharp",
"golden_published": true,
- "model_name": "Grok 4",
- "total_tests": 4,
+ "model_name": "Grok 3 Mini (Beta)",
+ "total_tests": 3,
"passed_tests": 0,
"llm_output": null,
"category": "basics",
- "route_api_model": "grok-4",
- "golden_db": "basics-t-007-crud-golden",
- "llm_db": "basics-t-007-crud-grok-4-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\grok-4\\llm",
+ "route_api_model": "grok-3-mini",
+ "golden_db": "basics-t-011-helper-function-golden",
+ "llm_db": "basics-t-011-helper-function-grok-3-mini-beta-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\grok-3-mini-beta\\llm",
"scorer_details": {
"publish_error": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\grok-4\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'users' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\grok-4\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\grok-4\\llm\\Lib.cs(23,14): error CS1061: 'Module.Users?' does not contain a definition for 'Name' and no accessible extension method 'Name' accepting a first argument of type 'Module.Users?' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\grok-4\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\grok-4\\llm\\Lib.cs(24,14): error CS1061: 'Module.Users?' does not contain a definition for 'Age' and no accessible extension method 'Age' accepting a first argument of type 'Module.Users?' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\grok-4\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\grok-4\\llm\\Lib.cs(25,14): error CS1061: 'Module.Users?' does not contain a definition for 'Active' and no accessible extension method 'Active' accepting a first argument of type 'Module.Users?' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\grok-4\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\grok-4\\llm\\Lib.cs(26,32): error CS1503: Argument 1: cannot convert from 'Module.Users?' to 'Module.Users' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\grok-4\\llm\\StdbModule.csproj]\r\n",
+ "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\grok-3-mini-beta\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(56,48): error CS0050: Inconsistent accessibility: return type 'IEnumerable' is less accessible than method 'results.Iter()' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\grok-3-mini-beta\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(57,35): error CS0050: Inconsistent accessibility: return type 'Module.Results' is less accessible than method 'results.Insert(Module.Results)' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\grok-3-mini-beta\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(57,35): error CS0051: Inconsistent accessibility: parameter type 'Module.Results' is less accessible than method 'results.Insert(Module.Results)' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\grok-3-mini-beta\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(58,17): error CS0051: Inconsistent accessibility: parameter type 'Module.Results' is less accessible than method 'results.Delete(Module.Results)' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\grok-3-mini-beta\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(60,25): error CS0060: Inconsistent accessibility: base class 'UniqueIndex' is less accessible than class 'results.IdUniqueIndex' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\grok-3-mini-beta\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(65,36): error CS0050: Inconsistent accessibility: return type 'Module.Results?' is less accessible than method 'results.IdUniqueIndex.Find(int)' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\grok-3-mini-beta\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(66,35): error CS0050: Inconsistent accessibility: return type 'Module.Results' is less accessible than method 'results.IdUniqueIndex.Update(Module.Results)' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\grok-3-mini-beta\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(66,35): error CS0051: Inconsistent accessibility: parameter type 'Module.Results' is less accessible than method 'results.IdUniqueIndex.Update(Module.Results)' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\grok-3-mini-beta\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'results' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\n",
"phase": "build_or_publish"
}
}
},
"vendor": "xai",
- "started_at": "2025-10-21T01:13:55.219200200Z",
- "finished_at": "2025-10-21T01:14:52.587564400Z"
+ "started_at": "2025-10-21T01:20:27.944261500Z",
+ "finished_at": "2025-10-21T01:20:41.228179900Z"
},
- "t_011_helper_function": {
+ "t_012_spacetime_product_type": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_011_helper_function",
+ "task": "t_012_spacetime_product_type",
"lang": "csharp",
"golden_published": true,
- "model_name": "Grok 4",
+ "model_name": "Grok 3 Mini (Beta)",
"total_tests": 3,
"passed_tests": 3,
"llm_output": null,
- "category": "basics",
- "route_api_model": "grok-4",
- "golden_db": "basics-t-011-helper-function-golden",
- "llm_db": "basics-t-011-helper-function-grok-4-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\grok-4\\llm",
+ "category": "schema",
+ "route_api_model": "grok-3-mini",
+ "golden_db": "schema-t-012-spacetime-product-type-golden",
+ "llm_db": "schema-t-012-spacetime-product-type-grok-3-mini-beta-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\grok-3-mini-beta\\llm",
"scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-011-helper-function-golden",
- "llm_db": "basics-t-011-helper-function-grok-4-llm",
+ "golden_db": "schema-t-012-spacetime-product-type-golden",
+ "llm_db": "schema-t-012-spacetime-product-type-grok-3-mini-beta-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -6867,7 +7681,7 @@
"tables_equal": true
}
},
- "helper_func_sum_parity": {
+ "product_type_row_parity": {
"pass": true,
"partial": 1.0,
"notes": {
@@ -6876,58 +7690,51 @@
2,
3
],
- "golden_db": "basics-t-011-helper-function-golden",
- "golden_out": "Id | Sum ----+----- 1 | 5",
- "llm_db": "basics-t-011-helper-function-grok-4-llm",
- "llm_out": "Id | Sum ----+----- 1 | 5",
- "query": "SELECT Id, Sum FROM results WHERE Id=1",
- "reducer": "ComputeSum",
+ "golden_db": "schema-t-012-spacetime-product-type-golden",
+ "golden_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)",
+ "llm_db": "schema-t-012-spacetime-product-type-grok-3-mini-beta-llm",
+ "llm_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)",
+ "query": "SELECT Id, Value FROM results WHERE Id=1",
+ "reducer": "SetScore",
"server": "local"
}
},
- "helper_func_sum_abs": {
+ "product_type_row_count": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1 AND Sum=5"
+ "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1"
}
}
},
"vendor": "xai",
- "started_at": "2025-10-21T01:13:57.232654500Z",
- "finished_at": "2025-10-21T01:15:30.130814500Z"
+ "started_at": "2025-10-21T01:23:30.794831300Z",
+ "finished_at": "2025-10-21T01:24:40.267312200Z"
},
- "t_005_update": {
+ "t_013_spacetime_sum_type": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_005_update",
+ "task": "t_013_spacetime_sum_type",
"lang": "csharp",
"golden_published": true,
- "model_name": "Grok 4",
+ "model_name": "Grok 3 Mini (Beta)",
"total_tests": 3,
"passed_tests": 3,
"llm_output": null,
- "category": "basics",
- "route_api_model": "grok-4",
- "golden_db": "basics-t-005-update-golden",
- "llm_db": "basics-t-005-update-grok-4-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_005_update\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_005_update\\csharp\\server\\grok-4\\llm",
+ "category": "schema",
+ "route_api_model": "grok-3-mini",
+ "golden_db": "schema-t-013-spacetime-sum-type-golden",
+ "llm_db": "schema-t-013-spacetime-sum-type-grok-3-mini-beta-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\grok-3-mini-beta\\llm",
"scorer_details": {
- "seed_users_row": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
- }
- },
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-005-update-golden",
- "llm_db": "basics-t-005-update-grok-4-llm",
+ "golden_db": "schema-t-013-spacetime-sum-type-golden",
+ "llm_db": "schema-t-013-spacetime-sum-type-grok-3-mini-beta-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -6935,332 +7742,65 @@
"tables_equal": true
}
},
- "data_parity_update_user": {
+ "sum_type_row_count": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1"
+ }
+ },
+ "sum_type_row_parity": {
"pass": true,
"partial": 1.0,
"notes": {
"args": [
1,
- "Alice2",
- 31,
- false
+ 10
],
- "golden_db": "basics-t-005-update-golden",
- "golden_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
- "llm_db": "basics-t-005-update-grok-4-llm",
- "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
- "query": "SELECT Id, Name, Age, Active FROM users WHERE Id=1",
- "reducer": "UpdateUser",
+ "golden_db": "schema-t-013-spacetime-sum-type-golden",
+ "golden_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))",
+ "llm_db": "schema-t-013-spacetime-sum-type-grok-3-mini-beta-llm",
+ "llm_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))",
+ "query": "SELECT Id, Value FROM results WHERE Id=1",
+ "reducer": "SetCircle",
"server": "local"
}
}
},
"vendor": "xai",
- "started_at": "2025-10-21T01:13:54.183350100Z",
- "finished_at": "2025-10-21T01:15:34.745090800Z"
+ "started_at": "2025-10-21T01:23:31.355572800Z",
+ "finished_at": "2025-10-21T01:24:42.115504300Z"
},
- "t_006_delete": {
+ "t_014_elementary_columns": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_006_delete",
+ "task": "t_014_elementary_columns",
"lang": "csharp",
"golden_published": true,
- "model_name": "Grok 4",
+ "model_name": "Grok 3 Mini (Beta)",
"total_tests": 3,
- "passed_tests": 3,
- "llm_output": null,
- "category": "basics",
- "route_api_model": "grok-4",
- "golden_db": "basics-t-006-delete-golden",
- "llm_db": "basics-t-006-delete-grok-4-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\grok-4\\llm",
- "scorer_details": {
- "seed_users_row": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
- }
- },
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "golden_db": "basics-t-006-delete-golden",
- "llm_db": "basics-t-006-delete-grok-4-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
- }
- },
- "delete_user_count_zero": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 0,
- "expected": 0,
- "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=1"
- }
- }
- },
- "vendor": "xai",
- "started_at": "2025-10-21T01:13:54.697276100Z",
- "finished_at": "2025-10-21T01:15:29.604568200Z"
- },
- "t_017_scheduled_columns": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_017_scheduled_columns",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "Grok 4",
- "total_tests": 2,
- "passed_tests": 2,
- "llm_output": null,
- "category": "schema",
- "route_api_model": "grok-4",
- "golden_db": "schema-t-017-scheduled-columns-golden",
- "llm_db": "schema-t-017-scheduled-columns-grok-4-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_017_scheduled_columns\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_017_scheduled_columns\\csharp\\server\\grok-4\\llm",
- "scorer_details": {
- "scheduled_seeded_one_row": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM tick_timer WHERE ScheduledId>=0"
- }
- },
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "golden_db": "schema-t-017-scheduled-columns-golden",
- "llm_db": "schema-t-017-scheduled-columns-grok-4-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
- }
- }
- },
- "vendor": "xai",
- "started_at": "2025-10-21T00:54:59.124579600Z",
- "finished_at": "2025-10-21T00:56:32.470601800Z"
- },
- "t_004_insert": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_004_insert",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "Grok 4",
- "total_tests": 2,
- "passed_tests": 2,
- "llm_output": null,
- "category": "basics",
- "route_api_model": "grok-4",
- "golden_db": "basics-t-004-insert-golden",
- "llm_db": "basics-t-004-insert-grok-4-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_004_insert\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_004_insert\\csharp\\server\\grok-4\\llm",
- "scorer_details": {
- "data_parity_insert_user": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "args": [
- 1,
- "Alice",
- 30,
- true
- ],
- "golden_db": "basics-t-004-insert-golden",
- "golden_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true",
- "llm_db": "basics-t-004-insert-grok-4-llm",
- "llm_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true",
- "query": "SELECT Id, Name, Age, Active FROM users WHERE Id=1",
- "reducer": "InsertUser",
- "server": "local"
- }
- },
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "golden_db": "basics-t-004-insert-golden",
- "llm_db": "basics-t-004-insert-grok-4-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
- }
- }
- },
- "vendor": "xai",
- "started_at": "2025-10-21T01:13:53.656401800Z",
- "finished_at": "2025-10-21T01:15:06.998060800Z"
- },
- "t_021_multi_column_index": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_021_multi_column_index",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "Grok 4",
- "total_tests": 4,
- "passed_tests": 4,
+ "passed_tests": 0,
"llm_output": null,
"category": "schema",
- "route_api_model": "grok-4",
- "golden_db": "schema-t-021-multi-column-index-golden",
- "llm_db": "schema-t-021-multi-column-index-grok-4-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\grok-4\\llm",
- "scorer_details": {
- "mcindex_lookup_u7_d1": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM logs WHERE UserId=7 AND Day=1"
- }
- },
- "mcindex_seed_count": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 3,
- "expected": 3,
- "sql": "SELECT COUNT(*) AS n FROM logs"
- }
- },
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "golden_db": "schema-t-021-multi-column-index-golden",
- "llm_db": "schema-t-021-multi-column-index-grok-4-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
- }
- },
- "mcindex_lookup_u7_d2": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM logs WHERE UserId=7 AND Day=2"
- }
- }
- },
- "vendor": "xai",
- "started_at": "2025-10-21T00:55:01.339990400Z",
- "finished_at": "2025-10-21T00:56:41.639356300Z"
- },
- "t_009_init": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_009_init",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "Grok 4",
- "total_tests": 4,
- "passed_tests": 4,
- "llm_output": null,
- "category": "basics",
- "route_api_model": "grok-4",
- "golden_db": "basics-t-009-init-golden",
- "llm_db": "basics-t-009-init-grok-4-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_009_init\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_009_init\\csharp\\server\\grok-4\\llm",
- "scorer_details": {
- "init_seed_alice": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true"
- }
- },
- "init_total_two": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 2,
- "expected": 2,
- "sql": "SELECT COUNT(*) AS n FROM users"
- }
- },
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "golden_db": "basics-t-009-init-golden",
- "llm_db": "basics-t-009-init-grok-4-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
- }
- },
- "init_seed_bob": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false"
- }
- }
- },
- "vendor": "xai",
- "started_at": "2025-10-21T01:13:56.243496200Z",
- "finished_at": "2025-10-21T01:15:45.919553400Z"
- }
- }
- },
- {
- "name": "Grok 3 Mini (Beta)",
- "route_api_model": "grok-3-mini",
- "tasks": {
- "t_005_update": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_005_update",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "Grok 3 Mini (Beta)",
- "total_tests": 3,
- "passed_tests": 0,
- "llm_output": null,
- "category": "basics",
"route_api_model": "grok-3-mini",
- "golden_db": "basics-t-005-update-golden",
- "llm_db": "basics-t-005-update-grok-3-mini-beta-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_005_update\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_005_update\\csharp\\server\\grok-3-mini-beta\\llm",
+ "golden_db": "schema-t-014-elementary-columns-golden",
+ "llm_db": "schema-t-014-elementary-columns-grok-3-mini-beta-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_014_elementary_columns\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_014_elementary_columns\\csharp\\server\\grok-3-mini-beta\\llm",
"scorer_details": {
"publish_error": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_005_update\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(14,1): error CS8803: Top-level statements must precede namespace and type declarations. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_005_update\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_005_update\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(15,1): error CS0106: The modifier 'public' is not valid for this item [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_005_update\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\n",
+ "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_014_elementary_columns\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(17,1): error CS8803: Top-level statements must precede namespace and type declarations. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_014_elementary_columns\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_014_elementary_columns\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(18,1): error CS0106: The modifier 'public' is not valid for this item [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_014_elementary_columns\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\n",
"phase": "build_or_publish"
}
}
},
"vendor": "xai",
- "started_at": "2025-10-21T01:20:24.865269700Z",
- "finished_at": "2025-10-21T01:20:43.203415300Z"
+ "started_at": "2025-10-21T01:23:31.889084400Z",
+ "finished_at": "2025-10-21T01:23:46.420570100Z"
},
"t_015_product_type_columns": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
@@ -7319,68 +7859,34 @@
"started_at": "2025-10-21T01:23:32.438519600Z",
"finished_at": "2025-10-21T01:24:41.151609300Z"
},
- "t_011_helper_function": {
+ "t_016_sum_type_columns": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_011_helper_function",
+ "task": "t_016_sum_type_columns",
"lang": "csharp",
"golden_published": true,
"model_name": "Grok 3 Mini (Beta)",
"total_tests": 3,
"passed_tests": 0,
"llm_output": null,
- "category": "basics",
+ "category": "schema",
"route_api_model": "grok-3-mini",
- "golden_db": "basics-t-011-helper-function-golden",
- "llm_db": "basics-t-011-helper-function-grok-3-mini-beta-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\grok-3-mini-beta\\llm",
+ "golden_db": "schema-t-016-sum-type-columns-golden",
+ "llm_db": "schema-t-016-sum-type-columns-grok-3-mini-beta-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\grok-3-mini-beta\\llm",
"scorer_details": {
"publish_error": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\grok-3-mini-beta\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(56,48): error CS0050: Inconsistent accessibility: return type 'IEnumerable' is less accessible than method 'results.Iter()' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\grok-3-mini-beta\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(57,35): error CS0050: Inconsistent accessibility: return type 'Module.Results' is less accessible than method 'results.Insert(Module.Results)' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\grok-3-mini-beta\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(57,35): error CS0051: Inconsistent accessibility: parameter type 'Module.Results' is less accessible than method 'results.Insert(Module.Results)' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\grok-3-mini-beta\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(58,17): error CS0051: Inconsistent accessibility: parameter type 'Module.Results' is less accessible than method 'results.Delete(Module.Results)' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\grok-3-mini-beta\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(60,25): error CS0060: Inconsistent accessibility: base class 'UniqueIndex' is less accessible than class 'results.IdUniqueIndex' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\grok-3-mini-beta\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(65,36): error CS0050: Inconsistent accessibility: return type 'Module.Results?' is less accessible than method 'results.IdUniqueIndex.Find(int)' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\grok-3-mini-beta\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(66,35): error CS0050: Inconsistent accessibility: return type 'Module.Results' is less accessible than method 'results.IdUniqueIndex.Update(Module.Results)' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\grok-3-mini-beta\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(66,35): error CS0051: Inconsistent accessibility: parameter type 'Module.Results' is less accessible than method 'results.IdUniqueIndex.Update(Module.Results)' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\grok-3-mini-beta\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'results' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\n",
+ "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(29,1): error CS8803: Top-level statements must precede namespace and type declarations. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(30,1): error CS0106: The modifier 'public' is not valid for this item [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\n",
"phase": "build_or_publish"
}
}
},
"vendor": "xai",
- "started_at": "2025-10-21T01:20:27.944261500Z",
- "finished_at": "2025-10-21T01:20:41.228179900Z"
- },
- "t_001_basic_tables": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_001_basic_tables",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "Grok 3 Mini (Beta)",
- "total_tests": 1,
- "passed_tests": 1,
- "llm_output": null,
- "category": "basics",
- "route_api_model": "grok-3-mini",
- "golden_db": "basics-t-001-basic-tables-golden",
- "llm_db": "basics-t-001-basic-tables-grok-3-mini-beta-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\grok-3-mini-beta\\llm",
- "scorer_details": {
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "golden_db": "basics-t-001-basic-tables-golden",
- "llm_db": "basics-t-001-basic-tables-grok-3-mini-beta-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
- }
- }
- },
- "vendor": "xai",
- "started_at": "2025-10-21T01:20:22.848063200Z",
- "finished_at": "2025-10-21T01:21:42.592621100Z"
+ "started_at": "2025-10-21T01:23:32.976426200Z",
+ "finished_at": "2025-10-21T01:23:48.720322600Z"
},
"t_017_scheduled_columns": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
@@ -7411,57 +7917,28 @@
"started_at": "2025-10-21T01:23:33.505002300Z",
"finished_at": "2025-10-21T01:23:51.641209600Z"
},
- "t_007_crud": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_007_crud",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "Grok 3 Mini (Beta)",
- "total_tests": 4,
- "passed_tests": 0,
- "llm_output": null,
- "category": "basics",
- "route_api_model": "grok-3-mini",
- "golden_db": "basics-t-007-crud-golden",
- "llm_db": "basics-t-007-crud-grok-3-mini-beta-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\grok-3-mini-beta\\llm",
- "scorer_details": {
- "publish_error": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(14,1): error CS8803: Top-level statements must precede namespace and type declarations. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(15,1): error CS0106: The modifier 'public' is not valid for this item [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\n",
- "phase": "build_or_publish"
- }
- }
- },
- "vendor": "xai",
- "started_at": "2025-10-21T01:20:25.895347300Z",
- "finished_at": "2025-10-21T01:20:44.275623400Z"
- },
- "t_004_insert": {
+ "t_018_constraints": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_004_insert",
+ "task": "t_018_constraints",
"lang": "csharp",
"golden_published": true,
"model_name": "Grok 3 Mini (Beta)",
- "total_tests": 2,
- "passed_tests": 2,
+ "total_tests": 3,
+ "passed_tests": 3,
"llm_output": null,
- "category": "basics",
+ "category": "schema",
"route_api_model": "grok-3-mini",
- "golden_db": "basics-t-004-insert-golden",
- "llm_db": "basics-t-004-insert-grok-3-mini-beta-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_004_insert\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_004_insert\\csharp\\server\\grok-3-mini-beta\\llm",
+ "golden_db": "schema-t-018-constraints-golden",
+ "llm_db": "schema-t-018-constraints-grok-3-mini-beta-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\grok-3-mini-beta\\llm",
"scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-004-insert-golden",
- "llm_db": "basics-t-004-insert-grok-3-mini-beta-llm",
+ "golden_db": "schema-t-018-constraints-golden",
+ "llm_db": "schema-t-018-constraints-grok-3-mini-beta-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -7469,46 +7946,6 @@
"tables_equal": true
}
},
- "data_parity_insert_user": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "args": [
- 1,
- "Alice",
- 30,
- true
- ],
- "golden_db": "basics-t-004-insert-golden",
- "golden_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true",
- "llm_db": "basics-t-004-insert-grok-3-mini-beta-llm",
- "llm_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true",
- "query": "SELECT Id, Name, Age, Active FROM users WHERE Id=1",
- "reducer": "InsertUser",
- "server": "local"
- }
- }
- },
- "vendor": "xai",
- "started_at": "2025-10-21T01:20:24.355687300Z",
- "finished_at": "2025-10-21T01:21:39.029068800Z"
- },
- "t_018_constraints": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_018_constraints",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "Grok 3 Mini (Beta)",
- "total_tests": 3,
- "passed_tests": 3,
- "llm_output": null,
- "category": "schema",
- "route_api_model": "grok-3-mini",
- "golden_db": "schema-t-018-constraints-golden",
- "llm_db": "schema-t-018-constraints-grok-3-mini-beta-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\grok-3-mini-beta\\llm",
- "scorer_details": {
"constraints_row_parity_after_seed": {
"pass": true,
"partial": 1.0,
@@ -7531,25 +7968,41 @@
"expected": 1,
"sql": "SELECT COUNT(*) AS n FROM accounts WHERE Id=2"
}
- },
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "golden_db": "schema-t-018-constraints-golden",
- "llm_db": "schema-t-018-constraints-grok-3-mini-beta-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
- }
}
},
"vendor": "xai",
"started_at": "2025-10-21T01:23:34.018434300Z",
"finished_at": "2025-10-21T01:24:40.728727200Z"
},
+ "t_019_many_to_many": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_019_many_to_many",
+ "lang": "csharp",
+ "golden_published": true,
+ "model_name": "Grok 3 Mini (Beta)",
+ "total_tests": 5,
+ "passed_tests": 0,
+ "llm_output": null,
+ "category": "schema",
+ "route_api_model": "grok-3-mini",
+ "golden_db": "schema-t-019-many-to-many-golden",
+ "llm_db": "schema-t-019-many-to-many-grok-3-mini-beta-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\grok-3-mini-beta\\llm",
+ "scorer_details": {
+ "publish_error": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(31,1): error CS8803: Top-level statements must precede namespace and type declarations. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(32,1): error CS0106: The modifier 'public' is not valid for this item [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\n",
+ "phase": "build_or_publish"
+ }
+ }
+ },
+ "vendor": "xai",
+ "started_at": "2025-10-21T01:23:34.554684Z",
+ "finished_at": "2025-10-21T01:23:46.881244300Z"
+ },
"t_020_ecs": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
"task": "t_020_ecs",
@@ -7566,24 +8019,6 @@
"work_dir_golden": "target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\golden",
"work_dir_llm": "target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\grok-3-mini-beta\\llm",
"scorer_details": {
- "ecs_next_pos_entity1": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM next_positions WHERE EntityId=1 AND X=1 AND Y=0"
- }
- },
- "ecs_step_next_positions_count": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 2,
- "expected": 2,
- "sql": "SELECT COUNT(*) AS n FROM next_positions"
- }
- },
"ecs_seed_positions_count": {
"pass": true,
"partial": 1.0,
@@ -7614,182 +8049,189 @@
"expected": 1,
"sql": "SELECT COUNT(*) AS n FROM next_positions WHERE EntityId=2 AND X=8 AND Y=3"
}
+ },
+ "ecs_step_next_positions_count": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 2,
+ "expected": 2,
+ "sql": "SELECT COUNT(*) AS n FROM next_positions"
+ }
+ },
+ "ecs_next_pos_entity1": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM next_positions WHERE EntityId=1 AND X=1 AND Y=0"
+ }
}
},
"vendor": "xai",
"started_at": "2025-10-21T01:23:35.115024700Z",
"finished_at": "2025-10-21T01:24:41.741813200Z"
},
- "t_002_scheduled_table": {
+ "t_021_multi_column_index": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_002_scheduled_table",
+ "task": "t_021_multi_column_index",
"lang": "csharp",
"golden_published": true,
"model_name": "Grok 3 Mini (Beta)",
- "total_tests": 1,
+ "total_tests": 4,
"passed_tests": 0,
"llm_output": null,
- "category": "basics",
+ "category": "schema",
"route_api_model": "grok-3-mini",
- "golden_db": "basics-t-002-scheduled-table-golden",
- "llm_db": "basics-t-002-scheduled-table-grok-3-mini-beta-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_002_scheduled_table\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_002_scheduled_table\\csharp\\server\\grok-3-mini-beta\\llm",
+ "golden_db": "schema-t-021-multi-column-index-golden",
+ "llm_db": "schema-t-021-multi-column-index-grok-3-mini-beta-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\grok-3-mini-beta\\llm",
"scorer_details": {
"publish_error": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_002_scheduled_table\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(3,19): error CS0234: The type or namespace name 'Types' does not exist in the namespace 'SpacetimeDB' (are you missing an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_002_scheduled_table\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_002_scheduled_table\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(5,60): error CS0103: The name 'Tick' does not exist in the current context [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_002_scheduled_table\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\n",
+ "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(2,7): error CS0246: The type or namespace name 'Spacetimedb' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(18,29): error CS0246: The type or namespace name 'ReducerContext' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(17,6): error CS0246: The type or namespace name 'Spacetimedb' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(6,6): error CS0246: The type or namespace name 'Spacetimedb' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(9,10): error CS0246: The type or namespace name 'Spacetimedb' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(11,10): error CS0246: The type or namespace name 'Spacetimedb' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(11,66): error CS9176: There is no target type for the collection expression. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\n",
"phase": "build_or_publish"
}
}
},
"vendor": "xai",
- "started_at": "2025-10-21T01:20:23.355931300Z",
- "finished_at": "2025-10-21T01:20:46.972683Z"
- },
- "t_009_init": {
+ "started_at": "2025-10-21T01:23:35.680781500Z",
+ "finished_at": "2025-10-21T01:23:46.401438300Z"
+ }
+ }
+ },
+ {
+ "name": "Gemini 2.5 Pro",
+ "route_api_model": "gemini-2.5-pro",
+ "tasks": {
+ "t_000_empty_reducers": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_009_init",
+ "task": "t_000_empty_reducers",
"lang": "csharp",
"golden_published": true,
- "model_name": "Grok 3 Mini (Beta)",
- "total_tests": 4,
- "passed_tests": 4,
- "llm_output": null,
+ "model_name": "Gemini 2.5 Pro",
+ "total_tests": 1,
+ "passed_tests": 1,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Reducer]\n public static void EmptyReducer_NoArgs(ReducerContext ctx)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithInt(ReducerContext ctx, int count)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithString(ReducerContext ctx, string name)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithTwoArgs(ReducerContext ctx, int count, string name)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithThreeArgs(ReducerContext ctx, bool active, float ratio, string label)\n {\n }\n}",
"category": "basics",
- "route_api_model": "grok-3-mini",
- "golden_db": "basics-t-009-init-golden",
- "llm_db": "basics-t-009-init-grok-3-mini-beta-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_009_init\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_009_init\\csharp\\server\\grok-3-mini-beta\\llm",
+ "route_api_model": "gemini-2.5-pro",
+ "golden_db": "basics-t-000-empty-reducers-golden",
+ "llm_db": "basics-t-000-empty-reducers-gemini-2-5-pro-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\gemini-2-5-pro\\llm",
"scorer_details": {
- "init_seed_alice": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true"
- }
- },
- "init_seed_bob": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false"
- }
- },
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-009-init-golden",
- "llm_db": "basics-t-009-init-grok-3-mini-beta-llm",
+ "golden_db": "basics-t-000-empty-reducers-golden",
+ "llm_db": "basics-t-000-empty-reducers-gemini-2-5-pro-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
"tables_diff": null,
"tables_equal": true
}
- },
- "init_total_two": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 2,
- "expected": 2,
- "sql": "SELECT COUNT(*) AS n FROM users"
- }
}
},
- "vendor": "xai",
- "started_at": "2025-10-21T01:20:26.945205600Z",
- "finished_at": "2025-10-21T01:21:40.007191100Z"
+ "vendor": "google",
+ "started_at": "2025-10-21T17:53:27.205121400Z",
+ "finished_at": "2025-10-21T17:55:00.557516300Z"
},
- "t_019_many_to_many": {
+ "t_001_basic_tables": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_019_many_to_many",
+ "task": "t_001_basic_tables",
"lang": "csharp",
"golden_published": true,
- "model_name": "Grok 3 Mini (Beta)",
- "total_tests": 5,
- "passed_tests": 0,
- "llm_output": null,
- "category": "schema",
- "route_api_model": "grok-3-mini",
- "golden_db": "schema-t-019-many-to-many-golden",
- "llm_db": "schema-t-019-many-to-many-grok-3-mini-beta-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\grok-3-mini-beta\\llm",
+ "model_name": "Gemini 2.5 Pro",
+ "total_tests": 1,
+ "passed_tests": 1,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\")]\n public partial struct Users\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"products\")]\n public partial struct Products\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Title;\n public float Price;\n public bool InStock;\n }\n\n [SpacetimeDB.Table(Name = \"notes\")]\n public partial struct Notes\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Body;\n public long Rating;\n public bool Pinned;\n }\n}\n",
+ "category": "basics",
+ "route_api_model": "gemini-2.5-pro",
+ "golden_db": "basics-t-001-basic-tables-golden",
+ "llm_db": "basics-t-001-basic-tables-gemini-2-5-pro-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\gemini-2-5-pro\\llm",
"scorer_details": {
- "publish_error": {
- "pass": false,
- "partial": 0.0,
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
"notes": {
- "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(31,1): error CS8803: Top-level statements must precede namespace and type declarations. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(32,1): error CS0106: The modifier 'public' is not valid for this item [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\n",
- "phase": "build_or_publish"
+ "golden_db": "basics-t-001-basic-tables-golden",
+ "llm_db": "basics-t-001-basic-tables-gemini-2-5-pro-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
}
}
},
- "vendor": "xai",
- "started_at": "2025-10-21T01:23:34.554684Z",
- "finished_at": "2025-10-21T01:23:46.881244300Z"
+ "vendor": "google",
+ "started_at": "2025-10-21T17:53:27.283937900Z",
+ "finished_at": "2025-10-21T17:55:01.828931400Z"
},
- "t_014_elementary_columns": {
+ "t_002_scheduled_table": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_014_elementary_columns",
+ "task": "t_002_scheduled_table",
"lang": "csharp",
"golden_published": true,
- "model_name": "Grok 3 Mini (Beta)",
- "total_tests": 3,
- "passed_tests": 0,
- "llm_output": null,
- "category": "schema",
- "route_api_model": "grok-3-mini",
- "golden_db": "schema-t-014-elementary-columns-golden",
- "llm_db": "schema-t-014-elementary-columns-grok-3-mini-beta-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_014_elementary_columns\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_014_elementary_columns\\csharp\\server\\grok-3-mini-beta\\llm",
+ "model_name": "Gemini 2.5 Pro",
+ "total_tests": 1,
+ "passed_tests": 1,
+ "llm_output": "```csharp\nusing SpacetimeDB;\nusing System;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"tick_timer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public ulong ScheduledId;\n public SpacetimeDB.ScheduleAt ScheduledAt;\n }\n\n [SpacetimeDB.Reducer]\n public static void Tick(ReducerContext ctx, TickTimer timer)\n {\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.tick_timer.Insert(new TickTimer\n {\n ScheduledId = 0,\n ScheduledAt = new SpacetimeDB.ScheduleAt.Interval(TimeSpan.FromMicroseconds(50_000)),\n });\n }\n}\n```",
+ "category": "basics",
+ "route_api_model": "gemini-2.5-pro",
+ "golden_db": "basics-t-002-scheduled-table-golden",
+ "llm_db": "basics-t-002-scheduled-table-gemini-2-5-pro-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_002_scheduled_table\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_002_scheduled_table\\csharp\\server\\gemini-2-5-pro\\llm",
"scorer_details": {
- "publish_error": {
- "pass": false,
- "partial": 0.0,
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
"notes": {
- "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_014_elementary_columns\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(17,1): error CS8803: Top-level statements must precede namespace and type declarations. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_014_elementary_columns\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_014_elementary_columns\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(18,1): error CS0106: The modifier 'public' is not valid for this item [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_014_elementary_columns\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\n",
- "phase": "build_or_publish"
+ "golden_db": "basics-t-002-scheduled-table-golden",
+ "llm_db": "basics-t-002-scheduled-table-gemini-2-5-pro-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
}
}
},
- "vendor": "xai",
- "started_at": "2025-10-21T01:23:31.889084400Z",
- "finished_at": "2025-10-21T01:23:46.420570100Z"
+ "vendor": "google",
+ "started_at": "2025-10-21T17:53:27.329794400Z",
+ "finished_at": "2025-10-21T17:55:47.101376900Z"
},
"t_003_struct_in_table": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
"task": "t_003_struct_in_table",
"lang": "csharp",
"golden_published": true,
- "model_name": "Grok 3 Mini (Beta)",
+ "model_name": "Gemini 2.5 Pro",
"total_tests": 1,
"passed_tests": 1,
- "llm_output": null,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"entities\")]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n\n public Position Pos;\n }\n}",
"category": "basics",
- "route_api_model": "grok-3-mini",
+ "route_api_model": "gemini-2.5-pro",
"golden_db": "basics-t-003-struct-in-table-golden",
- "llm_db": "basics-t-003-struct-in-table-grok-3-mini-beta-llm",
+ "llm_db": "basics-t-003-struct-in-table-gemini-2-5-pro-llm",
"work_dir_golden": "target\\llm-runs\\basics\\t_003_struct_in_table\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_003_struct_in_table\\csharp\\server\\grok-3-mini-beta\\llm",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_003_struct_in_table\\csharp\\server\\gemini-2-5-pro\\llm",
"scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
"golden_db": "basics-t-003-struct-in-table-golden",
- "llm_db": "basics-t-003-struct-in-table-grok-3-mini-beta-llm",
+ "llm_db": "basics-t-003-struct-in-table-gemini-2-5-pro-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -7798,58 +8240,51 @@
}
}
},
- "vendor": "xai",
- "started_at": "2025-10-21T01:20:23.854245800Z",
- "finished_at": "2025-10-21T01:21:38.441196600Z"
+ "vendor": "google",
+ "started_at": "2025-10-21T17:53:27.376426300Z",
+ "finished_at": "2025-10-21T17:55:36.051605600Z"
},
- "t_013_spacetime_sum_type": {
+ "t_004_insert": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_013_spacetime_sum_type",
+ "task": "t_004_insert",
"lang": "csharp",
"golden_published": true,
- "model_name": "Grok 3 Mini (Beta)",
- "total_tests": 3,
- "passed_tests": 3,
- "llm_output": null,
- "category": "schema",
- "route_api_model": "grok-3-mini",
- "golden_db": "schema-t-013-spacetime-sum-type-golden",
- "llm_db": "schema-t-013-spacetime-sum-type-grok-3-mini-beta-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\grok-3-mini-beta\\llm",
+ "model_name": "Gemini 2.5 Pro",
+ "total_tests": 2,
+ "passed_tests": 2,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void InsertUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n ctx.Db.users.Insert(new User\n {\n Id = id,\n Name = name,\n Age = age,\n Active = active,\n });\n }\n}",
+ "category": "basics",
+ "route_api_model": "gemini-2.5-pro",
+ "golden_db": "basics-t-004-insert-golden",
+ "llm_db": "basics-t-004-insert-gemini-2-5-pro-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_004_insert\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_004_insert\\csharp\\server\\gemini-2-5-pro\\llm",
"scorer_details": {
- "sum_type_row_parity": {
+ "data_parity_insert_user": {
"pass": true,
"partial": 1.0,
"notes": {
"args": [
1,
- 10
+ "Alice",
+ 30,
+ true
],
- "golden_db": "schema-t-013-spacetime-sum-type-golden",
- "golden_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))",
- "llm_db": "schema-t-013-spacetime-sum-type-grok-3-mini-beta-llm",
- "llm_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))",
- "query": "SELECT Id, Value FROM results WHERE Id=1",
- "reducer": "SetCircle",
+ "golden_db": "basics-t-004-insert-golden",
+ "golden_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true",
+ "llm_db": "basics-t-004-insert-gemini-2-5-pro-llm",
+ "llm_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true",
+ "query": "SELECT Id, Name, Age, Active FROM users WHERE Id=1",
+ "reducer": "InsertUser",
"server": "local"
}
},
- "sum_type_row_count": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1"
- }
- },
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-013-spacetime-sum-type-golden",
- "llm_db": "schema-t-013-spacetime-sum-type-grok-3-mini-beta-llm",
+ "golden_db": "basics-t-004-insert-golden",
+ "llm_db": "basics-t-004-insert-gemini-2-5-pro-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -7858,90 +8293,92 @@
}
}
},
- "vendor": "xai",
- "started_at": "2025-10-21T01:23:31.355572800Z",
- "finished_at": "2025-10-21T01:24:42.115504300Z"
+ "vendor": "google",
+ "started_at": "2025-10-21T17:53:27.427049100Z",
+ "finished_at": "2025-10-21T17:55:05.566960600Z"
},
- "t_016_sum_type_columns": {
+ "t_005_update": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_016_sum_type_columns",
+ "task": "t_005_update",
"lang": "csharp",
"golden_published": true,
- "model_name": "Grok 3 Mini (Beta)",
+ "model_name": "Gemini 2.5 Pro",
"total_tests": 3,
- "passed_tests": 0,
- "llm_output": null,
- "category": "schema",
- "route_api_model": "grok-3-mini",
- "golden_db": "schema-t-016-sum-type-columns-golden",
- "llm_db": "schema-t-016-sum-type-columns-grok-3-mini-beta-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\grok-3-mini-beta\\llm",
+ "passed_tests": 3,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n User user = new User\n {\n Id = id,\n Name = name,\n Age = age,\n Active = active,\n };\n ctx.Db.users.Id.Update(user);\n }\n}",
+ "category": "basics",
+ "route_api_model": "gemini-2.5-pro",
+ "golden_db": "basics-t-005-update-golden",
+ "llm_db": "basics-t-005-update-gemini-2-5-pro-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_005_update\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_005_update\\csharp\\server\\gemini-2-5-pro\\llm",
"scorer_details": {
- "publish_error": {
- "pass": false,
- "partial": 0.0,
+ "seed_users_row": {
+ "pass": true,
+ "partial": 1.0,
"notes": {
- "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(29,1): error CS8803: Top-level statements must precede namespace and type declarations. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(30,1): error CS0106: The modifier 'public' is not valid for this item [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\n",
- "phase": "build_or_publish"
+ "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
}
- }
- },
- "vendor": "xai",
- "started_at": "2025-10-21T01:23:32.976426200Z",
- "finished_at": "2025-10-21T01:23:48.720322600Z"
- },
- "t_008_index_lookup": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_008_index_lookup",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "Grok 3 Mini (Beta)",
- "total_tests": 3,
- "passed_tests": 0,
- "llm_output": null,
- "category": "basics",
- "route_api_model": "grok-3-mini",
- "golden_db": "basics-t-008-index-lookup-golden",
- "llm_db": "basics-t-008-index-lookup-grok-3-mini-beta-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\grok-3-mini-beta\\llm",
- "scorer_details": {
- "publish_error": {
- "pass": false,
- "partial": 0.0,
+ },
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
"notes": {
- "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(22,1): error CS8803: Top-level statements must precede namespace and type declarations. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(23,1): error CS0106: The modifier 'public' is not valid for this item [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\n",
- "phase": "build_or_publish"
+ "golden_db": "basics-t-005-update-golden",
+ "llm_db": "basics-t-005-update-gemini-2-5-pro-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
+ }
+ },
+ "data_parity_update_user": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "args": [
+ 1,
+ "Alice2",
+ 31,
+ false
+ ],
+ "golden_db": "basics-t-005-update-golden",
+ "golden_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
+ "llm_db": "basics-t-005-update-gemini-2-5-pro-llm",
+ "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
+ "query": "SELECT Id, Name, Age, Active FROM users WHERE Id=1",
+ "reducer": "UpdateUser",
+ "server": "local"
}
}
},
- "vendor": "xai",
- "started_at": "2025-10-21T01:20:26.421931300Z",
- "finished_at": "2025-10-21T01:20:43.941929700Z"
+ "vendor": "google",
+ "started_at": "2025-10-21T17:53:27.478569400Z",
+ "finished_at": "2025-10-21T17:55:03.848315200Z"
},
- "t_012_spacetime_product_type": {
+ "t_006_delete": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_012_spacetime_product_type",
+ "task": "t_006_delete",
"lang": "csharp",
"golden_published": true,
- "model_name": "Grok 3 Mini (Beta)",
+ "model_name": "Gemini 2.5 Pro",
"total_tests": 3,
"passed_tests": 3,
- "llm_output": null,
- "category": "schema",
- "route_api_model": "grok-3-mini",
- "golden_db": "schema-t-012-spacetime-product-type-golden",
- "llm_db": "schema-t-012-spacetime-product-type-grok-3-mini-beta-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\grok-3-mini-beta\\llm",
+ "llm_output": "```csharp\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void DeleteUser(ReducerContext ctx, int id)\n {\n ctx.Db.users.Id.Delete(id);\n }\n}\n```",
+ "category": "basics",
+ "route_api_model": "gemini-2.5-pro",
+ "golden_db": "basics-t-006-delete-golden",
+ "llm_db": "basics-t-006-delete-gemini-2-5-pro-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\gemini-2-5-pro\\llm",
"scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-012-spacetime-product-type-golden",
- "llm_db": "schema-t-012-spacetime-product-type-grok-3-mini-beta-llm",
+ "golden_db": "basics-t-006-delete-golden",
+ "llm_db": "basics-t-006-delete-gemini-2-5-pro-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -7949,60 +8386,81 @@
"tables_equal": true
}
},
- "product_type_row_parity": {
+ "seed_users_row": {
"pass": true,
"partial": 1.0,
"notes": {
- "args": [
- 1,
- 2,
- 3
- ],
- "golden_db": "schema-t-012-spacetime-product-type-golden",
- "golden_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)",
- "llm_db": "schema-t-012-spacetime-product-type-grok-3-mini-beta-llm",
- "llm_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)",
- "query": "SELECT Id, Value FROM results WHERE Id=1",
- "reducer": "SetScore",
- "server": "local"
+ "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
}
},
- "product_type_row_count": {
+ "delete_user_count_zero": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1"
+ "actual": 0,
+ "expected": 0,
+ "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=1"
}
}
},
- "vendor": "xai",
- "started_at": "2025-10-21T01:23:30.794831300Z",
- "finished_at": "2025-10-21T01:24:40.267312200Z"
+ "vendor": "google",
+ "started_at": "2025-10-21T17:53:27.525358200Z",
+ "finished_at": "2025-10-21T17:55:01.144593Z"
},
- "t_010_connect": {
+ "t_007_crud": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_010_connect",
+ "task": "t_007_crud",
"lang": "csharp",
"golden_published": true,
- "model_name": "Grok 3 Mini (Beta)",
- "total_tests": 1,
- "passed_tests": 1,
- "llm_output": null,
+ "model_name": "Gemini 2.5 Pro",
+ "total_tests": 4,
+ "passed_tests": 4,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void Crud(ReducerContext ctx)\n {\n ctx.Db.users.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.users.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n ctx.Db.users.Id.Update(new User { Id = 1, Name = \"Alice2\", Age = 31, Active = false });\n ctx.Db.users.Id.Delete(2);\n }\n}",
"category": "basics",
- "route_api_model": "grok-3-mini",
- "golden_db": "basics-t-010-connect-golden",
- "llm_db": "basics-t-010-connect-grok-3-mini-beta-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_010_connect\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_010_connect\\csharp\\server\\grok-3-mini-beta\\llm",
+ "route_api_model": "gemini-2.5-pro",
+ "golden_db": "basics-t-007-crud-golden",
+ "llm_db": "basics-t-007-crud-gemini-2-5-pro-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\gemini-2-5-pro\\llm",
"scorer_details": {
+ "crud_row_id1_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "args": [],
+ "golden_db": "basics-t-007-crud-golden",
+ "golden_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
+ "llm_db": "basics-t-007-crud-gemini-2-5-pro-llm",
+ "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
+ "query": "SELECT Id, Name, Age, Active FROM users WHERE Id=1",
+ "reducer": "Crud",
+ "server": "local"
+ }
+ },
+ "crud_total_count_one": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM users"
+ }
+ },
+ "crud_row_id2_deleted": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 0,
+ "expected": 0,
+ "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=2"
+ }
+ },
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-010-connect-golden",
- "llm_db": "basics-t-010-connect-grok-3-mini-beta-llm",
+ "golden_db": "basics-t-007-crud-golden",
+ "llm_db": "basics-t-007-crud-gemini-2-5-pro-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -8011,210 +8469,107 @@
}
}
},
- "vendor": "xai",
- "started_at": "2025-10-21T01:20:27.445891100Z",
- "finished_at": "2025-10-21T01:21:42.754747300Z"
+ "vendor": "google",
+ "started_at": "2025-10-21T17:53:27.572175300Z",
+ "finished_at": "2025-10-21T17:55:02.606944700Z"
},
- "t_006_delete": {
+ "t_008_index_lookup": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_006_delete",
+ "task": "t_008_index_lookup",
"lang": "csharp",
"golden_published": true,
- "model_name": "Grok 3 Mini (Beta)",
+ "model_name": "Gemini 2.5 Pro",
"total_tests": 3,
- "passed_tests": 0,
- "llm_output": null,
+ "passed_tests": 3,
+ "llm_output": "```csharp\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"results\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void LookupUserName(ReducerContext ctx, int id)\n {\n var user = ctx.Db.users.Id.Find(id);\n if (user.HasValue)\n {\n var foundUser = user.Value;\n ctx.Db.results.Insert(new Result { Id = foundUser.Id, Name = foundUser.Name });\n }\n }\n}\n```",
"category": "basics",
- "route_api_model": "grok-3-mini",
- "golden_db": "basics-t-006-delete-golden",
- "llm_db": "basics-t-006-delete-grok-3-mini-beta-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\grok-3-mini-beta\\llm",
+ "route_api_model": "gemini-2.5-pro",
+ "golden_db": "basics-t-008-index-lookup-golden",
+ "llm_db": "basics-t-008-index-lookup-gemini-2-5-pro-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gemini-2-5-pro\\llm",
"scorer_details": {
- "publish_error": {
- "pass": false,
- "partial": 0.0,
+ "seed_user_row": {
+ "pass": true,
+ "partial": 1.0,
"notes": {
- "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\grok-3-mini-beta\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(56,46): error CS0050: Inconsistent accessibility: return type 'IEnumerable' is less accessible than method 'users.Iter()' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\grok-3-mini-beta\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(57,33): error CS0050: Inconsistent accessibility: return type 'Module.Users' is less accessible than method 'users.Insert(Module.Users)' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\grok-3-mini-beta\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(57,33): error CS0051: Inconsistent accessibility: parameter type 'Module.Users' is less accessible than method 'users.Insert(Module.Users)' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\grok-3-mini-beta\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(58,17): error CS0051: Inconsistent accessibility: parameter type 'Module.Users' is less accessible than method 'users.Delete(Module.Users)' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\grok-3-mini-beta\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(60,25): error CS0060: Inconsistent accessibility: base class 'UniqueIndex' is less accessible than class 'users.IdUniqueIndex' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\grok-3-mini-beta\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(65,34): error CS0050: Inconsistent accessibility: return type 'Module.Users?' is less accessible than method 'users.IdUniqueIndex.Find(int)' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\grok-3-mini-beta\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(66,33): error CS0050: Inconsistent accessibility: return type 'Module.Users' is less accessible than method 'users.IdUniqueIndex.Update(Module.Users)' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\grok-3-mini-beta\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(66,33): error CS0051: Inconsistent accessibility: parameter type 'Module.Users' is less accessible than method 'users.IdUniqueIndex.Update(Module.Users)' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\grok-3-mini-beta\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'users' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\n",
- "phase": "build_or_publish"
+ "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
}
- }
- },
- "vendor": "xai",
- "started_at": "2025-10-21T01:20:25.377070500Z",
- "finished_at": "2025-10-21T01:20:43.300117900Z"
- },
- "t_021_multi_column_index": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_021_multi_column_index",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "Grok 3 Mini (Beta)",
- "total_tests": 4,
- "passed_tests": 0,
- "llm_output": null,
- "category": "schema",
- "route_api_model": "grok-3-mini",
- "golden_db": "schema-t-021-multi-column-index-golden",
- "llm_db": "schema-t-021-multi-column-index-grok-3-mini-beta-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\grok-3-mini-beta\\llm",
- "scorer_details": {
- "publish_error": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(2,7): error CS0246: The type or namespace name 'Spacetimedb' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(18,29): error CS0246: The type or namespace name 'ReducerContext' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(17,6): error CS0246: The type or namespace name 'Spacetimedb' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(6,6): error CS0246: The type or namespace name 'Spacetimedb' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(9,10): error CS0246: The type or namespace name 'Spacetimedb' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(11,10): error CS0246: The type or namespace name 'Spacetimedb' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(11,66): error CS9176: There is no target type for the collection expression. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\n",
- "phase": "build_or_publish"
- }
- }
- },
- "vendor": "xai",
- "started_at": "2025-10-21T01:23:35.680781500Z",
- "finished_at": "2025-10-21T01:23:46.401438300Z"
- },
- "t_000_empty_reducers": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_000_empty_reducers",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "Grok 3 Mini (Beta)",
- "total_tests": 1,
- "passed_tests": 0,
- "llm_output": "using Spacetimedb;\n\npublic static partial class Module\n{\n [Reducer]\n public static void EmptyReducer_NoArgs(ReducerContext ctx)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithInt(ReducerContext ctx, int count)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithString(ReducerContext ctx, string name)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithTwoArgs(ReducerContext ctx, int count, string name)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithThreeArgs(ReducerContext ctx, bool active, float ratio, string label)\n {\n }\n}",
- "category": "basics",
- "route_api_model": "grok-3-mini",
- "golden_db": "basics-t-000-empty-reducers-golden",
- "llm_db": "basics-t-000-empty-reducers-grok-3-mini-beta-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm",
- "scorer_details": {
- "publish_error": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(2,7): error CS0246: The type or namespace name 'Spacetimedb' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(7,44): error CS0246: The type or namespace name 'ReducerContext' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(12,45): error CS0246: The type or namespace name 'ReducerContext' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(17,48): error CS0246: The type or namespace name 'ReducerContext' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(22,49): error CS0246: The type or namespace name 'ReducerContext' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(27,51): error CS0246: The type or namespace name 'ReducerContext' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(6,6): error CS0246: The type or namespace name 'ReducerAttribute' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(6,6): error CS0246: The type or namespace name 'Reducer' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(11,6): error CS0246: The type or namespace name 'ReducerAttribute' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(11,6): error CS0246: The type or namespace name 'Reducer' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(16,6): error CS0246: The type or namespace name 'ReducerAttribute' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(16,6): error CS0246: The type or namespace name 'Reducer' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(21,6): error CS0246: The type or namespace name 'ReducerAttribute' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(21,6): error CS0246: The type or namespace name 'Reducer' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(26,6): error CS0246: The type or namespace name 'ReducerAttribute' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\Lib.cs(26,6): error CS0246: The type or namespace name 'Reducer' could not be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\grok-3-mini-beta\\llm\\StdbModule.csproj]\r\n",
- "phase": "build_or_publish"
- }
- }
- },
- "vendor": "xai",
- "started_at": "2025-10-21T23:59:35.577464800Z",
- "finished_at": "2025-10-22T00:00:02.000437300Z"
- }
- }
- },
- {
- "name": "Gemini 2.5 Pro",
- "route_api_model": "gemini-2.5-pro",
- "tasks": {
- "t_001_basic_tables": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_001_basic_tables",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "Gemini 2.5 Pro",
- "total_tests": 1,
- "passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\")]\n public partial struct Users\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"products\")]\n public partial struct Products\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Title;\n public float Price;\n public bool InStock;\n }\n\n [SpacetimeDB.Table(Name = \"notes\")]\n public partial struct Notes\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Body;\n public long Rating;\n public bool Pinned;\n }\n}\n",
- "category": "basics",
- "route_api_model": "gemini-2.5-pro",
- "golden_db": "basics-t-001-basic-tables-golden",
- "llm_db": "basics-t-001-basic-tables-gemini-2-5-pro-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\gemini-2-5-pro\\llm",
- "scorer_details": {
+ },
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-001-basic-tables-golden",
- "llm_db": "basics-t-001-basic-tables-gemini-2-5-pro-llm",
+ "golden_db": "basics-t-008-index-lookup-golden",
+ "llm_db": "basics-t-008-index-lookup-gemini-2-5-pro-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
"tables_diff": null,
"tables_equal": true
}
+ },
+ "index_lookup_projection_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "args": [
+ 1
+ ],
+ "golden_db": "basics-t-008-index-lookup-golden",
+ "golden_out": "Id | Name ----+--------- 1 | \"Alice\"",
+ "llm_db": "basics-t-008-index-lookup-gemini-2-5-pro-llm",
+ "llm_out": "Id | Name ----+--------- 1 | \"Alice\"",
+ "query": "SELECT Id, Name FROM results WHERE Id=1",
+ "reducer": "LookupUserName",
+ "server": "local"
+ }
}
},
"vendor": "google",
- "started_at": "2025-10-21T17:53:27.283937900Z",
- "finished_at": "2025-10-21T17:55:01.828931400Z"
+ "started_at": "2025-10-21T17:53:27.617213300Z",
+ "finished_at": "2025-10-21T17:55:06.239467800Z"
},
- "t_010_connect": {
+ "t_009_init": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_010_connect",
+ "task": "t_009_init",
"lang": "csharp",
"golden_published": true,
"model_name": "Gemini 2.5 Pro",
- "total_tests": 1,
- "passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"events\")]\n public partial struct Event\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public int Id;\n public string Kind;\n }\n\n [Reducer(ReducerKind.ClientConnected)]\n public static void ClientConnected(ReducerContext ctx)\n {\n ctx.Db.events.Insert(new Event { Kind = \"connected\" });\n }\n\n [Reducer(ReducerKind.ClientDisconnected)]\n public static void ClientDisconnected(ReducerContext ctx)\n {\n ctx.Db.events.Insert(new Event { Kind = \"disconnected\" });\n }\n}",
+ "total_tests": 4,
+ "passed_tests": 4,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.users.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.users.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n }\n}",
"category": "basics",
"route_api_model": "gemini-2.5-pro",
- "golden_db": "basics-t-010-connect-golden",
- "llm_db": "basics-t-010-connect-gemini-2-5-pro-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_010_connect\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_010_connect\\csharp\\server\\gemini-2-5-pro\\llm",
+ "golden_db": "basics-t-009-init-golden",
+ "llm_db": "basics-t-009-init-gemini-2-5-pro-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_009_init\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_009_init\\csharp\\server\\gemini-2-5-pro\\llm",
"scorer_details": {
- "schema_parity": {
+ "init_total_two": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-010-connect-golden",
- "llm_db": "basics-t-010-connect-gemini-2-5-pro-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
+ "actual": 2,
+ "expected": 2,
+ "sql": "SELECT COUNT(*) AS n FROM users"
}
- }
- },
- "vendor": "google",
- "started_at": "2025-10-21T17:53:27.710554700Z",
- "finished_at": "2025-10-21T17:55:07.487533500Z"
- },
- "t_013_spacetime_sum_type": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_013_spacetime_sum_type",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "Gemini 2.5 Pro",
- "total_tests": 3,
- "passed_tests": 3,
- "llm_output": "```csharp\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : SpacetimeDB.TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"results\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetCircle(ReducerContext ctx, int id, int radius)\n {\n var circleValue = new Shape.Circle(new Circle { Radius = radius });\n ctx.Db.results.Insert(new Result { Id = id, Value = circleValue });\n }\n}\n```",
- "category": "schema",
- "route_api_model": "gemini-2.5-pro",
- "golden_db": "schema-t-013-spacetime-sum-type-golden",
- "llm_db": "schema-t-013-spacetime-sum-type-gemini-2-5-pro-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\gemini-2-5-pro\\llm",
- "scorer_details": {
- "sum_type_row_parity": {
+ },
+ "init_seed_alice": {
"pass": true,
"partial": 1.0,
"notes": {
- "args": [
- 1,
- 10
- ],
- "golden_db": "schema-t-013-spacetime-sum-type-golden",
- "golden_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))",
- "llm_db": "schema-t-013-spacetime-sum-type-gemini-2-5-pro-llm",
- "llm_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))",
- "query": "SELECT Id, Value FROM results WHERE Id=1",
- "reducer": "SetCircle",
- "server": "local"
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true"
}
},
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-013-spacetime-sum-type-golden",
- "llm_db": "schema-t-013-spacetime-sum-type-gemini-2-5-pro-llm",
+ "golden_db": "basics-t-009-init-golden",
+ "llm_db": "basics-t-009-init-gemini-2-5-pro-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -8222,112 +8577,76 @@
"tables_equal": true
}
},
- "sum_type_row_count": {
+ "init_seed_bob": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1"
+ "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false"
}
}
},
"vendor": "google",
- "started_at": "2025-10-21T17:49:25.759595400Z",
- "finished_at": "2025-10-21T17:50:59.217566800Z"
+ "started_at": "2025-10-21T17:53:27.665289500Z",
+ "finished_at": "2025-10-21T17:55:01.619051Z"
},
- "t_019_many_to_many": {
+ "t_010_connect": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_019_many_to_many",
+ "task": "t_010_connect",
"lang": "csharp",
"golden_published": true,
"model_name": "Gemini 2.5 Pro",
- "total_tests": 5,
- "passed_tests": 5,
- "llm_output": "```csharp\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int UserId;\n public string Name;\n }\n\n [SpacetimeDB.Table(Name = \"groups\")]\n public partial struct Group\n {\n [SpacetimeDB.PrimaryKey]\n public int GroupId;\n public string Title;\n }\n\n [SpacetimeDB.Table(Name = \"memberships\")]\n [SpacetimeDB.Index.BTree(Name = \"by_user\", Columns = [nameof(UserId)])]\n [SpacetimeDB.Index.BTree(Name = \"by_group\", Columns = [nameof(GroupId)])]\n public partial struct Membership\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int GroupId;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.users.Insert(new User { UserId = 1, Name = \"Alice\" });\n ctx.Db.users.Insert(new User { UserId = 2, Name = \"Bob\" });\n\n ctx.Db.groups.Insert(new Group { GroupId = 10, Title = \"Admin\" });\n ctx.Db.groups.Insert(new Group { GroupId = 20, Title = \"Dev\" });\n\n ctx.Db.memberships.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\n ctx.Db.memberships.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\n ctx.Db.memberships.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\n }\n}\n```",
- "category": "schema",
+ "total_tests": 1,
+ "passed_tests": 1,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"events\")]\n public partial struct Event\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public int Id;\n public string Kind;\n }\n\n [Reducer(ReducerKind.ClientConnected)]\n public static void ClientConnected(ReducerContext ctx)\n {\n ctx.Db.events.Insert(new Event { Kind = \"connected\" });\n }\n\n [Reducer(ReducerKind.ClientDisconnected)]\n public static void ClientDisconnected(ReducerContext ctx)\n {\n ctx.Db.events.Insert(new Event { Kind = \"disconnected\" });\n }\n}",
+ "category": "basics",
"route_api_model": "gemini-2.5-pro",
- "golden_db": "schema-t-019-many-to-many-golden",
- "llm_db": "schema-t-019-many-to-many-gemini-2-5-pro-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\gemini-2-5-pro\\llm",
+ "golden_db": "basics-t-010-connect-golden",
+ "llm_db": "basics-t-010-connect-gemini-2-5-pro-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_010_connect\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_010_connect\\csharp\\server\\gemini-2-5-pro\\llm",
"scorer_details": {
- "m2m_has_1_20": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM memberships WHERE UserId=1 AND GroupId=20"
- }
- },
- "m2m_has_2_20": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM memberships WHERE UserId=2 AND GroupId=20"
- }
- },
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-019-many-to-many-golden",
- "llm_db": "schema-t-019-many-to-many-gemini-2-5-pro-llm",
+ "golden_db": "basics-t-010-connect-golden",
+ "llm_db": "basics-t-010-connect-gemini-2-5-pro-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
"tables_diff": null,
"tables_equal": true
}
- },
- "memberships_three_rows": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 3,
- "expected": 3,
- "sql": "SELECT COUNT(*) AS n FROM memberships"
- }
- },
- "m2m_has_1_10": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM memberships WHERE UserId=1 AND GroupId=10"
- }
}
},
"vendor": "google",
- "started_at": "2025-10-21T17:49:26.024779Z",
- "finished_at": "2025-10-21T17:50:59.715018500Z"
+ "started_at": "2025-10-21T17:53:27.710554700Z",
+ "finished_at": "2025-10-21T17:55:07.487533500Z"
},
- "t_007_crud": {
+ "t_011_helper_function": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_007_crud",
+ "task": "t_011_helper_function",
"lang": "csharp",
"golden_published": true,
"model_name": "Gemini 2.5 Pro",
- "total_tests": 4,
- "passed_tests": 4,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void Crud(ReducerContext ctx)\n {\n ctx.Db.users.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.users.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n ctx.Db.users.Id.Update(new User { Id = 1, Name = \"Alice2\", Age = 31, Active = false });\n ctx.Db.users.Id.Delete(2);\n }\n}",
+ "total_tests": 3,
+ "passed_tests": 3,
+ "llm_output": "```csharp\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"results\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Sum;\n }\n\n private static int Add(int a, int b)\n {\n return a + b;\n }\n\n [SpacetimeDB.Reducer]\n public static void ComputeSum(ReducerContext ctx, int id, int a, int b)\n {\n int sum = Add(a, b);\n ctx.Db.results.Insert(new Result { Id = id, Sum = sum });\n }\n}\n```",
"category": "basics",
"route_api_model": "gemini-2.5-pro",
- "golden_db": "basics-t-007-crud-golden",
- "llm_db": "basics-t-007-crud-gemini-2-5-pro-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\gemini-2-5-pro\\llm",
+ "golden_db": "basics-t-011-helper-function-golden",
+ "llm_db": "basics-t-011-helper-function-gemini-2-5-pro-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\gemini-2-5-pro\\llm",
"scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-007-crud-golden",
- "llm_db": "basics-t-007-crud-gemini-2-5-pro-llm",
+ "golden_db": "basics-t-011-helper-function-golden",
+ "llm_db": "basics-t-011-helper-function-gemini-2-5-pro-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -8335,42 +8654,37 @@
"tables_equal": true
}
},
- "crud_total_count_one": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM users"
- }
- },
- "crud_row_id1_parity": {
+ "helper_func_sum_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "args": [],
- "golden_db": "basics-t-007-crud-golden",
- "golden_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
- "llm_db": "basics-t-007-crud-gemini-2-5-pro-llm",
- "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
- "query": "SELECT Id, Name, Age, Active FROM users WHERE Id=1",
- "reducer": "Crud",
+ "args": [
+ 1,
+ 2,
+ 3
+ ],
+ "golden_db": "basics-t-011-helper-function-golden",
+ "golden_out": "Id | Sum ----+----- 1 | 5",
+ "llm_db": "basics-t-011-helper-function-gemini-2-5-pro-llm",
+ "llm_out": "Id | Sum ----+----- 1 | 5",
+ "query": "SELECT Id, Sum FROM results WHERE Id=1",
+ "reducer": "ComputeSum",
"server": "local"
}
},
- "crud_row_id2_deleted": {
+ "helper_func_sum_abs": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 0,
- "expected": 0,
- "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=2"
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1 AND Sum=5"
}
}
},
"vendor": "google",
- "started_at": "2025-10-21T17:53:27.572175300Z",
- "finished_at": "2025-10-21T17:55:02.606944700Z"
+ "started_at": "2025-10-21T17:53:27.757251300Z",
+ "finished_at": "2025-10-21T17:55:03.225122800Z"
},
"t_012_spacetime_product_type": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
@@ -8388,37 +8702,6 @@
"work_dir_golden": "target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\golden",
"work_dir_llm": "target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\gemini-2-5-pro\\llm",
"scorer_details": {
- "product_type_row_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "args": [
- 1,
- 2,
- 3
- ],
- "golden_db": "schema-t-012-spacetime-product-type-golden",
- "golden_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)",
- "llm_db": "schema-t-012-spacetime-product-type-gemini-2-5-pro-llm",
- "llm_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)",
- "query": "SELECT Id, Value FROM results WHERE Id=1",
- "reducer": "SetScore",
- "server": "local"
- }
- },
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "golden_db": "schema-t-012-spacetime-product-type-golden",
- "llm_db": "schema-t-012-spacetime-product-type-gemini-2-5-pro-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
- }
- },
"product_type_row_count": {
"pass": true,
"partial": 1.0,
@@ -8427,68 +8710,13 @@
"expected": 1,
"sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1"
}
- }
- },
- "vendor": "google",
- "started_at": "2025-10-21T17:49:25.675389700Z",
- "finished_at": "2025-10-21T17:50:47.530591800Z"
- },
- "t_000_empty_reducers": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_000_empty_reducers",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "Gemini 2.5 Pro",
- "total_tests": 1,
- "passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Reducer]\n public static void EmptyReducer_NoArgs(ReducerContext ctx)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithInt(ReducerContext ctx, int count)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithString(ReducerContext ctx, string name)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithTwoArgs(ReducerContext ctx, int count, string name)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithThreeArgs(ReducerContext ctx, bool active, float ratio, string label)\n {\n }\n}",
- "category": "basics",
- "route_api_model": "gemini-2.5-pro",
- "golden_db": "basics-t-000-empty-reducers-golden",
- "llm_db": "basics-t-000-empty-reducers-gemini-2-5-pro-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\gemini-2-5-pro\\llm",
- "scorer_details": {
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "golden_db": "basics-t-000-empty-reducers-golden",
- "llm_db": "basics-t-000-empty-reducers-gemini-2-5-pro-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
- }
- }
- },
- "vendor": "google",
- "started_at": "2025-10-21T17:53:27.205121400Z",
- "finished_at": "2025-10-21T17:55:00.557516300Z"
- },
- "t_004_insert": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_004_insert",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "Gemini 2.5 Pro",
- "total_tests": 2,
- "passed_tests": 2,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void InsertUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n ctx.Db.users.Insert(new User\n {\n Id = id,\n Name = name,\n Age = age,\n Active = active,\n });\n }\n}",
- "category": "basics",
- "route_api_model": "gemini-2.5-pro",
- "golden_db": "basics-t-004-insert-golden",
- "llm_db": "basics-t-004-insert-gemini-2-5-pro-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_004_insert\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_004_insert\\csharp\\server\\gemini-2-5-pro\\llm",
- "scorer_details": {
+ },
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-004-insert-golden",
- "llm_db": "basics-t-004-insert-gemini-2-5-pro-llm",
+ "golden_db": "schema-t-012-spacetime-product-type-golden",
+ "llm_db": "schema-t-012-spacetime-product-type-gemini-2-5-pro-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -8496,59 +8724,51 @@
"tables_equal": true
}
},
- "data_parity_insert_user": {
+ "product_type_row_parity": {
"pass": true,
"partial": 1.0,
"notes": {
"args": [
1,
- "Alice",
- 30,
- true
+ 2,
+ 3
],
- "golden_db": "basics-t-004-insert-golden",
- "golden_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true",
- "llm_db": "basics-t-004-insert-gemini-2-5-pro-llm",
- "llm_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true",
- "query": "SELECT Id, Name, Age, Active FROM users WHERE Id=1",
- "reducer": "InsertUser",
+ "golden_db": "schema-t-012-spacetime-product-type-golden",
+ "golden_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)",
+ "llm_db": "schema-t-012-spacetime-product-type-gemini-2-5-pro-llm",
+ "llm_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)",
+ "query": "SELECT Id, Value FROM results WHERE Id=1",
+ "reducer": "SetScore",
"server": "local"
}
}
},
"vendor": "google",
- "started_at": "2025-10-21T17:53:27.427049100Z",
- "finished_at": "2025-10-21T17:55:05.566960600Z"
+ "started_at": "2025-10-21T17:49:25.675389700Z",
+ "finished_at": "2025-10-21T17:50:47.530591800Z"
},
- "t_005_update": {
+ "t_013_spacetime_sum_type": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_005_update",
+ "task": "t_013_spacetime_sum_type",
"lang": "csharp",
"golden_published": true,
"model_name": "Gemini 2.5 Pro",
"total_tests": 3,
"passed_tests": 3,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n User user = new User\n {\n Id = id,\n Name = name,\n Age = age,\n Active = active,\n };\n ctx.Db.users.Id.Update(user);\n }\n}",
- "category": "basics",
+ "llm_output": "```csharp\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : SpacetimeDB.TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"results\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetCircle(ReducerContext ctx, int id, int radius)\n {\n var circleValue = new Shape.Circle(new Circle { Radius = radius });\n ctx.Db.results.Insert(new Result { Id = id, Value = circleValue });\n }\n}\n```",
+ "category": "schema",
"route_api_model": "gemini-2.5-pro",
- "golden_db": "basics-t-005-update-golden",
- "llm_db": "basics-t-005-update-gemini-2-5-pro-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_005_update\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_005_update\\csharp\\server\\gemini-2-5-pro\\llm",
+ "golden_db": "schema-t-013-spacetime-sum-type-golden",
+ "llm_db": "schema-t-013-spacetime-sum-type-gemini-2-5-pro-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\gemini-2-5-pro\\llm",
"scorer_details": {
- "seed_users_row": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
- }
- },
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-005-update-golden",
- "llm_db": "basics-t-005-update-gemini-2-5-pro-llm",
+ "golden_db": "schema-t-013-spacetime-sum-type-golden",
+ "llm_db": "schema-t-013-spacetime-sum-type-gemini-2-5-pro-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -8556,29 +8776,36 @@
"tables_equal": true
}
},
- "data_parity_update_user": {
+ "sum_type_row_parity": {
"pass": true,
"partial": 1.0,
"notes": {
"args": [
1,
- "Alice2",
- 31,
- false
+ 10
],
- "golden_db": "basics-t-005-update-golden",
- "golden_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
- "llm_db": "basics-t-005-update-gemini-2-5-pro-llm",
- "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
- "query": "SELECT Id, Name, Age, Active FROM users WHERE Id=1",
- "reducer": "UpdateUser",
+ "golden_db": "schema-t-013-spacetime-sum-type-golden",
+ "golden_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))",
+ "llm_db": "schema-t-013-spacetime-sum-type-gemini-2-5-pro-llm",
+ "llm_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))",
+ "query": "SELECT Id, Value FROM results WHERE Id=1",
+ "reducer": "SetCircle",
"server": "local"
}
+ },
+ "sum_type_row_count": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1"
+ }
}
},
"vendor": "google",
- "started_at": "2025-10-21T17:53:27.478569400Z",
- "finished_at": "2025-10-21T17:55:03.848315200Z"
+ "started_at": "2025-10-21T17:49:25.759595400Z",
+ "finished_at": "2025-10-21T17:50:59.217566800Z"
},
"t_014_elementary_columns": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
@@ -8637,117 +8864,44 @@
"started_at": "2025-10-21T17:49:25.804451200Z",
"finished_at": "2025-10-21T17:50:50.109087Z"
},
- "t_002_scheduled_table": {
+ "t_015_product_type_columns": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_002_scheduled_table",
+ "task": "t_015_product_type_columns",
"lang": "csharp",
"golden_published": true,
"model_name": "Gemini 2.5 Pro",
- "total_tests": 1,
- "passed_tests": 1,
- "llm_output": "```csharp\nusing SpacetimeDB;\nusing System;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"tick_timer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public ulong ScheduledId;\n public SpacetimeDB.ScheduleAt ScheduledAt;\n }\n\n [SpacetimeDB.Reducer]\n public static void Tick(ReducerContext ctx, TickTimer timer)\n {\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.tick_timer.Insert(new TickTimer\n {\n ScheduledId = 0,\n ScheduledAt = new SpacetimeDB.ScheduleAt.Interval(TimeSpan.FromMicroseconds(50_000)),\n });\n }\n}\n```",
- "category": "basics",
+ "total_tests": 3,
+ "passed_tests": 3,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Address\n {\n public string Street;\n public int Zip;\n }\n\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"profiles\")]\n public partial struct Profile\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Address Home;\n public Address Work;\n public Position Pos;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.profiles.Insert(new Profile\n {\n Id = 1,\n Home = new Address { Street = \"1 Main\", Zip = 11111 },\n Work = new Address { Street = \"2 Broad\", Zip = 22222 },\n Pos = new Position { X = 7, Y = 9 },\n });\n }\n}",
+ "category": "schema",
"route_api_model": "gemini-2.5-pro",
- "golden_db": "basics-t-002-scheduled-table-golden",
- "llm_db": "basics-t-002-scheduled-table-gemini-2-5-pro-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_002_scheduled_table\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_002_scheduled_table\\csharp\\server\\gemini-2-5-pro\\llm",
+ "golden_db": "schema-t-015-product-type-columns-golden",
+ "llm_db": "schema-t-015-product-type-columns-gemini-2-5-pro-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\gemini-2-5-pro\\llm",
"scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-002-scheduled-table-golden",
- "llm_db": "basics-t-002-scheduled-table-gemini-2-5-pro-llm",
+ "golden_db": "schema-t-015-product-type-columns-golden",
+ "llm_db": "schema-t-015-product-type-columns-gemini-2-5-pro-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
"tables_diff": null,
"tables_equal": true
}
- }
- },
- "vendor": "google",
- "started_at": "2025-10-21T17:53:27.329794400Z",
- "finished_at": "2025-10-21T17:55:47.101376900Z"
- },
- "t_011_helper_function": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_011_helper_function",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "Gemini 2.5 Pro",
- "total_tests": 3,
- "passed_tests": 3,
- "llm_output": "```csharp\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"results\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Sum;\n }\n\n private static int Add(int a, int b)\n {\n return a + b;\n }\n\n [SpacetimeDB.Reducer]\n public static void ComputeSum(ReducerContext ctx, int id, int a, int b)\n {\n int sum = Add(a, b);\n ctx.Db.results.Insert(new Result { Id = id, Sum = sum });\n }\n}\n```",
- "category": "basics",
- "route_api_model": "gemini-2.5-pro",
- "golden_db": "basics-t-011-helper-function-golden",
- "llm_db": "basics-t-011-helper-function-gemini-2-5-pro-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\gemini-2-5-pro\\llm",
- "scorer_details": {
- "helper_func_sum_abs": {
+ },
+ "product_type_columns_row_count": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1 AND Sum=5"
- }
- },
- "helper_func_sum_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "args": [
- 1,
- 2,
- 3
- ],
- "golden_db": "basics-t-011-helper-function-golden",
- "golden_out": "Id | Sum ----+----- 1 | 5",
- "llm_db": "basics-t-011-helper-function-gemini-2-5-pro-llm",
- "llm_out": "Id | Sum ----+----- 1 | 5",
- "query": "SELECT Id, Sum FROM results WHERE Id=1",
- "reducer": "ComputeSum",
- "server": "local"
+ "sql": "SELECT COUNT(*) AS n FROM profiles WHERE Id=1"
}
},
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "golden_db": "basics-t-011-helper-function-golden",
- "llm_db": "basics-t-011-helper-function-gemini-2-5-pro-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
- }
- }
- },
- "vendor": "google",
- "started_at": "2025-10-21T17:53:27.757251300Z",
- "finished_at": "2025-10-21T17:55:03.225122800Z"
- },
- "t_015_product_type_columns": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_015_product_type_columns",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "Gemini 2.5 Pro",
- "total_tests": 3,
- "passed_tests": 3,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Address\n {\n public string Street;\n public int Zip;\n }\n\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"profiles\")]\n public partial struct Profile\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Address Home;\n public Address Work;\n public Position Pos;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.profiles.Insert(new Profile\n {\n Id = 1,\n Home = new Address { Street = \"1 Main\", Zip = 11111 },\n Work = new Address { Street = \"2 Broad\", Zip = 22222 },\n Pos = new Position { X = 7, Y = 9 },\n });\n }\n}",
- "category": "schema",
- "route_api_model": "gemini-2.5-pro",
- "golden_db": "schema-t-015-product-type-columns-golden",
- "llm_db": "schema-t-015-product-type-columns-gemini-2-5-pro-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\gemini-2-5-pro\\llm",
- "scorer_details": {
"product_type_columns_row_parity": {
"pass": true,
"partial": 1.0,
@@ -8761,56 +8915,34 @@
"reducer": "Seed",
"server": "local"
}
- },
- "product_type_columns_row_count": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM profiles WHERE Id=1"
- }
- },
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "golden_db": "schema-t-015-product-type-columns-golden",
- "llm_db": "schema-t-015-product-type-columns-gemini-2-5-pro-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
- }
}
},
"vendor": "google",
"started_at": "2025-10-21T17:49:25.848880500Z",
"finished_at": "2025-10-21T17:50:52.302556500Z"
},
- "t_009_init": {
+ "t_016_sum_type_columns": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_009_init",
+ "task": "t_016_sum_type_columns",
"lang": "csharp",
"golden_published": true,
"model_name": "Gemini 2.5 Pro",
- "total_tests": 4,
- "passed_tests": 4,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.users.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.users.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n }\n}",
- "category": "basics",
+ "total_tests": 3,
+ "passed_tests": 3,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)>\n {\n }\n\n [Table(Name = \"drawings\")]\n public partial struct Drawing\n {\n [PrimaryKey]\n public int Id;\n public Shape A;\n public Shape B;\n }\n\n [Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.drawings.Insert(new Drawing\n {\n Id = 1,\n A = new Shape.Circle(new Circle { Radius = 10 }),\n B = new Shape.Rectangle(new Rectangle { Width = 4, Height = 6 })\n });\n }\n}",
+ "category": "schema",
"route_api_model": "gemini-2.5-pro",
- "golden_db": "basics-t-009-init-golden",
- "llm_db": "basics-t-009-init-gemini-2-5-pro-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_009_init\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_009_init\\csharp\\server\\gemini-2-5-pro\\llm",
+ "golden_db": "schema-t-016-sum-type-columns-golden",
+ "llm_db": "schema-t-016-sum-type-columns-gemini-2-5-pro-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\gemini-2-5-pro\\llm",
"scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-009-init-golden",
- "llm_db": "basics-t-009-init-gemini-2-5-pro-llm",
+ "golden_db": "schema-t-016-sum-type-columns-golden",
+ "llm_db": "schema-t-016-sum-type-columns-gemini-2-5-pro-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -8818,178 +8950,33 @@
"tables_equal": true
}
},
- "init_seed_bob": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false"
- }
- },
- "init_seed_alice": {
+ "sum_type_columns_row_count": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true"
- }
- },
- "init_total_two": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 2,
- "expected": 2,
- "sql": "SELECT COUNT(*) AS n FROM users"
- }
- }
- },
- "vendor": "google",
- "started_at": "2025-10-21T17:53:27.665289500Z",
- "finished_at": "2025-10-21T17:55:01.619051Z"
- },
- "t_006_delete": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_006_delete",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "Gemini 2.5 Pro",
- "total_tests": 3,
- "passed_tests": 3,
- "llm_output": "```csharp\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void DeleteUser(ReducerContext ctx, int id)\n {\n ctx.Db.users.Id.Delete(id);\n }\n}\n```",
- "category": "basics",
- "route_api_model": "gemini-2.5-pro",
- "golden_db": "basics-t-006-delete-golden",
- "llm_db": "basics-t-006-delete-gemini-2-5-pro-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\gemini-2-5-pro\\llm",
- "scorer_details": {
- "delete_user_count_zero": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 0,
- "expected": 0,
- "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=1"
- }
- },
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "golden_db": "basics-t-006-delete-golden",
- "llm_db": "basics-t-006-delete-gemini-2-5-pro-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
+ "sql": "SELECT COUNT(*) AS n FROM drawings WHERE Id=1"
}
},
- "seed_users_row": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
- }
- }
- },
- "vendor": "google",
- "started_at": "2025-10-21T17:53:27.525358200Z",
- "finished_at": "2025-10-21T17:55:01.144593Z"
- },
- "t_003_struct_in_table": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_003_struct_in_table",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "Gemini 2.5 Pro",
- "total_tests": 1,
- "passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"entities\")]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n\n public Position Pos;\n }\n}",
- "category": "basics",
- "route_api_model": "gemini-2.5-pro",
- "golden_db": "basics-t-003-struct-in-table-golden",
- "llm_db": "basics-t-003-struct-in-table-gemini-2-5-pro-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_003_struct_in_table\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_003_struct_in_table\\csharp\\server\\gemini-2-5-pro\\llm",
- "scorer_details": {
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "golden_db": "basics-t-003-struct-in-table-golden",
- "llm_db": "basics-t-003-struct-in-table-gemini-2-5-pro-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
- }
- }
- },
- "vendor": "google",
- "started_at": "2025-10-21T17:53:27.376426300Z",
- "finished_at": "2025-10-21T17:55:36.051605600Z"
- },
- "t_008_index_lookup": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_008_index_lookup",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "Gemini 2.5 Pro",
- "total_tests": 3,
- "passed_tests": 3,
- "llm_output": "```csharp\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"results\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void LookupUserName(ReducerContext ctx, int id)\n {\n var user = ctx.Db.users.Id.Find(id);\n if (user.HasValue)\n {\n var foundUser = user.Value;\n ctx.Db.results.Insert(new Result { Id = foundUser.Id, Name = foundUser.Name });\n }\n }\n}\n```",
- "category": "basics",
- "route_api_model": "gemini-2.5-pro",
- "golden_db": "basics-t-008-index-lookup-golden",
- "llm_db": "basics-t-008-index-lookup-gemini-2-5-pro-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gemini-2-5-pro\\llm",
- "scorer_details": {
- "index_lookup_projection_parity": {
+ "sum_type_columns_row_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "args": [
- 1
- ],
- "golden_db": "basics-t-008-index-lookup-golden",
- "golden_out": "Id | Name ----+--------- 1 | \"Alice\"",
- "llm_db": "basics-t-008-index-lookup-gemini-2-5-pro-llm",
- "llm_out": "Id | Name ----+--------- 1 | \"Alice\"",
- "query": "SELECT Id, Name FROM results WHERE Id=1",
- "reducer": "LookupUserName",
+ "args": [],
+ "golden_db": "schema-t-016-sum-type-columns-golden",
+ "golden_out": "Id | A | B ----+--------------------------+--------------------------------------- 1 | (Circle = (Radius = 10)) | (Rectangle = (Width = 4, Height = 6))",
+ "llm_db": "schema-t-016-sum-type-columns-gemini-2-5-pro-llm",
+ "llm_out": "Id | A | B ----+--------------------------+--------------------------------------- 1 | (Circle = (Radius = 10)) | (Rectangle = (Width = 4, Height = 6))",
+ "query": "SELECT Id, A, B FROM drawings WHERE Id=1",
+ "reducer": "Seed",
"server": "local"
}
- },
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "golden_db": "basics-t-008-index-lookup-golden",
- "llm_db": "basics-t-008-index-lookup-gemini-2-5-pro-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
- }
- },
- "seed_user_row": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
- }
}
},
"vendor": "google",
- "started_at": "2025-10-21T17:53:27.617213300Z",
- "finished_at": "2025-10-21T17:55:06.239467800Z"
+ "started_at": "2025-10-21T17:49:25.893219Z",
+ "finished_at": "2025-10-21T17:51:14.800291300Z"
},
"t_017_scheduled_columns": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
@@ -9034,28 +9021,28 @@
"started_at": "2025-10-21T17:49:25.936714100Z",
"finished_at": "2025-10-21T17:50:59.935881500Z"
},
- "t_021_multi_column_index": {
+ "t_018_constraints": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_021_multi_column_index",
+ "task": "t_018_constraints",
"lang": "csharp",
"golden_published": true,
"model_name": "Gemini 2.5 Pro",
- "total_tests": 4,
- "passed_tests": 4,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"logs\")]\n [SpacetimeDB.Index.BTree(Name = \"by_user_day\", Columns = [nameof(UserId), nameof(Day)])]\n public partial struct Log\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int Day;\n public string Message;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.logs.Insert(new Log { Id = 1, UserId = 7, Day = 1, Message = \"a\" });\n ctx.Db.logs.Insert(new Log { Id = 2, UserId = 7, Day = 2, Message = \"b\" });\n ctx.Db.logs.Insert(new Log { Id = 3, UserId = 9, Day = 1, Message = \"c\" });\n }\n}",
+ "total_tests": 3,
+ "passed_tests": 3,
+ "llm_output": "```csharp\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"accounts\")]\n [SpacetimeDB.Index.BTree(Name = \"by_name\", Columns = [nameof(Name)])]\n public partial struct Account\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n [SpacetimeDB.Unique]\n public string Email;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.accounts.Insert(new Account { Id = 1, Email = \"a@example.com\", Name = \"Alice\" });\n ctx.Db.accounts.Insert(new Account { Id = 2, Email = \"b@example.com\", Name = \"Bob\" });\n }\n}\n```",
"category": "schema",
"route_api_model": "gemini-2.5-pro",
- "golden_db": "schema-t-021-multi-column-index-golden",
- "llm_db": "schema-t-021-multi-column-index-gemini-2-5-pro-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\gemini-2-5-pro\\llm",
+ "golden_db": "schema-t-018-constraints-golden",
+ "llm_db": "schema-t-018-constraints-gemini-2-5-pro-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\gemini-2-5-pro\\llm",
"scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-021-multi-column-index-golden",
- "llm_db": "schema-t-021-multi-column-index-gemini-2-5-pro-llm",
+ "golden_db": "schema-t-018-constraints-golden",
+ "llm_db": "schema-t-018-constraints-gemini-2-5-pro-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -9063,94 +9050,103 @@
"tables_equal": true
}
},
- "mcindex_lookup_u7_d2": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM logs WHERE UserId=7 AND Day=2"
- }
- },
- "mcindex_seed_count": {
+ "constraints_row_parity_after_seed": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 3,
- "expected": 3,
- "sql": "SELECT COUNT(*) AS n FROM logs"
+ "args": [],
+ "golden_db": "schema-t-018-constraints-golden",
+ "golden_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"",
+ "llm_db": "schema-t-018-constraints-gemini-2-5-pro-llm",
+ "llm_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"",
+ "query": "SELECT Id, Email, Name FROM accounts WHERE Id=1",
+ "reducer": "Seed",
+ "server": "local"
}
},
- "mcindex_lookup_u7_d1": {
+ "constraints_seed_two_rows": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM logs WHERE UserId=7 AND Day=1"
+ "sql": "SELECT COUNT(*) AS n FROM accounts WHERE Id=2"
}
}
},
"vendor": "google",
- "started_at": "2025-10-21T17:49:26.113734400Z",
- "finished_at": "2025-10-21T17:50:55.494885900Z"
+ "started_at": "2025-10-21T17:49:25.981048100Z",
+ "finished_at": "2025-10-21T17:50:50.808868100Z"
},
- "t_018_constraints": {
+ "t_019_many_to_many": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_018_constraints",
+ "task": "t_019_many_to_many",
"lang": "csharp",
"golden_published": true,
"model_name": "Gemini 2.5 Pro",
- "total_tests": 3,
- "passed_tests": 3,
- "llm_output": "```csharp\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"accounts\")]\n [SpacetimeDB.Index.BTree(Name = \"by_name\", Columns = [nameof(Name)])]\n public partial struct Account\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n [SpacetimeDB.Unique]\n public string Email;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.accounts.Insert(new Account { Id = 1, Email = \"a@example.com\", Name = \"Alice\" });\n ctx.Db.accounts.Insert(new Account { Id = 2, Email = \"b@example.com\", Name = \"Bob\" });\n }\n}\n```",
+ "total_tests": 5,
+ "passed_tests": 5,
+ "llm_output": "```csharp\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int UserId;\n public string Name;\n }\n\n [SpacetimeDB.Table(Name = \"groups\")]\n public partial struct Group\n {\n [SpacetimeDB.PrimaryKey]\n public int GroupId;\n public string Title;\n }\n\n [SpacetimeDB.Table(Name = \"memberships\")]\n [SpacetimeDB.Index.BTree(Name = \"by_user\", Columns = [nameof(UserId)])]\n [SpacetimeDB.Index.BTree(Name = \"by_group\", Columns = [nameof(GroupId)])]\n public partial struct Membership\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int GroupId;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.users.Insert(new User { UserId = 1, Name = \"Alice\" });\n ctx.Db.users.Insert(new User { UserId = 2, Name = \"Bob\" });\n\n ctx.Db.groups.Insert(new Group { GroupId = 10, Title = \"Admin\" });\n ctx.Db.groups.Insert(new Group { GroupId = 20, Title = \"Dev\" });\n\n ctx.Db.memberships.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\n ctx.Db.memberships.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\n ctx.Db.memberships.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\n }\n}\n```",
"category": "schema",
"route_api_model": "gemini-2.5-pro",
- "golden_db": "schema-t-018-constraints-golden",
- "llm_db": "schema-t-018-constraints-gemini-2-5-pro-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\gemini-2-5-pro\\llm",
+ "golden_db": "schema-t-019-many-to-many-golden",
+ "llm_db": "schema-t-019-many-to-many-gemini-2-5-pro-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\gemini-2-5-pro\\llm",
"scorer_details": {
- "constraints_row_parity_after_seed": {
+ "m2m_has_2_20": {
"pass": true,
"partial": 1.0,
"notes": {
- "args": [],
- "golden_db": "schema-t-018-constraints-golden",
- "golden_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"",
- "llm_db": "schema-t-018-constraints-gemini-2-5-pro-llm",
- "llm_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"",
- "query": "SELECT Id, Email, Name FROM accounts WHERE Id=1",
- "reducer": "Seed",
- "server": "local"
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM memberships WHERE UserId=2 AND GroupId=20"
}
},
- "constraints_seed_two_rows": {
+ "m2m_has_1_20": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM accounts WHERE Id=2"
+ "sql": "SELECT COUNT(*) AS n FROM memberships WHERE UserId=1 AND GroupId=20"
+ }
+ },
+ "memberships_three_rows": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 3,
+ "expected": 3,
+ "sql": "SELECT COUNT(*) AS n FROM memberships"
}
},
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-018-constraints-golden",
- "llm_db": "schema-t-018-constraints-gemini-2-5-pro-llm",
+ "golden_db": "schema-t-019-many-to-many-golden",
+ "llm_db": "schema-t-019-many-to-many-gemini-2-5-pro-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
"tables_diff": null,
"tables_equal": true
}
+ },
+ "m2m_has_1_10": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM memberships WHERE UserId=1 AND GroupId=10"
+ }
}
},
"vendor": "google",
- "started_at": "2025-10-21T17:49:25.981048100Z",
- "finished_at": "2025-10-21T17:50:50.808868100Z"
+ "started_at": "2025-10-21T17:49:26.024779Z",
+ "finished_at": "2025-10-21T17:50:59.715018500Z"
},
"t_020_ecs": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
@@ -9168,31 +9164,35 @@
"work_dir_golden": "target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\golden",
"work_dir_llm": "target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\gemini-2-5-pro\\llm",
"scorer_details": {
- "ecs_step_next_positions_count": {
+ "ecs_seed_positions_count": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 2,
"expected": 2,
- "sql": "SELECT COUNT(*) AS n FROM next_positions"
+ "sql": "SELECT COUNT(*) AS n FROM positions"
}
},
- "ecs_next_pos_entity2": {
+ "schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM next_positions WHERE EntityId=2 AND X=8 AND Y=3"
+ "golden_db": "schema-t-020-ecs-golden",
+ "llm_db": "schema-t-020-ecs-gemini-2-5-pro-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
}
},
- "ecs_seed_positions_count": {
+ "ecs_next_pos_entity2": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 2,
- "expected": 2,
- "sql": "SELECT COUNT(*) AS n FROM positions"
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM next_positions WHERE EntityId=2 AND X=8 AND Y=3"
}
},
"ecs_next_pos_entity1": {
@@ -9204,17 +9204,13 @@
"sql": "SELECT COUNT(*) AS n FROM next_positions WHERE EntityId=1 AND X=1 AND Y=0"
}
},
- "schema_parity": {
+ "ecs_step_next_positions_count": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-020-ecs-golden",
- "llm_db": "schema-t-020-ecs-gemini-2-5-pro-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
+ "actual": 2,
+ "expected": 2,
+ "sql": "SELECT COUNT(*) AS n FROM next_positions"
}
}
},
@@ -9222,62 +9218,66 @@
"started_at": "2025-10-21T17:49:26.069346500Z",
"finished_at": "2025-10-21T17:51:03.579099Z"
},
- "t_016_sum_type_columns": {
+ "t_021_multi_column_index": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_016_sum_type_columns",
+ "task": "t_021_multi_column_index",
"lang": "csharp",
"golden_published": true,
"model_name": "Gemini 2.5 Pro",
- "total_tests": 3,
- "passed_tests": 3,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)>\n {\n }\n\n [Table(Name = \"drawings\")]\n public partial struct Drawing\n {\n [PrimaryKey]\n public int Id;\n public Shape A;\n public Shape B;\n }\n\n [Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.drawings.Insert(new Drawing\n {\n Id = 1,\n A = new Shape.Circle(new Circle { Radius = 10 }),\n B = new Shape.Rectangle(new Rectangle { Width = 4, Height = 6 })\n });\n }\n}",
+ "total_tests": 4,
+ "passed_tests": 4,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"logs\")]\n [SpacetimeDB.Index.BTree(Name = \"by_user_day\", Columns = [nameof(UserId), nameof(Day)])]\n public partial struct Log\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int Day;\n public string Message;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.logs.Insert(new Log { Id = 1, UserId = 7, Day = 1, Message = \"a\" });\n ctx.Db.logs.Insert(new Log { Id = 2, UserId = 7, Day = 2, Message = \"b\" });\n ctx.Db.logs.Insert(new Log { Id = 3, UserId = 9, Day = 1, Message = \"c\" });\n }\n}",
"category": "schema",
"route_api_model": "gemini-2.5-pro",
- "golden_db": "schema-t-016-sum-type-columns-golden",
- "llm_db": "schema-t-016-sum-type-columns-gemini-2-5-pro-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\gemini-2-5-pro\\llm",
+ "golden_db": "schema-t-021-multi-column-index-golden",
+ "llm_db": "schema-t-021-multi-column-index-gemini-2-5-pro-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\gemini-2-5-pro\\llm",
"scorer_details": {
- "sum_type_columns_row_parity": {
+ "mcindex_lookup_u7_d2": {
"pass": true,
"partial": 1.0,
"notes": {
- "args": [],
- "golden_db": "schema-t-016-sum-type-columns-golden",
- "golden_out": "Id | A | B ----+--------------------------+--------------------------------------- 1 | (Circle = (Radius = 10)) | (Rectangle = (Width = 4, Height = 6))",
- "llm_db": "schema-t-016-sum-type-columns-gemini-2-5-pro-llm",
- "llm_out": "Id | A | B ----+--------------------------+--------------------------------------- 1 | (Circle = (Radius = 10)) | (Rectangle = (Width = 4, Height = 6))",
- "query": "SELECT Id, A, B FROM drawings WHERE Id=1",
- "reducer": "Seed",
- "server": "local"
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM logs WHERE UserId=7 AND Day=2"
}
},
- "sum_type_columns_row_count": {
+ "mcindex_lookup_u7_d1": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM drawings WHERE Id=1"
+ "sql": "SELECT COUNT(*) AS n FROM logs WHERE UserId=7 AND Day=1"
}
},
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-016-sum-type-columns-golden",
- "llm_db": "schema-t-016-sum-type-columns-gemini-2-5-pro-llm",
+ "golden_db": "schema-t-021-multi-column-index-golden",
+ "llm_db": "schema-t-021-multi-column-index-gemini-2-5-pro-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
"tables_diff": null,
"tables_equal": true
}
+ },
+ "mcindex_seed_count": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 3,
+ "expected": 3,
+ "sql": "SELECT COUNT(*) AS n FROM logs"
+ }
}
},
"vendor": "google",
- "started_at": "2025-10-21T17:49:25.893219Z",
- "finished_at": "2025-10-21T17:51:14.800291300Z"
+ "started_at": "2025-10-21T17:49:26.113734400Z",
+ "finished_at": "2025-10-21T17:50:55.494885900Z"
}
}
},
@@ -9285,116 +9285,62 @@
"name": "Gemini 2.5 Flash",
"route_api_model": "gemini-2.5-flash",
"tasks": {
- "t_011_helper_function": {
+ "t_000_empty_reducers": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_011_helper_function",
+ "task": "t_000_empty_reducers",
"lang": "csharp",
"golden_published": true,
"model_name": "Gemini 2.5 Flash",
- "total_tests": 3,
- "passed_tests": 3,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"results\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Sum;\n }\n\n private static int Add(int a, int b)\n {\n return a + b;\n }\n\n [SpacetimeDB.Reducer]\n public static void ComputeSum(ReducerContext ctx, int id, int a, int b)\n {\n ctx.Db.results.Insert(new Result { Id = id, Sum = Add(a, b) });\n }\n}",
+ "total_tests": 1,
+ "passed_tests": 1,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Reducer]\n public static void EmptyReducer_NoArgs(ReducerContext ctx)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithInt(ReducerContext ctx, int count)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithString(ReducerContext ctx, string name)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithTwoArgs(ReducerContext ctx, int count, string name)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithThreeArgs(ReducerContext ctx, bool active, float ratio, string label)\n {\n }\n}",
"category": "basics",
"route_api_model": "gemini-2.5-flash",
- "golden_db": "basics-t-011-helper-function-golden",
- "llm_db": "basics-t-011-helper-function-gemini-2-5-flash-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\gemini-2-5-flash\\llm",
+ "golden_db": "basics-t-000-empty-reducers-golden",
+ "llm_db": "basics-t-000-empty-reducers-gemini-2-5-flash-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\gemini-2-5-flash\\llm",
"scorer_details": {
- "helper_func_sum_abs": {
+ "schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1 AND Sum=5"
- }
- },
- "helper_func_sum_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "args": [
- 1,
- 2,
- 3
- ],
- "golden_db": "basics-t-011-helper-function-golden",
- "golden_out": "Id | Sum ----+----- 1 | 5",
- "llm_db": "basics-t-011-helper-function-gemini-2-5-flash-llm",
- "llm_out": "Id | Sum ----+----- 1 | 5",
- "query": "SELECT Id, Sum FROM results WHERE Id=1",
- "reducer": "ComputeSum",
- "server": "local"
- }
- },
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "golden_db": "basics-t-011-helper-function-golden",
- "llm_db": "basics-t-011-helper-function-gemini-2-5-flash-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
+ "golden_db": "basics-t-000-empty-reducers-golden",
+ "llm_db": "basics-t-000-empty-reducers-gemini-2-5-flash-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
}
}
},
"vendor": "google",
- "started_at": "2025-10-21T18:19:53.250788Z",
- "finished_at": "2025-10-21T18:22:04.874212400Z"
+ "started_at": "2025-10-21T18:19:52.681870300Z",
+ "finished_at": "2025-10-21T18:22:03.781677800Z"
},
- "t_012_spacetime_product_type": {
+ "t_001_basic_tables": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_012_spacetime_product_type",
+ "task": "t_001_basic_tables",
"lang": "csharp",
"golden_published": true,
"model_name": "Gemini 2.5 Flash",
- "total_tests": 3,
- "passed_tests": 3,
- "llm_output": "using SpacetimeDB;\nusing System;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Score\n {\n public int Left;\n public int Right;\n }\n\n [SpacetimeDB.Table(Name = \"results\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Score Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetScore(ReducerContext ctx, int id, int left, int right)\n {\n ctx.Db.results.Insert(new Result { Id = id, Value = new Score { Left = left, Right = right } });\n }\n}",
- "category": "schema",
+ "total_tests": 1,
+ "passed_tests": 1,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\", Public = true)]\n public partial struct Users\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"products\", Public = true)]\n public partial struct Products\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Title;\n public float Price;\n public bool InStock;\n }\n\n [SpacetimeDB.Table(Name = \"notes\", Public = true)]\n public partial struct Notes\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Body;\n public long Rating;\n public bool Pinned;\n }\n}",
+ "category": "basics",
"route_api_model": "gemini-2.5-flash",
- "golden_db": "schema-t-012-spacetime-product-type-golden",
- "llm_db": "schema-t-012-spacetime-product-type-gemini-2-5-flash-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\gemini-2-5-flash\\llm",
+ "golden_db": "basics-t-001-basic-tables-golden",
+ "llm_db": "basics-t-001-basic-tables-gemini-2-5-flash-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\gemini-2-5-flash\\llm",
"scorer_details": {
- "product_type_row_count": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1"
- }
- },
- "product_type_row_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "args": [
- 1,
- 2,
- 3
- ],
- "golden_db": "schema-t-012-spacetime-product-type-golden",
- "golden_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)",
- "llm_db": "schema-t-012-spacetime-product-type-gemini-2-5-flash-llm",
- "llm_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)",
- "query": "SELECT Id, Value FROM results WHERE Id=1",
- "reducer": "SetScore",
- "server": "local"
- }
- },
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-012-spacetime-product-type-golden",
- "llm_db": "schema-t-012-spacetime-product-type-gemini-2-5-flash-llm",
+ "golden_db": "basics-t-001-basic-tables-golden",
+ "llm_db": "basics-t-001-basic-tables-gemini-2-5-flash-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -9404,58 +9350,31 @@
}
},
"vendor": "google",
- "started_at": "2025-10-21T18:29:44.135466900Z",
- "finished_at": "2025-10-21T18:31:07.610446400Z"
+ "started_at": "2025-10-21T18:19:52.763159300Z",
+ "finished_at": "2025-10-21T18:20:57.330741700Z"
},
- "t_021_multi_column_index": {
+ "t_002_scheduled_table": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_021_multi_column_index",
+ "task": "t_002_scheduled_table",
"lang": "csharp",
"golden_published": true,
"model_name": "Gemini 2.5 Flash",
- "total_tests": 4,
- "passed_tests": 4,
- "llm_output": "using SpacetimeDB;\nusing System;\n\npublic static partial class Module\n{\n [Table(Name = \"logs\")]\n [SpacetimeDB.Index.BTree(Name = \"by_user_day\", Columns = new string[] { nameof(UserId), nameof(Day) })]\n public partial struct Log\n {\n [PrimaryKey]\n public int Id;\n public int UserId;\n public int Day;\n public string Message;\n }\n\n [Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.logs.Insert(new Log { Id = 1, UserId = 7, Day = 1, Message = \"a\" });\n ctx.Db.logs.Insert(new Log { Id = 2, UserId = 7, Day = 2, Message = \"b\" });\n ctx.Db.logs.Insert(new Log { Id = 3, UserId = 9, Day = 1, Message = \"c\" });\n }\n}",
- "category": "schema",
+ "total_tests": 1,
+ "passed_tests": 1,
+ "llm_output": "using SpacetimeDB;\nusing System;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"tick_timer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [SpacetimeDB.PrimaryKey, SpacetimeDB.AutoInc]\n public ulong ScheduledId;\n public SpacetimeDB.ScheduleAt ScheduledAt;\n }\n\n [SpacetimeDB.Reducer]\n public static void Tick(ReducerContext ctx, TickTimer timer)\n {\n // Reducer logic\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.tick_timer.Insert(new TickTimer\n {\n ScheduledId = 0, // Auto-increment will assign a value\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMicroseconds(50_000))\n });\n }\n}",
+ "category": "basics",
"route_api_model": "gemini-2.5-flash",
- "golden_db": "schema-t-021-multi-column-index-golden",
- "llm_db": "schema-t-021-multi-column-index-gemini-2-5-flash-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\gemini-2-5-flash\\llm",
+ "golden_db": "basics-t-002-scheduled-table-golden",
+ "llm_db": "basics-t-002-scheduled-table-gemini-2-5-flash-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_002_scheduled_table\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_002_scheduled_table\\csharp\\server\\gemini-2-5-flash\\llm",
"scorer_details": {
- "mcindex_lookup_u7_d2": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM logs WHERE UserId=7 AND Day=2"
- }
- },
- "mcindex_seed_count": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 3,
- "expected": 3,
- "sql": "SELECT COUNT(*) AS n FROM logs"
- }
- },
- "mcindex_lookup_u7_d1": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM logs WHERE UserId=7 AND Day=1"
- }
- },
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-021-multi-column-index-golden",
- "llm_db": "schema-t-021-multi-column-index-gemini-2-5-flash-llm",
+ "golden_db": "basics-t-002-scheduled-table-golden",
+ "llm_db": "basics-t-002-scheduled-table-gemini-2-5-flash-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -9465,48 +9384,31 @@
}
},
"vendor": "google",
- "started_at": "2025-10-21T18:29:44.593009700Z",
- "finished_at": "2025-10-21T18:30:54.000181400Z"
+ "started_at": "2025-10-21T18:19:52.810971500Z",
+ "finished_at": "2025-10-21T18:22:05.564885200Z"
},
- "t_016_sum_type_columns": {
+ "t_003_struct_in_table": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_016_sum_type_columns",
+ "task": "t_003_struct_in_table",
"lang": "csharp",
"golden_published": true,
"model_name": "Gemini 2.5 Flash",
- "total_tests": 3,
- "passed_tests": 2,
- "llm_output": "using SpacetimeDB;\nusing System;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : SpacetimeDB.TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"drawings\", Public = true)]\n public partial struct Drawing\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape A;\n public Shape B;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.drawings.Insert(new Drawing\n {\n Id = 1,\n A = new Shape.Circle(new Circle { Radius = 10 }),\n B = new Shape.Rectangle(new Rectangle { Width = 4, Height = 6 })\n });\n }\n}",
- "category": "schema",
+ "total_tests": 1,
+ "passed_tests": 1,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"entities\", Public = true)]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Position Pos;\n }\n}",
+ "category": "basics",
"route_api_model": "gemini-2.5-flash",
- "golden_db": "schema-t-016-sum-type-columns-golden",
- "llm_db": "schema-t-016-sum-type-columns-gemini-2-5-flash-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\gemini-2-5-flash\\llm",
+ "golden_db": "basics-t-003-struct-in-table-golden",
+ "llm_db": "basics-t-003-struct-in-table-gemini-2-5-flash-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_003_struct_in_table\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_003_struct_in_table\\csharp\\server\\gemini-2-5-flash\\llm",
"scorer_details": {
- "sum_type_columns_row_count": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM drawings WHERE Id=1"
- }
- },
- "sum_type_columns_row_parity": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: can't directly call special Init lifecycle reducer\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:3000/v1/database/c200a2db8e9e0283e4979c026c13ddd1e15efd8c5c0d117894c7a9bbec878b4c/call/Seed)\n",
- "phase": "call_reducer_llm"
- }
- },
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-016-sum-type-columns-golden",
- "llm_db": "schema-t-016-sum-type-columns-gemini-2-5-flash-llm",
+ "golden_db": "basics-t-003-struct-in-table-golden",
+ "llm_db": "basics-t-003-struct-in-table-gemini-2-5-flash-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -9516,119 +9418,116 @@
}
},
"vendor": "google",
- "started_at": "2025-10-21T18:29:44.354927100Z",
- "finished_at": "2025-10-21T18:31:07.912657800Z"
+ "started_at": "2025-10-21T18:19:52.859457600Z",
+ "finished_at": "2025-10-21T18:20:57.194798700Z"
},
- "t_017_scheduled_columns": {
+ "t_004_insert": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_017_scheduled_columns",
+ "task": "t_004_insert",
"lang": "csharp",
- "golden_published": true,
+ "golden_published": false,
"model_name": "Gemini 2.5 Flash",
- "total_tests": 2,
- "passed_tests": 2,
- "llm_output": "using SpacetimeDB;\nusing System;\n\npublic static partial class Module\n{\n [Table(Name = \"tick_timer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [PrimaryKey, AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [Reducer]\n public static void Tick(ReducerContext ctx, TickTimer timer)\n {\n // Reducer logic\n }\n\n [Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.tick_timer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMicroseconds(50_000))\n });\n }\n}",
- "category": "schema",
+ "total_tests": 1,
+ "passed_tests": 0,
+ "llm_output": null,
+ "category": "basics",
"route_api_model": "gemini-2.5-flash",
- "golden_db": "schema-t-017-scheduled-columns-golden",
- "llm_db": "schema-t-017-scheduled-columns-gemini-2-5-flash-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_017_scheduled_columns\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_017_scheduled_columns\\csharp\\server\\gemini-2-5-flash\\llm",
+ "golden_db": null,
+ "llm_db": null,
+ "work_dir_golden": null,
+ "work_dir_llm": null,
"scorer_details": {
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "golden_db": "schema-t-017-scheduled-columns-golden",
- "llm_db": "schema-t-017-scheduled-columns-gemini-2-5-flash-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
- }
- },
- "scheduled_seeded_one_row": {
- "pass": true,
- "partial": 1.0,
+ "publish_error": {
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM tick_timer WHERE ScheduledId>=0"
+ "error": "POST https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:generateContent?key=AIzaSyCX9OLo722iLo4_gdYisaPYQXlK3qDBuik -> 429 Too Many Requests: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits.\\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_paid_tier_input_token_count, limit: 1000000\\nPlease retry in 58.967954457s.\",\n \"status\": \"RESOURCE_EXHAUSTED\",\n \"details\": [\n {\n \"@type\": \"type.googleapis.com/google.rpc.QuotaFailure\",\n \"violations\": [\n {\n \"quotaMetric\": \"generativelanguage.googleapis.com/generate_content_paid_tier_input_token_count\",\n \"quotaId\": \"GenerateContentPaidTierInputTokensPerModelPerMinute\",\n \"quotaDimensions\": {\n \"location\": \"global\",\n \"model\": \"gemini-2.5-flash\"\n },\n \"quotaValue\": \"1000000\"\n }\n ]\n },\n {\n \"@type\": \"type.googleapis.com/google.rpc.Help\",\n \"links\": [\n {\n \"description\": \"Learn more about Gemini API quotas\",\n \"url\": \"https://ai.google.dev/gemini-api/docs/rate-limits\"\n }\n ]\n },\n {\n \"@type\": \"type.googleapis.com/google.rpc.RetryInfo\",\n \"retryDelay\": \"58s\"\n }\n ]\n }\n}\n",
+ "phase": "build_or_publish"
}
}
},
"vendor": "google",
- "started_at": "2025-10-21T18:29:44.403558400Z",
- "finished_at": "2025-10-21T18:31:08.141931800Z"
+ "started_at": "2025-10-21T18:22:05.567548300Z",
+ "finished_at": "2025-10-21T18:22:05.567548300Z"
},
- "t_010_connect": {
+ "t_005_update": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_010_connect",
+ "task": "t_005_update",
"lang": "csharp",
- "golden_published": true,
+ "golden_published": false,
"model_name": "Gemini 2.5 Flash",
"total_tests": 1,
- "passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"events\")]\n public partial struct Event\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public int Id;\n public string Kind;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientConnected)]\n public static void ClientConnected(ReducerContext ctx)\n {\n ctx.Db.events.Insert(new Event { Id = 0, Kind = \"connected\" });\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientDisconnected)]\n public static void ClientDisconnected(ReducerContext ctx)\n {\n ctx.Db.events.Insert(new Event { Id = 0, Kind = \"disconnected\" });\n }\n}",
+ "passed_tests": 0,
+ "llm_output": null,
"category": "basics",
"route_api_model": "gemini-2.5-flash",
- "golden_db": "basics-t-010-connect-golden",
- "llm_db": "basics-t-010-connect-gemini-2-5-flash-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_010_connect\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_010_connect\\csharp\\server\\gemini-2-5-flash\\llm",
+ "golden_db": null,
+ "llm_db": null,
+ "work_dir_golden": null,
+ "work_dir_llm": null,
"scorer_details": {
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
+ "publish_error": {
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "golden_db": "basics-t-010-connect-golden",
- "llm_db": "basics-t-010-connect-gemini-2-5-flash-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
+ "error": "POST https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:generateContent?key=AIzaSyCX9OLo722iLo4_gdYisaPYQXlK3qDBuik -> 429 Too Many Requests: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits.\\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_paid_tier_input_token_count, limit: 1000000\\nPlease retry in 58.970895358s.\",\n \"status\": \"RESOURCE_EXHAUSTED\",\n \"details\": [\n {\n \"@type\": \"type.googleapis.com/google.rpc.QuotaFailure\",\n \"violations\": [\n {\n \"quotaMetric\": \"generativelanguage.googleapis.com/generate_content_paid_tier_input_token_count\",\n \"quotaId\": \"GenerateContentPaidTierInputTokensPerModelPerMinute\",\n \"quotaDimensions\": {\n \"location\": \"global\",\n \"model\": \"gemini-2.5-flash\"\n },\n \"quotaValue\": \"1000000\"\n }\n ]\n },\n {\n \"@type\": \"type.googleapis.com/google.rpc.Help\",\n \"links\": [\n {\n \"description\": \"Learn more about Gemini API quotas\",\n \"url\": \"https://ai.google.dev/gemini-api/docs/rate-limits\"\n }\n ]\n },\n {\n \"@type\": \"type.googleapis.com/google.rpc.RetryInfo\",\n \"retryDelay\": \"58s\"\n }\n ]\n }\n}\n",
+ "phase": "build_or_publish"
}
}
},
"vendor": "google",
- "started_at": "2025-10-21T18:19:53.201774300Z",
- "finished_at": "2025-10-21T18:20:57.462301100Z"
+ "started_at": "2025-10-21T18:22:05.566510800Z",
+ "finished_at": "2025-10-21T18:22:05.566510800Z"
},
- "t_003_struct_in_table": {
+ "t_006_delete": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_003_struct_in_table",
+ "task": "t_006_delete",
"lang": "csharp",
"golden_published": true,
"model_name": "Gemini 2.5 Flash",
- "total_tests": 1,
- "passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"entities\", Public = true)]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Position Pos;\n }\n}",
+ "total_tests": 3,
+ "passed_tests": 3,
+ "llm_output": "using SpacetimeDB;\nusing System;\n\npublic static partial class Module\n{\n [Table(Name = \"users\", Public = true)]\n public partial struct User\n {\n [PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [Reducer]\n public static void DeleteUser(ReducerContext ctx, int Id)\n {\n ctx.Db.users.Id.Delete(Id);\n }\n}",
"category": "basics",
"route_api_model": "gemini-2.5-flash",
- "golden_db": "basics-t-003-struct-in-table-golden",
- "llm_db": "basics-t-003-struct-in-table-gemini-2-5-flash-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_003_struct_in_table\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_003_struct_in_table\\csharp\\server\\gemini-2-5-flash\\llm",
+ "golden_db": "basics-t-006-delete-golden",
+ "llm_db": "basics-t-006-delete-gemini-2-5-flash-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\gemini-2-5-flash\\llm",
"scorer_details": {
+ "delete_user_count_zero": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 0,
+ "expected": 0,
+ "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=1"
+ }
+ },
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-003-struct-in-table-golden",
- "llm_db": "basics-t-003-struct-in-table-gemini-2-5-flash-llm",
+ "golden_db": "basics-t-006-delete-golden",
+ "llm_db": "basics-t-006-delete-gemini-2-5-flash-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
"tables_diff": null,
"tables_equal": true
}
+ },
+ "seed_users_row": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
+ }
}
},
"vendor": "google",
- "started_at": "2025-10-21T18:19:52.859457600Z",
- "finished_at": "2025-10-21T18:20:57.194798700Z"
+ "started_at": "2025-10-21T18:19:53.008659Z",
+ "finished_at": "2025-10-21T18:20:57.062808700Z"
},
"t_007_crud": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
@@ -9646,18 +9545,13 @@
"work_dir_golden": "target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\golden",
"work_dir_llm": "target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\gemini-2-5-flash\\llm",
"scorer_details": {
- "crud_row_id1_parity": {
+ "crud_row_id2_deleted": {
"pass": true,
"partial": 1.0,
"notes": {
- "args": [],
- "golden_db": "basics-t-007-crud-golden",
- "golden_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
- "llm_db": "basics-t-007-crud-gemini-2-5-flash-llm",
- "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
- "query": "SELECT Id, Name, Age, Active FROM users WHERE Id=1",
- "reducer": "Crud",
- "server": "local"
+ "actual": 0,
+ "expected": 0,
+ "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=2"
}
},
"schema_parity": {
@@ -9673,22 +9567,27 @@
"tables_equal": true
}
},
- "crud_row_id2_deleted": {
+ "crud_total_count_one": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 0,
- "expected": 0,
- "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=2"
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM users"
}
},
- "crud_total_count_one": {
+ "crud_row_id1_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM users"
+ "args": [],
+ "golden_db": "basics-t-007-crud-golden",
+ "golden_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
+ "llm_db": "basics-t-007-crud-gemini-2-5-flash-llm",
+ "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
+ "query": "SELECT Id, Name, Age, Active FROM users WHERE Id=1",
+ "reducer": "Crud",
+ "server": "local"
}
}
},
@@ -9696,68 +9595,62 @@
"started_at": "2025-10-21T18:19:53.055521800Z",
"finished_at": "2025-10-21T18:22:04.220473200Z"
},
- "t_000_empty_reducers": {
+ "t_008_index_lookup": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_000_empty_reducers",
+ "task": "t_008_index_lookup",
"lang": "csharp",
"golden_published": true,
"model_name": "Gemini 2.5 Flash",
- "total_tests": 1,
- "passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Reducer]\n public static void EmptyReducer_NoArgs(ReducerContext ctx)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithInt(ReducerContext ctx, int count)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithString(ReducerContext ctx, string name)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithTwoArgs(ReducerContext ctx, int count, string name)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithThreeArgs(ReducerContext ctx, bool active, float ratio, string label)\n {\n }\n}",
+ "total_tests": 3,
+ "passed_tests": 3,
+ "llm_output": "using SpacetimeDB;\nusing System;\n\npublic static partial class Module\n{\n [Table(Name = \"users\", Public = true)]\n public partial struct User\n {\n [PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [Table(Name = \"results\", Public = true)]\n public partial struct Result\n {\n [PrimaryKey]\n public int Id;\n public string Name;\n }\n\n [Reducer]\n public static void LookupUserName(ReducerContext ctx, int id)\n {\n var user = ctx.Db.users.Id.Find(id);\n if (user.HasValue)\n {\n ctx.Db.results.Insert(new Result { Id = user.Value.Id, Name = user.Value.Name });\n }\n }\n}",
"category": "basics",
"route_api_model": "gemini-2.5-flash",
- "golden_db": "basics-t-000-empty-reducers-golden",
- "llm_db": "basics-t-000-empty-reducers-gemini-2-5-flash-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\gemini-2-5-flash\\llm",
+ "golden_db": "basics-t-008-index-lookup-golden",
+ "llm_db": "basics-t-008-index-lookup-gemini-2-5-flash-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gemini-2-5-flash\\llm",
"scorer_details": {
+ "index_lookup_projection_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "args": [
+ 1
+ ],
+ "golden_db": "basics-t-008-index-lookup-golden",
+ "golden_out": "Id | Name ----+--------- 1 | \"Alice\"",
+ "llm_db": "basics-t-008-index-lookup-gemini-2-5-flash-llm",
+ "llm_out": "Id | Name ----+--------- 1 | \"Alice\"",
+ "query": "SELECT Id, Name FROM results WHERE Id=1",
+ "reducer": "LookupUserName",
+ "server": "local"
+ }
+ },
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-000-empty-reducers-golden",
- "llm_db": "basics-t-000-empty-reducers-gemini-2-5-flash-llm",
+ "golden_db": "basics-t-008-index-lookup-golden",
+ "llm_db": "basics-t-008-index-lookup-gemini-2-5-flash-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
"tables_diff": null,
"tables_equal": true
}
- }
- },
- "vendor": "google",
- "started_at": "2025-10-21T18:19:52.681870300Z",
- "finished_at": "2025-10-21T18:22:03.781677800Z"
- },
- "t_004_insert": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_004_insert",
- "lang": "csharp",
- "golden_published": false,
- "model_name": "Gemini 2.5 Flash",
- "total_tests": 1,
- "passed_tests": 0,
- "llm_output": null,
- "category": "basics",
- "route_api_model": "gemini-2.5-flash",
- "golden_db": null,
- "llm_db": null,
- "work_dir_golden": null,
- "work_dir_llm": null,
- "scorer_details": {
- "publish_error": {
- "pass": false,
- "partial": 0.0,
+ },
+ "seed_user_row": {
+ "pass": true,
+ "partial": 1.0,
"notes": {
- "error": "POST https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:generateContent?key=AIzaSyCX9OLo722iLo4_gdYisaPYQXlK3qDBuik -> 429 Too Many Requests: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits.\\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_paid_tier_input_token_count, limit: 1000000\\nPlease retry in 58.967954457s.\",\n \"status\": \"RESOURCE_EXHAUSTED\",\n \"details\": [\n {\n \"@type\": \"type.googleapis.com/google.rpc.QuotaFailure\",\n \"violations\": [\n {\n \"quotaMetric\": \"generativelanguage.googleapis.com/generate_content_paid_tier_input_token_count\",\n \"quotaId\": \"GenerateContentPaidTierInputTokensPerModelPerMinute\",\n \"quotaDimensions\": {\n \"location\": \"global\",\n \"model\": \"gemini-2.5-flash\"\n },\n \"quotaValue\": \"1000000\"\n }\n ]\n },\n {\n \"@type\": \"type.googleapis.com/google.rpc.Help\",\n \"links\": [\n {\n \"description\": \"Learn more about Gemini API quotas\",\n \"url\": \"https://ai.google.dev/gemini-api/docs/rate-limits\"\n }\n ]\n },\n {\n \"@type\": \"type.googleapis.com/google.rpc.RetryInfo\",\n \"retryDelay\": \"58s\"\n }\n ]\n }\n}\n",
- "phase": "build_or_publish"
+ "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
}
}
},
"vendor": "google",
- "started_at": "2025-10-21T18:22:05.567548300Z",
- "finished_at": "2025-10-21T18:22:05.567548300Z"
+ "started_at": "2025-10-21T18:19:53.103991700Z",
+ "finished_at": "2025-10-21T18:22:05.395134800Z"
},
"t_009_init": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
@@ -9788,114 +9681,132 @@
"started_at": "2025-10-21T18:19:53.153898200Z",
"finished_at": "2025-10-21T18:20:03.308265500Z"
},
- "t_006_delete": {
+ "t_010_connect": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_006_delete",
+ "task": "t_010_connect",
"lang": "csharp",
"golden_published": true,
"model_name": "Gemini 2.5 Flash",
- "total_tests": 3,
- "passed_tests": 3,
- "llm_output": "using SpacetimeDB;\nusing System;\n\npublic static partial class Module\n{\n [Table(Name = \"users\", Public = true)]\n public partial struct User\n {\n [PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [Reducer]\n public static void DeleteUser(ReducerContext ctx, int Id)\n {\n ctx.Db.users.Id.Delete(Id);\n }\n}",
+ "total_tests": 1,
+ "passed_tests": 1,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"events\")]\n public partial struct Event\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public int Id;\n public string Kind;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientConnected)]\n public static void ClientConnected(ReducerContext ctx)\n {\n ctx.Db.events.Insert(new Event { Id = 0, Kind = \"connected\" });\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientDisconnected)]\n public static void ClientDisconnected(ReducerContext ctx)\n {\n ctx.Db.events.Insert(new Event { Id = 0, Kind = \"disconnected\" });\n }\n}",
"category": "basics",
"route_api_model": "gemini-2.5-flash",
- "golden_db": "basics-t-006-delete-golden",
- "llm_db": "basics-t-006-delete-gemini-2-5-flash-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\gemini-2-5-flash\\llm",
+ "golden_db": "basics-t-010-connect-golden",
+ "llm_db": "basics-t-010-connect-gemini-2-5-flash-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_010_connect\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_010_connect\\csharp\\server\\gemini-2-5-flash\\llm",
"scorer_details": {
- "seed_users_row": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
- }
- },
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-006-delete-golden",
- "llm_db": "basics-t-006-delete-gemini-2-5-flash-llm",
+ "golden_db": "basics-t-010-connect-golden",
+ "llm_db": "basics-t-010-connect-gemini-2-5-flash-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
"tables_diff": null,
"tables_equal": true
}
- },
- "delete_user_count_zero": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 0,
- "expected": 0,
- "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=1"
- }
- }
- },
- "vendor": "google",
- "started_at": "2025-10-21T18:19:53.008659Z",
- "finished_at": "2025-10-21T18:20:57.062808700Z"
- },
- "t_005_update": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_005_update",
- "lang": "csharp",
- "golden_published": false,
- "model_name": "Gemini 2.5 Flash",
- "total_tests": 1,
- "passed_tests": 0,
- "llm_output": null,
- "category": "basics",
- "route_api_model": "gemini-2.5-flash",
- "golden_db": null,
- "llm_db": null,
- "work_dir_golden": null,
- "work_dir_llm": null,
- "scorer_details": {
- "publish_error": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "POST https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:generateContent?key=AIzaSyCX9OLo722iLo4_gdYisaPYQXlK3qDBuik -> 429 Too Many Requests: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits.\\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_paid_tier_input_token_count, limit: 1000000\\nPlease retry in 58.970895358s.\",\n \"status\": \"RESOURCE_EXHAUSTED\",\n \"details\": [\n {\n \"@type\": \"type.googleapis.com/google.rpc.QuotaFailure\",\n \"violations\": [\n {\n \"quotaMetric\": \"generativelanguage.googleapis.com/generate_content_paid_tier_input_token_count\",\n \"quotaId\": \"GenerateContentPaidTierInputTokensPerModelPerMinute\",\n \"quotaDimensions\": {\n \"location\": \"global\",\n \"model\": \"gemini-2.5-flash\"\n },\n \"quotaValue\": \"1000000\"\n }\n ]\n },\n {\n \"@type\": \"type.googleapis.com/google.rpc.Help\",\n \"links\": [\n {\n \"description\": \"Learn more about Gemini API quotas\",\n \"url\": \"https://ai.google.dev/gemini-api/docs/rate-limits\"\n }\n ]\n },\n {\n \"@type\": \"type.googleapis.com/google.rpc.RetryInfo\",\n \"retryDelay\": \"58s\"\n }\n ]\n }\n}\n",
- "phase": "build_or_publish"
- }
}
},
"vendor": "google",
- "started_at": "2025-10-21T18:22:05.566510800Z",
- "finished_at": "2025-10-21T18:22:05.566510800Z"
+ "started_at": "2025-10-21T18:19:53.201774300Z",
+ "finished_at": "2025-10-21T18:20:57.462301100Z"
},
- "t_008_index_lookup": {
+ "t_011_helper_function": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_008_index_lookup",
+ "task": "t_011_helper_function",
"lang": "csharp",
"golden_published": true,
"model_name": "Gemini 2.5 Flash",
"total_tests": 3,
"passed_tests": 3,
- "llm_output": "using SpacetimeDB;\nusing System;\n\npublic static partial class Module\n{\n [Table(Name = \"users\", Public = true)]\n public partial struct User\n {\n [PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [Table(Name = \"results\", Public = true)]\n public partial struct Result\n {\n [PrimaryKey]\n public int Id;\n public string Name;\n }\n\n [Reducer]\n public static void LookupUserName(ReducerContext ctx, int id)\n {\n var user = ctx.Db.users.Id.Find(id);\n if (user.HasValue)\n {\n ctx.Db.results.Insert(new Result { Id = user.Value.Id, Name = user.Value.Name });\n }\n }\n}",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"results\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Sum;\n }\n\n private static int Add(int a, int b)\n {\n return a + b;\n }\n\n [SpacetimeDB.Reducer]\n public static void ComputeSum(ReducerContext ctx, int id, int a, int b)\n {\n ctx.Db.results.Insert(new Result { Id = id, Sum = Add(a, b) });\n }\n}",
"category": "basics",
"route_api_model": "gemini-2.5-flash",
- "golden_db": "basics-t-008-index-lookup-golden",
- "llm_db": "basics-t-008-index-lookup-gemini-2-5-flash-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\gemini-2-5-flash\\llm",
+ "golden_db": "basics-t-011-helper-function-golden",
+ "llm_db": "basics-t-011-helper-function-gemini-2-5-flash-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\gemini-2-5-flash\\llm",
"scorer_details": {
- "seed_user_row": {
+ "schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
- }
+ "golden_db": "basics-t-011-helper-function-golden",
+ "llm_db": "basics-t-011-helper-function-gemini-2-5-flash-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
+ }
+ },
+ "helper_func_sum_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "args": [
+ 1,
+ 2,
+ 3
+ ],
+ "golden_db": "basics-t-011-helper-function-golden",
+ "golden_out": "Id | Sum ----+----- 1 | 5",
+ "llm_db": "basics-t-011-helper-function-gemini-2-5-flash-llm",
+ "llm_out": "Id | Sum ----+----- 1 | 5",
+ "query": "SELECT Id, Sum FROM results WHERE Id=1",
+ "reducer": "ComputeSum",
+ "server": "local"
+ }
+ },
+ "helper_func_sum_abs": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1 AND Sum=5"
+ }
+ }
+ },
+ "vendor": "google",
+ "started_at": "2025-10-21T18:19:53.250788Z",
+ "finished_at": "2025-10-21T18:22:04.874212400Z"
+ },
+ "t_012_spacetime_product_type": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_012_spacetime_product_type",
+ "lang": "csharp",
+ "golden_published": true,
+ "model_name": "Gemini 2.5 Flash",
+ "total_tests": 3,
+ "passed_tests": 3,
+ "llm_output": "using SpacetimeDB;\nusing System;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Score\n {\n public int Left;\n public int Right;\n }\n\n [SpacetimeDB.Table(Name = \"results\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Score Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetScore(ReducerContext ctx, int id, int left, int right)\n {\n ctx.Db.results.Insert(new Result { Id = id, Value = new Score { Left = left, Right = right } });\n }\n}",
+ "category": "schema",
+ "route_api_model": "gemini-2.5-flash",
+ "golden_db": "schema-t-012-spacetime-product-type-golden",
+ "llm_db": "schema-t-012-spacetime-product-type-gemini-2-5-flash-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\gemini-2-5-flash\\llm",
+ "scorer_details": {
+ "product_type_row_count": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1"
+ }
},
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-008-index-lookup-golden",
- "llm_db": "basics-t-008-index-lookup-gemini-2-5-flash-llm",
+ "golden_db": "schema-t-012-spacetime-product-type-golden",
+ "llm_db": "schema-t-012-spacetime-product-type-gemini-2-5-flash-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -9903,26 +9814,28 @@
"tables_equal": true
}
},
- "index_lookup_projection_parity": {
+ "product_type_row_parity": {
"pass": true,
"partial": 1.0,
"notes": {
"args": [
- 1
+ 1,
+ 2,
+ 3
],
- "golden_db": "basics-t-008-index-lookup-golden",
- "golden_out": "Id | Name ----+--------- 1 | \"Alice\"",
- "llm_db": "basics-t-008-index-lookup-gemini-2-5-flash-llm",
- "llm_out": "Id | Name ----+--------- 1 | \"Alice\"",
- "query": "SELECT Id, Name FROM results WHERE Id=1",
- "reducer": "LookupUserName",
+ "golden_db": "schema-t-012-spacetime-product-type-golden",
+ "golden_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)",
+ "llm_db": "schema-t-012-spacetime-product-type-gemini-2-5-flash-llm",
+ "llm_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)",
+ "query": "SELECT Id, Value FROM results WHERE Id=1",
+ "reducer": "SetScore",
"server": "local"
}
}
},
"vendor": "google",
- "started_at": "2025-10-21T18:19:53.103991700Z",
- "finished_at": "2025-10-21T18:22:05.395134800Z"
+ "started_at": "2025-10-21T18:29:44.135466900Z",
+ "finished_at": "2025-10-21T18:31:07.610446400Z"
},
"t_013_spacetime_sum_type": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
@@ -9953,15 +9866,6 @@
"tables_equal": true
}
},
- "sum_type_row_count": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1"
- }
- },
"sum_type_row_parity": {
"pass": true,
"partial": 1.0,
@@ -9978,6 +9882,15 @@
"reducer": "SetCircle",
"server": "local"
}
+ },
+ "sum_type_row_count": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1"
+ }
}
},
"vendor": "google",
@@ -10041,61 +9954,117 @@
"started_at": "2025-10-21T18:29:44.261045200Z",
"finished_at": "2025-10-21T18:30:56.726289Z"
},
- "t_020_ecs": {
+ "t_015_product_type_columns": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_020_ecs",
+ "task": "t_015_product_type_columns",
"lang": "csharp",
"golden_published": true,
"model_name": "Gemini 2.5 Flash",
- "total_tests": 5,
- "passed_tests": 2,
- "llm_output": "using SpacetimeDB;\nusing System;\nusing System.Linq;\n\npublic static partial class Module\n{\n [Table(Name = \"entities\", Public = true)]\n public partial struct Entity\n {\n [PrimaryKey]\n public int Id;\n }\n\n [Table(Name = \"positions\", Public = true)]\n public partial struct Position\n {\n [PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [Table(Name = \"velocities\", Public = true)]\n public partial struct Velocity\n {\n [PrimaryKey]\n public int EntityId;\n public int VX;\n public int VY;\n }\n\n [Table(Name = \"next_positions\", Public = true)]\n public partial struct NextPosition\n {\n [PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.entities.Insert(new Entity { Id = 1 });\n ctx.Db.positions.Insert(new Position { EntityId = 1, X = 0, Y = 0 });\n ctx.Db.velocities.Insert(new Velocity { EntityId = 1, VX = 1, VY = 0 });\n\n ctx.Db.entities.Insert(new Entity { Id = 2 });\n ctx.Db.positions.Insert(new Position { EntityId = 2, X = 10, Y = 0 });\n ctx.Db.velocities.Insert(new Velocity { EntityId = 2, VX = -2, VY = 3 });\n }\n\n [Reducer]\n public static void Step(ReducerContext ctx)\n {\n foreach (var position in ctx.Db.positions.Iter())\n {\n var velocity = ctx.Db.velocities.EntityId.Find(position.EntityId);\n if (velocity.HasValue)\n {\n var newX = position.X + velocity.Value.VX;\n var newY = position.Y + velocity.Value.VY;\n\n var nextPosition = new NextPosition\n {\n EntityId = position.EntityId,\n X = newX,\n Y = newY\n };\n\n // Upsert: if exists, update; else, insert. PrimaryKey.Update handles this.\n ctx.Db.next_positions.EntityId.Update(nextPosition);\n }\n }\n }\n}",
+ "total_tests": 3,
+ "passed_tests": 0,
+ "llm_output": "// \n\nusing System;\nusing System.Linq;\nusing SpacetimeDB;\nusing SpacetimeDB.Reducer;\nusing SpacetimeDB.Table;\nusing SpacetimeDB.Type;\n\npublic static partial class Module\n{\n [Type]\n public partial struct Address\n {\n public string Street;\n public int Zip;\n }\n\n [Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [Table(Name = \"profiles\", Public = true)]\n public partial struct Profile\n {\n [PrimaryKey]\n public int Id;\n public Address Home;\n public Address Work;\n public Position Pos;\n }\n\n [Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.profiles.Insert(new Profile\n {\n Id = 1,\n Home = new Address { Street = \"1 Main\", Zip = 11111 },\n Work = new Address { Street = \"2 Broad\", Zip = 22222 },\n Pos = new Position { X = 7, Y = 9 }\n });\n }\n}",
"category": "schema",
"route_api_model": "gemini-2.5-flash",
- "golden_db": "schema-t-020-ecs-golden",
- "llm_db": "schema-t-020-ecs-gemini-2-5-flash-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\gemini-2-5-flash\\llm",
+ "golden_db": "schema-t-015-product-type-columns-golden",
+ "llm_db": "schema-t-015-product-type-columns-gemini-2-5-flash-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\gemini-2-5-flash\\llm",
"scorer_details": {
- "ecs_next_pos_entity1": {
+ "publish_error": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: SpacetimeDB.NoSuchRowException: The row was not found, e.g., in an update call\n at SpacetimeDB.Internal.FFI.CheckedStatus.Marshaller.ConvertToManaged(Errno )\n at SpacetimeDB.Internal.FFI.datastore_update_bsatn(TableId , IndexId , Span`1 , UInt32& )\n at SpacetimeDB.Internal.UniqueIndex`4[[SpacetimeDB.Internal.TableHandles.next_positions, StdbModule, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null],[Module.NextPosition, StdbModule, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null],[System.Int32, System.Private.CoreLib, Version=8.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e],[SpacetimeDB.BSATN.I32, SpacetimeDB.BSATN.Runtime, Version=1.5.0.0, Culture=neutral, PublicKeyToken=null]].DoUpdate(NextPosition )\n at SpacetimeDB.Internal.TableHandles.next_positions.EntityIdUniqueIndex.Update(NextPosition )\n at Module.Step(ReducerContext )\n at ModuleRegistration.Step.Invoke(BinaryReader , IReducerContext )\n at SpacetimeDB.Internal.Module.__call_reducer__(UInt32 id, UInt64 sender_0, UInt64 sender_1, UInt64 sender_2, UInt64 sender_3, UInt64 conn_id_0, UInt64 conn_id_1, Timestamp timestamp, BytesSource args, BytesSink error)\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:3000/v1/database/c200ee39ad5e309897ed5661d2c2058524c1e4fa53562b68479545fdd3c0e21d/call/Step)\n",
- "phase": "call_reducer"
+ "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\gemini-2-5-flash\\llm\\Lib.cs(7,19): error CS0234: The type or namespace name 'Reducer' does not exist in the namespace 'SpacetimeDB' (are you missing an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\gemini-2-5-flash\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\gemini-2-5-flash\\llm\\Lib.cs(8,19): error CS0234: The type or namespace name 'Table' does not exist in the namespace 'SpacetimeDB' (are you missing an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\gemini-2-5-flash\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\gemini-2-5-flash\\llm\\Lib.cs(9,19): error CS0234: The type or namespace name 'Type' does not exist in the namespace 'SpacetimeDB' (are you missing an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\gemini-2-5-flash\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\gemini-2-5-flash\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'profiles' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\gemini-2-5-flash\\llm\\StdbModule.csproj]\r\n",
+ "phase": "build_or_publish"
}
- },
- "ecs_seed_positions_count": {
+ }
+ },
+ "vendor": "google",
+ "started_at": "2025-10-21T18:29:44.307929500Z",
+ "finished_at": "2025-10-21T18:30:07.250420600Z"
+ },
+ "t_016_sum_type_columns": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_016_sum_type_columns",
+ "lang": "csharp",
+ "golden_published": true,
+ "model_name": "Gemini 2.5 Flash",
+ "total_tests": 3,
+ "passed_tests": 2,
+ "llm_output": "using SpacetimeDB;\nusing System;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : SpacetimeDB.TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"drawings\", Public = true)]\n public partial struct Drawing\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape A;\n public Shape B;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.drawings.Insert(new Drawing\n {\n Id = 1,\n A = new Shape.Circle(new Circle { Radius = 10 }),\n B = new Shape.Rectangle(new Rectangle { Width = 4, Height = 6 })\n });\n }\n}",
+ "category": "schema",
+ "route_api_model": "gemini-2.5-flash",
+ "golden_db": "schema-t-016-sum-type-columns-golden",
+ "llm_db": "schema-t-016-sum-type-columns-gemini-2-5-flash-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\gemini-2-5-flash\\llm",
+ "scorer_details": {
+ "schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 2,
- "expected": 2,
- "sql": "SELECT COUNT(*) AS n FROM positions"
+ "golden_db": "schema-t-016-sum-type-columns-golden",
+ "llm_db": "schema-t-016-sum-type-columns-gemini-2-5-flash-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
}
},
- "ecs_step_next_positions_count": {
- "pass": false,
- "partial": 0.0,
+ "sum_type_columns_row_count": {
+ "pass": true,
+ "partial": 1.0,
"notes": {
- "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: SpacetimeDB.NoSuchRowException: The row was not found, e.g., in an update call\n at SpacetimeDB.Internal.FFI.CheckedStatus.Marshaller.ConvertToManaged(Errno )\n at SpacetimeDB.Internal.FFI.datastore_update_bsatn(TableId , IndexId , Span`1 , UInt32& )\n at SpacetimeDB.Internal.UniqueIndex`4[[SpacetimeDB.Internal.TableHandles.next_positions, StdbModule, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null],[Module.NextPosition, StdbModule, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null],[System.Int32, System.Private.CoreLib, Version=8.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e],[SpacetimeDB.BSATN.I32, SpacetimeDB.BSATN.Runtime, Version=1.5.0.0, Culture=neutral, PublicKeyToken=null]].DoUpdate(NextPosition )\n at SpacetimeDB.Internal.TableHandles.next_positions.EntityIdUniqueIndex.Update(NextPosition )\n at Module.Step(ReducerContext )\n at ModuleRegistration.Step.Invoke(BinaryReader , IReducerContext )\n at SpacetimeDB.Internal.Module.__call_reducer__(UInt32 id, UInt64 sender_0, UInt64 sender_1, UInt64 sender_2, UInt64 sender_3, UInt64 conn_id_0, UInt64 conn_id_1, Timestamp timestamp, BytesSource args, BytesSink error)\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:3000/v1/database/c200ee39ad5e309897ed5661d2c2058524c1e4fa53562b68479545fdd3c0e21d/call/Step)\n",
- "phase": "call_reducer"
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM drawings WHERE Id=1"
}
},
- "ecs_next_pos_entity2": {
+ "sum_type_columns_row_parity": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: SpacetimeDB.NoSuchRowException: The row was not found, e.g., in an update call\n at SpacetimeDB.Internal.FFI.CheckedStatus.Marshaller.ConvertToManaged(Errno )\n at SpacetimeDB.Internal.FFI.datastore_update_bsatn(TableId , IndexId , Span`1 , UInt32& )\n at SpacetimeDB.Internal.UniqueIndex`4[[SpacetimeDB.Internal.TableHandles.next_positions, StdbModule, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null],[Module.NextPosition, StdbModule, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null],[System.Int32, System.Private.CoreLib, Version=8.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e],[SpacetimeDB.BSATN.I32, SpacetimeDB.BSATN.Runtime, Version=1.5.0.0, Culture=neutral, PublicKeyToken=null]].DoUpdate(NextPosition )\n at SpacetimeDB.Internal.TableHandles.next_positions.EntityIdUniqueIndex.Update(NextPosition )\n at Module.Step(ReducerContext )\n at ModuleRegistration.Step.Invoke(BinaryReader , IReducerContext )\n at SpacetimeDB.Internal.Module.__call_reducer__(UInt32 id, UInt64 sender_0, UInt64 sender_1, UInt64 sender_2, UInt64 sender_3, UInt64 conn_id_0, UInt64 conn_id_1, Timestamp timestamp, BytesSource args, BytesSink error)\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:3000/v1/database/c200ee39ad5e309897ed5661d2c2058524c1e4fa53562b68479545fdd3c0e21d/call/Step)\n",
- "phase": "call_reducer"
+ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: can't directly call special Init lifecycle reducer\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:3000/v1/database/c200a2db8e9e0283e4979c026c13ddd1e15efd8c5c0d117894c7a9bbec878b4c/call/Seed)\n",
+ "phase": "call_reducer_llm"
+ }
+ }
+ },
+ "vendor": "google",
+ "started_at": "2025-10-21T18:29:44.354927100Z",
+ "finished_at": "2025-10-21T18:31:07.912657800Z"
+ },
+ "t_017_scheduled_columns": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_017_scheduled_columns",
+ "lang": "csharp",
+ "golden_published": true,
+ "model_name": "Gemini 2.5 Flash",
+ "total_tests": 2,
+ "passed_tests": 2,
+ "llm_output": "using SpacetimeDB;\nusing System;\n\npublic static partial class Module\n{\n [Table(Name = \"tick_timer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [PrimaryKey, AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [Reducer]\n public static void Tick(ReducerContext ctx, TickTimer timer)\n {\n // Reducer logic\n }\n\n [Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.tick_timer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMicroseconds(50_000))\n });\n }\n}",
+ "category": "schema",
+ "route_api_model": "gemini-2.5-flash",
+ "golden_db": "schema-t-017-scheduled-columns-golden",
+ "llm_db": "schema-t-017-scheduled-columns-gemini-2-5-flash-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_017_scheduled_columns\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_017_scheduled_columns\\csharp\\server\\gemini-2-5-flash\\llm",
+ "scorer_details": {
+ "scheduled_seeded_one_row": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM tick_timer WHERE ScheduledId>=0"
}
},
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-020-ecs-golden",
- "llm_db": "schema-t-020-ecs-gemini-2-5-flash-llm",
+ "golden_db": "schema-t-017-scheduled-columns-golden",
+ "llm_db": "schema-t-017-scheduled-columns-gemini-2-5-flash-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -10105,163 +10074,194 @@
}
},
"vendor": "google",
- "started_at": "2025-10-21T18:29:44.544520200Z",
- "finished_at": "2025-10-21T18:30:54.717774100Z"
+ "started_at": "2025-10-21T18:29:44.403558400Z",
+ "finished_at": "2025-10-21T18:31:08.141931800Z"
},
- "t_019_many_to_many": {
+ "t_018_constraints": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_019_many_to_many",
+ "task": "t_018_constraints",
"lang": "csharp",
"golden_published": true,
"model_name": "Gemini 2.5 Flash",
- "total_tests": 5,
+ "total_tests": 3,
"passed_tests": 0,
- "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"users\", Public = true)]\n public partial struct User\n {\n [PrimaryKey]\n public int UserId;\n public string Name;\n }\n\n [Table(Name = \"groups\", Public = true)]\n public partial struct Group\n {\n [PrimaryKey]\n public int GroupId;\n public string Title;\n }\n\n [Table(Name = \"memberships\", Public = true)]\n [Index.BTree(Name = \"by_user\", Columns = new string[] { nameof(UserId) })]\n [Index.BTree(Name = \"by_group\", Columns = new string[] { nameof(GroupId) })]\n public partial struct Membership\n {\n [PrimaryKey]\n public int Id;\n public int UserId;\n public int GroupId;\n }\n\n [Reducer(ReducerKind.Init)]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.users.Insert(new User { UserId = 1, Name = \"Alice\" });\n ctx.Db.users.Insert(new User { UserId = 2, Name = \"Bob\" });\n\n ctx.Db.groups.Insert(new Group { GroupId = 10, Title = \"Admin\" });\n ctx.Db.groups.Insert(new Group { GroupId = 20, Title = \"Dev\" });\n\n ctx.Db.memberships.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\n ctx.Db.memberships.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\n ctx.Db.memberships.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\n }\n}",
+ "llm_output": "using SpacetimeDB;\nusing System;\nusing System.Collections.Generic;\n\npublic static partial class Module\n{\n [Table(Name = \"accounts\")]\n [Index.BTree(Name = \"by_name\", Columns = new string[] { nameof(Name) })]\n public partial struct Account\n {\n [PrimaryKey]\n public int Id;\n [Unique]\n public string Email;\n public string Name;\n }\n\n [Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.accounts.Insert(new Account { Id = 1, Email = \"a@example.com\", Name = \"Alice\" });\n ctx.Db.accounts.Insert(new Account { Id = 2, Email = \"b@example.com\", Name = \"Bob\" });\n }\n}",
"category": "schema",
"route_api_model": "gemini-2.5-flash",
- "golden_db": "schema-t-019-many-to-many-golden",
- "llm_db": "schema-t-019-many-to-many-gemini-2-5-flash-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\gemini-2-5-flash\\llm",
+ "golden_db": "schema-t-018-constraints-golden",
+ "llm_db": "schema-t-018-constraints-gemini-2-5-flash-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\gemini-2-5-flash\\llm",
"scorer_details": {
"publish_error": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\gemini-2-5-flash\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'groups' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\gemini-2-5-flash\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\gemini-2-5-flash\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(70,24): warning CS8981: The type name 'memberships' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\gemini-2-5-flash\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\gemini-2-5-flash\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(113,24): warning CS8981: The type name 'users' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\gemini-2-5-flash\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\gemini-2-5-flash\\llm\\Lib.cs(24,6): error CS0104: 'Index' is an ambiguous reference between 'SpacetimeDB.Index' and 'System.Index' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\gemini-2-5-flash\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\gemini-2-5-flash\\llm\\Lib.cs(25,6): error CS0104: 'Index' is an ambiguous reference between 'SpacetimeDB.Index' and 'System.Index' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\gemini-2-5-flash\\llm\\StdbModule.csproj]\r\n",
+ "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\gemini-2-5-flash\\llm\\Lib.cs(9,6): error CS0104: 'Index' is an ambiguous reference between 'SpacetimeDB.Index' and 'System.Index' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\gemini-2-5-flash\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\gemini-2-5-flash\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'accounts' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\gemini-2-5-flash\\llm\\StdbModule.csproj]\r\n",
"phase": "build_or_publish"
}
}
},
"vendor": "google",
- "started_at": "2025-10-21T18:29:44.497171300Z",
- "finished_at": "2025-10-21T18:30:07.594242500Z"
+ "started_at": "2025-10-21T18:29:44.449549900Z",
+ "finished_at": "2025-10-21T18:29:49.866622800Z"
},
- "t_015_product_type_columns": {
+ "t_019_many_to_many": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_015_product_type_columns",
+ "task": "t_019_many_to_many",
"lang": "csharp",
"golden_published": true,
"model_name": "Gemini 2.5 Flash",
- "total_tests": 3,
+ "total_tests": 5,
"passed_tests": 0,
- "llm_output": "// \n\nusing System;\nusing System.Linq;\nusing SpacetimeDB;\nusing SpacetimeDB.Reducer;\nusing SpacetimeDB.Table;\nusing SpacetimeDB.Type;\n\npublic static partial class Module\n{\n [Type]\n public partial struct Address\n {\n public string Street;\n public int Zip;\n }\n\n [Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [Table(Name = \"profiles\", Public = true)]\n public partial struct Profile\n {\n [PrimaryKey]\n public int Id;\n public Address Home;\n public Address Work;\n public Position Pos;\n }\n\n [Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.profiles.Insert(new Profile\n {\n Id = 1,\n Home = new Address { Street = \"1 Main\", Zip = 11111 },\n Work = new Address { Street = \"2 Broad\", Zip = 22222 },\n Pos = new Position { X = 7, Y = 9 }\n });\n }\n}",
+ "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"users\", Public = true)]\n public partial struct User\n {\n [PrimaryKey]\n public int UserId;\n public string Name;\n }\n\n [Table(Name = \"groups\", Public = true)]\n public partial struct Group\n {\n [PrimaryKey]\n public int GroupId;\n public string Title;\n }\n\n [Table(Name = \"memberships\", Public = true)]\n [Index.BTree(Name = \"by_user\", Columns = new string[] { nameof(UserId) })]\n [Index.BTree(Name = \"by_group\", Columns = new string[] { nameof(GroupId) })]\n public partial struct Membership\n {\n [PrimaryKey]\n public int Id;\n public int UserId;\n public int GroupId;\n }\n\n [Reducer(ReducerKind.Init)]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.users.Insert(new User { UserId = 1, Name = \"Alice\" });\n ctx.Db.users.Insert(new User { UserId = 2, Name = \"Bob\" });\n\n ctx.Db.groups.Insert(new Group { GroupId = 10, Title = \"Admin\" });\n ctx.Db.groups.Insert(new Group { GroupId = 20, Title = \"Dev\" });\n\n ctx.Db.memberships.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\n ctx.Db.memberships.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\n ctx.Db.memberships.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\n }\n}",
"category": "schema",
"route_api_model": "gemini-2.5-flash",
- "golden_db": "schema-t-015-product-type-columns-golden",
- "llm_db": "schema-t-015-product-type-columns-gemini-2-5-flash-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\gemini-2-5-flash\\llm",
+ "golden_db": "schema-t-019-many-to-many-golden",
+ "llm_db": "schema-t-019-many-to-many-gemini-2-5-flash-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\gemini-2-5-flash\\llm",
"scorer_details": {
"publish_error": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\gemini-2-5-flash\\llm\\Lib.cs(7,19): error CS0234: The type or namespace name 'Reducer' does not exist in the namespace 'SpacetimeDB' (are you missing an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\gemini-2-5-flash\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\gemini-2-5-flash\\llm\\Lib.cs(8,19): error CS0234: The type or namespace name 'Table' does not exist in the namespace 'SpacetimeDB' (are you missing an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\gemini-2-5-flash\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\gemini-2-5-flash\\llm\\Lib.cs(9,19): error CS0234: The type or namespace name 'Type' does not exist in the namespace 'SpacetimeDB' (are you missing an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\gemini-2-5-flash\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\gemini-2-5-flash\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'profiles' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\gemini-2-5-flash\\llm\\StdbModule.csproj]\r\n",
+ "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\gemini-2-5-flash\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'groups' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\gemini-2-5-flash\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\gemini-2-5-flash\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(70,24): warning CS8981: The type name 'memberships' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\gemini-2-5-flash\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\gemini-2-5-flash\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(113,24): warning CS8981: The type name 'users' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\gemini-2-5-flash\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\gemini-2-5-flash\\llm\\Lib.cs(24,6): error CS0104: 'Index' is an ambiguous reference between 'SpacetimeDB.Index' and 'System.Index' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\gemini-2-5-flash\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\gemini-2-5-flash\\llm\\Lib.cs(25,6): error CS0104: 'Index' is an ambiguous reference between 'SpacetimeDB.Index' and 'System.Index' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\gemini-2-5-flash\\llm\\StdbModule.csproj]\r\n",
"phase": "build_or_publish"
}
}
},
"vendor": "google",
- "started_at": "2025-10-21T18:29:44.307929500Z",
- "finished_at": "2025-10-21T18:30:07.250420600Z"
+ "started_at": "2025-10-21T18:29:44.497171300Z",
+ "finished_at": "2025-10-21T18:30:07.594242500Z"
},
- "t_018_constraints": {
+ "t_020_ecs": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_018_constraints",
+ "task": "t_020_ecs",
"lang": "csharp",
"golden_published": true,
"model_name": "Gemini 2.5 Flash",
- "total_tests": 3,
- "passed_tests": 0,
- "llm_output": "using SpacetimeDB;\nusing System;\nusing System.Collections.Generic;\n\npublic static partial class Module\n{\n [Table(Name = \"accounts\")]\n [Index.BTree(Name = \"by_name\", Columns = new string[] { nameof(Name) })]\n public partial struct Account\n {\n [PrimaryKey]\n public int Id;\n [Unique]\n public string Email;\n public string Name;\n }\n\n [Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.accounts.Insert(new Account { Id = 1, Email = \"a@example.com\", Name = \"Alice\" });\n ctx.Db.accounts.Insert(new Account { Id = 2, Email = \"b@example.com\", Name = \"Bob\" });\n }\n}",
+ "total_tests": 5,
+ "passed_tests": 2,
+ "llm_output": "using SpacetimeDB;\nusing System;\nusing System.Linq;\n\npublic static partial class Module\n{\n [Table(Name = \"entities\", Public = true)]\n public partial struct Entity\n {\n [PrimaryKey]\n public int Id;\n }\n\n [Table(Name = \"positions\", Public = true)]\n public partial struct Position\n {\n [PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [Table(Name = \"velocities\", Public = true)]\n public partial struct Velocity\n {\n [PrimaryKey]\n public int EntityId;\n public int VX;\n public int VY;\n }\n\n [Table(Name = \"next_positions\", Public = true)]\n public partial struct NextPosition\n {\n [PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.entities.Insert(new Entity { Id = 1 });\n ctx.Db.positions.Insert(new Position { EntityId = 1, X = 0, Y = 0 });\n ctx.Db.velocities.Insert(new Velocity { EntityId = 1, VX = 1, VY = 0 });\n\n ctx.Db.entities.Insert(new Entity { Id = 2 });\n ctx.Db.positions.Insert(new Position { EntityId = 2, X = 10, Y = 0 });\n ctx.Db.velocities.Insert(new Velocity { EntityId = 2, VX = -2, VY = 3 });\n }\n\n [Reducer]\n public static void Step(ReducerContext ctx)\n {\n foreach (var position in ctx.Db.positions.Iter())\n {\n var velocity = ctx.Db.velocities.EntityId.Find(position.EntityId);\n if (velocity.HasValue)\n {\n var newX = position.X + velocity.Value.VX;\n var newY = position.Y + velocity.Value.VY;\n\n var nextPosition = new NextPosition\n {\n EntityId = position.EntityId,\n X = newX,\n Y = newY\n };\n\n // Upsert: if exists, update; else, insert. PrimaryKey.Update handles this.\n ctx.Db.next_positions.EntityId.Update(nextPosition);\n }\n }\n }\n}",
"category": "schema",
"route_api_model": "gemini-2.5-flash",
- "golden_db": "schema-t-018-constraints-golden",
- "llm_db": "schema-t-018-constraints-gemini-2-5-flash-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\gemini-2-5-flash\\llm",
+ "golden_db": "schema-t-020-ecs-golden",
+ "llm_db": "schema-t-020-ecs-gemini-2-5-flash-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\gemini-2-5-flash\\llm",
"scorer_details": {
- "publish_error": {
+ "ecs_seed_positions_count": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 2,
+ "expected": 2,
+ "sql": "SELECT COUNT(*) AS n FROM positions"
+ }
+ },
+ "ecs_next_pos_entity2": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\gemini-2-5-flash\\llm\\Lib.cs(9,6): error CS0104: 'Index' is an ambiguous reference between 'SpacetimeDB.Index' and 'System.Index' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\gemini-2-5-flash\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\gemini-2-5-flash\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'accounts' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\gemini-2-5-flash\\llm\\StdbModule.csproj]\r\n",
- "phase": "build_or_publish"
+ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: SpacetimeDB.NoSuchRowException: The row was not found, e.g., in an update call\n at SpacetimeDB.Internal.FFI.CheckedStatus.Marshaller.ConvertToManaged(Errno )\n at SpacetimeDB.Internal.FFI.datastore_update_bsatn(TableId , IndexId , Span`1 , UInt32& )\n at SpacetimeDB.Internal.UniqueIndex`4[[SpacetimeDB.Internal.TableHandles.next_positions, StdbModule, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null],[Module.NextPosition, StdbModule, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null],[System.Int32, System.Private.CoreLib, Version=8.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e],[SpacetimeDB.BSATN.I32, SpacetimeDB.BSATN.Runtime, Version=1.5.0.0, Culture=neutral, PublicKeyToken=null]].DoUpdate(NextPosition )\n at SpacetimeDB.Internal.TableHandles.next_positions.EntityIdUniqueIndex.Update(NextPosition )\n at Module.Step(ReducerContext )\n at ModuleRegistration.Step.Invoke(BinaryReader , IReducerContext )\n at SpacetimeDB.Internal.Module.__call_reducer__(UInt32 id, UInt64 sender_0, UInt64 sender_1, UInt64 sender_2, UInt64 sender_3, UInt64 conn_id_0, UInt64 conn_id_1, Timestamp timestamp, BytesSource args, BytesSink error)\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:3000/v1/database/c200ee39ad5e309897ed5661d2c2058524c1e4fa53562b68479545fdd3c0e21d/call/Step)\n",
+ "phase": "call_reducer"
}
- }
- },
- "vendor": "google",
- "started_at": "2025-10-21T18:29:44.449549900Z",
- "finished_at": "2025-10-21T18:29:49.866622800Z"
- },
- "t_001_basic_tables": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_001_basic_tables",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "Gemini 2.5 Flash",
- "total_tests": 1,
- "passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\", Public = true)]\n public partial struct Users\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"products\", Public = true)]\n public partial struct Products\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Title;\n public float Price;\n public bool InStock;\n }\n\n [SpacetimeDB.Table(Name = \"notes\", Public = true)]\n public partial struct Notes\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Body;\n public long Rating;\n public bool Pinned;\n }\n}",
- "category": "basics",
- "route_api_model": "gemini-2.5-flash",
- "golden_db": "basics-t-001-basic-tables-golden",
- "llm_db": "basics-t-001-basic-tables-gemini-2-5-flash-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\gemini-2-5-flash\\llm",
- "scorer_details": {
+ },
+ "ecs_next_pos_entity1": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: SpacetimeDB.NoSuchRowException: The row was not found, e.g., in an update call\n at SpacetimeDB.Internal.FFI.CheckedStatus.Marshaller.ConvertToManaged(Errno )\n at SpacetimeDB.Internal.FFI.datastore_update_bsatn(TableId , IndexId , Span`1 , UInt32& )\n at SpacetimeDB.Internal.UniqueIndex`4[[SpacetimeDB.Internal.TableHandles.next_positions, StdbModule, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null],[Module.NextPosition, StdbModule, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null],[System.Int32, System.Private.CoreLib, Version=8.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e],[SpacetimeDB.BSATN.I32, SpacetimeDB.BSATN.Runtime, Version=1.5.0.0, Culture=neutral, PublicKeyToken=null]].DoUpdate(NextPosition )\n at SpacetimeDB.Internal.TableHandles.next_positions.EntityIdUniqueIndex.Update(NextPosition )\n at Module.Step(ReducerContext )\n at ModuleRegistration.Step.Invoke(BinaryReader , IReducerContext )\n at SpacetimeDB.Internal.Module.__call_reducer__(UInt32 id, UInt64 sender_0, UInt64 sender_1, UInt64 sender_2, UInt64 sender_3, UInt64 conn_id_0, UInt64 conn_id_1, Timestamp timestamp, BytesSource args, BytesSink error)\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:3000/v1/database/c200ee39ad5e309897ed5661d2c2058524c1e4fa53562b68479545fdd3c0e21d/call/Step)\n",
+ "phase": "call_reducer"
+ }
+ },
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-001-basic-tables-golden",
- "llm_db": "basics-t-001-basic-tables-gemini-2-5-flash-llm",
+ "golden_db": "schema-t-020-ecs-golden",
+ "llm_db": "schema-t-020-ecs-gemini-2-5-flash-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
"tables_diff": null,
"tables_equal": true
}
+ },
+ "ecs_step_next_positions_count": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: SpacetimeDB.NoSuchRowException: The row was not found, e.g., in an update call\n at SpacetimeDB.Internal.FFI.CheckedStatus.Marshaller.ConvertToManaged(Errno )\n at SpacetimeDB.Internal.FFI.datastore_update_bsatn(TableId , IndexId , Span`1 , UInt32& )\n at SpacetimeDB.Internal.UniqueIndex`4[[SpacetimeDB.Internal.TableHandles.next_positions, StdbModule, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null],[Module.NextPosition, StdbModule, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null],[System.Int32, System.Private.CoreLib, Version=8.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e],[SpacetimeDB.BSATN.I32, SpacetimeDB.BSATN.Runtime, Version=1.5.0.0, Culture=neutral, PublicKeyToken=null]].DoUpdate(NextPosition )\n at SpacetimeDB.Internal.TableHandles.next_positions.EntityIdUniqueIndex.Update(NextPosition )\n at Module.Step(ReducerContext )\n at ModuleRegistration.Step.Invoke(BinaryReader , IReducerContext )\n at SpacetimeDB.Internal.Module.__call_reducer__(UInt32 id, UInt64 sender_0, UInt64 sender_1, UInt64 sender_2, UInt64 sender_3, UInt64 conn_id_0, UInt64 conn_id_1, Timestamp timestamp, BytesSource args, BytesSink error)\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:3000/v1/database/c200ee39ad5e309897ed5661d2c2058524c1e4fa53562b68479545fdd3c0e21d/call/Step)\n",
+ "phase": "call_reducer"
+ }
}
},
"vendor": "google",
- "started_at": "2025-10-21T18:19:52.763159300Z",
- "finished_at": "2025-10-21T18:20:57.330741700Z"
+ "started_at": "2025-10-21T18:29:44.544520200Z",
+ "finished_at": "2025-10-21T18:30:54.717774100Z"
},
- "t_002_scheduled_table": {
+ "t_021_multi_column_index": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_002_scheduled_table",
+ "task": "t_021_multi_column_index",
"lang": "csharp",
"golden_published": true,
"model_name": "Gemini 2.5 Flash",
- "total_tests": 1,
- "passed_tests": 1,
- "llm_output": "using SpacetimeDB;\nusing System;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"tick_timer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [SpacetimeDB.PrimaryKey, SpacetimeDB.AutoInc]\n public ulong ScheduledId;\n public SpacetimeDB.ScheduleAt ScheduledAt;\n }\n\n [SpacetimeDB.Reducer]\n public static void Tick(ReducerContext ctx, TickTimer timer)\n {\n // Reducer logic\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.tick_timer.Insert(new TickTimer\n {\n ScheduledId = 0, // Auto-increment will assign a value\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMicroseconds(50_000))\n });\n }\n}",
- "category": "basics",
+ "total_tests": 4,
+ "passed_tests": 4,
+ "llm_output": "using SpacetimeDB;\nusing System;\n\npublic static partial class Module\n{\n [Table(Name = \"logs\")]\n [SpacetimeDB.Index.BTree(Name = \"by_user_day\", Columns = new string[] { nameof(UserId), nameof(Day) })]\n public partial struct Log\n {\n [PrimaryKey]\n public int Id;\n public int UserId;\n public int Day;\n public string Message;\n }\n\n [Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.logs.Insert(new Log { Id = 1, UserId = 7, Day = 1, Message = \"a\" });\n ctx.Db.logs.Insert(new Log { Id = 2, UserId = 7, Day = 2, Message = \"b\" });\n ctx.Db.logs.Insert(new Log { Id = 3, UserId = 9, Day = 1, Message = \"c\" });\n }\n}",
+ "category": "schema",
"route_api_model": "gemini-2.5-flash",
- "golden_db": "basics-t-002-scheduled-table-golden",
- "llm_db": "basics-t-002-scheduled-table-gemini-2-5-flash-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_002_scheduled_table\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_002_scheduled_table\\csharp\\server\\gemini-2-5-flash\\llm",
+ "golden_db": "schema-t-021-multi-column-index-golden",
+ "llm_db": "schema-t-021-multi-column-index-gemini-2-5-flash-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\gemini-2-5-flash\\llm",
"scorer_details": {
+ "mcindex_lookup_u7_d1": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM logs WHERE UserId=7 AND Day=1"
+ }
+ },
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-002-scheduled-table-golden",
- "llm_db": "basics-t-002-scheduled-table-gemini-2-5-flash-llm",
+ "golden_db": "schema-t-021-multi-column-index-golden",
+ "llm_db": "schema-t-021-multi-column-index-gemini-2-5-flash-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
"tables_diff": null,
"tables_equal": true
}
+ },
+ "mcindex_lookup_u7_d2": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM logs WHERE UserId=7 AND Day=2"
+ }
+ },
+ "mcindex_seed_count": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 3,
+ "expected": 3,
+ "sql": "SELECT COUNT(*) AS n FROM logs"
+ }
}
},
"vendor": "google",
- "started_at": "2025-10-21T18:19:52.810971500Z",
- "finished_at": "2025-10-21T18:22:05.564885200Z"
+ "started_at": "2025-10-21T18:29:44.593009700Z",
+ "finished_at": "2025-10-21T18:30:54.000181400Z"
}
}
},
@@ -10269,80 +10269,62 @@
"name": "Claude 4.5 Haiku",
"route_api_model": "claude-haiku-4-5",
"tasks": {
- "t_017_scheduled_columns": {
+ "t_000_empty_reducers": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_017_scheduled_columns",
+ "task": "t_000_empty_reducers",
"lang": "csharp",
"golden_published": true,
"model_name": "Claude 4.5 Haiku",
- "total_tests": 2,
- "passed_tests": 0,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"tick_timer\", Scheduled = nameof(Tick), ScheduledAt = nameof(scheduled_at))]\n public partial struct TickTimer\n {\n [PrimaryKey, AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.tick_timer.Insert(new TickTimer\n {\n ScheduledId = 0,\n ScheduledAt = new ScheduleAt.Interval(new TimeDuration { Microseconds = 50_000 })\n });\n }\n\n [Reducer]\n public static void Tick(ReducerContext ctx, TickTimer timer)\n {\n }\n}",
- "category": "schema",
+ "total_tests": 1,
+ "passed_tests": 1,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Reducer]\n public static void EmptyReducer_NoArgs(ReducerContext ctx)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithInt(ReducerContext ctx, int count)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithString(ReducerContext ctx, string name)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithTwoArgs(ReducerContext ctx, int count, string name)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithThreeArgs(ReducerContext ctx, bool active, float ratio, string label)\n {\n }\n}",
+ "category": "basics",
"route_api_model": "claude-haiku-4-5",
- "golden_db": "schema-t-017-scheduled-columns-golden",
- "llm_db": "schema-t-017-scheduled-columns-claude-4-5-haiku-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_017_scheduled_columns\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_017_scheduled_columns\\csharp\\server\\claude-4-5-haiku\\llm",
+ "golden_db": "basics-t-000-empty-reducers-golden",
+ "llm_db": "basics-t-000-empty-reducers-claude-4-5-haiku-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\claude-4-5-haiku\\llm",
"scorer_details": {
- "publish_error": {
- "pass": false,
- "partial": 0.0,
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
"notes": {
- "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_017_scheduled_columns\\csharp\\server\\claude-4-5-haiku\\llm\\Lib.cs(6,6): error STDB0014: Could not find the specified column scheduled_at in TickTimer. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_017_scheduled_columns\\csharp\\server\\claude-4-5-haiku\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_017_scheduled_columns\\csharp\\server\\claude-4-5-haiku\\llm\\Lib.cs(6,80): error CS0103: The name 'scheduled_at' does not exist in the current context [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_017_scheduled_columns\\csharp\\server\\claude-4-5-haiku\\llm\\StdbModule.csproj]\r\n",
- "phase": "build_or_publish"
+ "golden_db": "basics-t-000-empty-reducers-golden",
+ "llm_db": "basics-t-000-empty-reducers-claude-4-5-haiku-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
}
}
},
"vendor": "anthropic",
- "started_at": "2025-10-21T21:21:57.636347400Z",
- "finished_at": "2025-10-21T21:22:10.168663100Z"
+ "started_at": "2025-10-21T21:15:55.039324800Z",
+ "finished_at": "2025-10-21T21:17:35.767014400Z"
},
- "t_014_elementary_columns": {
+ "t_001_basic_tables": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_014_elementary_columns",
+ "task": "t_001_basic_tables",
"lang": "csharp",
"golden_published": true,
"model_name": "Claude 4.5 Haiku",
- "total_tests": 3,
- "passed_tests": 3,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"primitives\")]\n public partial struct Primitive\n {\n [PrimaryKey]\n public int Id;\n public int Count;\n public long Total;\n public float Price;\n public double Ratio;\n public bool Active;\n public string Name;\n }\n\n [Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.primitives.Insert(new Primitive\n {\n Id = 1,\n Count = 2,\n Total = 3000000000,\n Price = 1.5f,\n Ratio = 2.25,\n Active = true,\n Name = \"Alice\"\n });\n }\n}",
- "category": "schema",
+ "total_tests": 1,
+ "passed_tests": 1,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"users\", Public = true)]\n public partial struct Users\n {\n [PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [Table(Name = \"products\", Public = true)]\n public partial struct Products\n {\n [PrimaryKey]\n public int Id;\n public string Title;\n public float Price;\n public bool InStock;\n }\n\n [Table(Name = \"notes\", Public = true)]\n public partial struct Notes\n {\n [PrimaryKey]\n public int Id;\n public string Body;\n public long Rating;\n public bool Pinned;\n }\n}",
+ "category": "basics",
"route_api_model": "claude-haiku-4-5",
- "golden_db": "schema-t-014-elementary-columns-golden",
- "llm_db": "schema-t-014-elementary-columns-claude-4-5-haiku-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_014_elementary_columns\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_014_elementary_columns\\csharp\\server\\claude-4-5-haiku\\llm",
+ "golden_db": "basics-t-001-basic-tables-golden",
+ "llm_db": "basics-t-001-basic-tables-claude-4-5-haiku-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\claude-4-5-haiku\\llm",
"scorer_details": {
- "elementary_columns_row_count": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM primitives WHERE Id=1"
- }
- },
- "elementary_columns_row_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "args": [],
- "golden_db": "schema-t-014-elementary-columns-golden",
- "golden_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"",
- "llm_db": "schema-t-014-elementary-columns-claude-4-5-haiku-llm",
- "llm_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"",
- "query": "SELECT Id, Count, Total, Price, Ratio, Active, Name FROM primitives WHERE Id=1",
- "reducer": "Seed",
- "server": "local"
- }
- },
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-014-elementary-columns-golden",
- "llm_db": "schema-t-014-elementary-columns-claude-4-5-haiku-llm",
+ "golden_db": "basics-t-001-basic-tables-golden",
+ "llm_db": "basics-t-001-basic-tables-claude-4-5-haiku-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -10352,8 +10334,8 @@
}
},
"vendor": "anthropic",
- "started_at": "2025-10-21T21:21:55.454990Z",
- "finished_at": "2025-10-21T21:23:13.070138300Z"
+ "started_at": "2025-10-21T21:15:55.834612100Z",
+ "finished_at": "2025-10-21T21:17:34.859654500Z"
},
"t_002_scheduled_table": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
@@ -10389,161 +10371,39 @@
"started_at": "2025-10-21T21:15:56.641560400Z",
"finished_at": "2025-10-21T21:17:35.024405300Z"
},
- "t_008_index_lookup": {
+ "t_003_struct_in_table": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_008_index_lookup",
+ "task": "t_003_struct_in_table",
"lang": "csharp",
"golden_published": true,
"model_name": "Claude 4.5 Haiku",
- "total_tests": 3,
- "passed_tests": 3,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"users\", Public = true)]\n public partial struct User\n {\n [PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [Table(Name = \"results\", Public = true)]\n public partial struct Result\n {\n [PrimaryKey]\n public int Id;\n public string Name;\n }\n\n [Reducer]\n public static void LookupUserName(ReducerContext ctx, int id)\n {\n var user = ctx.Db.users.Id.Find(id);\n if (user.HasValue)\n {\n ctx.Db.results.Insert(new Result\n {\n Id = user.Value.Id,\n Name = user.Value.Name\n });\n }\n }\n}",
+ "total_tests": 1,
+ "passed_tests": 1,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"entities\", Public = true)]\n public partial struct Entity\n {\n [PrimaryKey]\n public int Id;\n public Position Pos;\n }\n}",
"category": "basics",
"route_api_model": "claude-haiku-4-5",
- "golden_db": "basics-t-008-index-lookup-golden",
- "llm_db": "basics-t-008-index-lookup-claude-4-5-haiku-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\claude-4-5-haiku\\llm",
+ "golden_db": "basics-t-003-struct-in-table-golden",
+ "llm_db": "basics-t-003-struct-in-table-claude-4-5-haiku-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_003_struct_in_table\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_003_struct_in_table\\csharp\\server\\claude-4-5-haiku\\llm",
"scorer_details": {
- "index_lookup_projection_parity": {
+ "schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "args": [
- 1
- ],
- "golden_db": "basics-t-008-index-lookup-golden",
- "golden_out": "Id | Name ----+--------- 1 | \"Alice\"",
- "llm_db": "basics-t-008-index-lookup-claude-4-5-haiku-llm",
- "llm_out": "Id | Name ----+--------- 1 | \"Alice\"",
- "query": "SELECT Id, Name FROM results WHERE Id=1",
- "reducer": "LookupUserName",
- "server": "local"
- }
- },
- "seed_user_row": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
- }
- },
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "golden_db": "basics-t-008-index-lookup-golden",
- "llm_db": "basics-t-008-index-lookup-claude-4-5-haiku-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
- }
- }
- },
- "vendor": "anthropic",
- "started_at": "2025-10-21T21:16:01.281757Z",
- "finished_at": "2025-10-21T21:17:34.692711Z"
- },
- "t_018_constraints": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_018_constraints",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "Claude 4.5 Haiku",
- "total_tests": 3,
- "passed_tests": 0,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"accounts\", Public = true)]\n [Index(Name = \"by_name\", BTree = [nameof(Name)])]\n public partial struct Account\n {\n [PrimaryKey]\n public int Id;\n\n [Unique]\n public string Email;\n\n public string Name;\n }\n\n [Reducer(ReducerKind.Init)]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.accounts.Insert(new Account\n {\n Id = 1,\n Email = \"a@example.com\",\n Name = \"Alice\"\n });\n\n ctx.Db.accounts.Insert(new Account\n {\n Id = 2,\n Email = \"b@example.com\",\n Name = \"Bob\"\n });\n }\n}",
- "category": "schema",
- "route_api_model": "claude-haiku-4-5",
- "golden_db": "schema-t-018-constraints-golden",
- "llm_db": "schema-t-018-constraints-claude-4-5-haiku-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\claude-4-5-haiku\\llm",
- "scorer_details": {
- "publish_error": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\claude-4-5-haiku\\llm\\Lib.cs(7,6): error CS0104: 'Index' is an ambiguous reference between 'SpacetimeDB.Index' and 'System.Index' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\claude-4-5-haiku\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\claude-4-5-haiku\\llm\\Lib.cs(7,38): error CS9176: There is no target type for the collection expression. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\claude-4-5-haiku\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\claude-4-5-haiku\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'accounts' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\claude-4-5-haiku\\llm\\StdbModule.csproj]\r\n",
- "phase": "build_or_publish"
- }
- }
- },
- "vendor": "anthropic",
- "started_at": "2025-10-21T21:21:58.345339Z",
- "finished_at": "2025-10-21T21:22:11.208616800Z"
- },
- "t_020_ecs": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_020_ecs",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "Claude 4.5 Haiku",
- "total_tests": 5,
- "passed_tests": 5,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"entities\", Public = true)]\n public partial struct Entity\n {\n [PrimaryKey]\n public int Id;\n }\n\n [Table(Name = \"positions\", Public = true)]\n public partial struct Position\n {\n [PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [Table(Name = \"velocities\", Public = true)]\n public partial struct Velocity\n {\n [PrimaryKey]\n public int EntityId;\n public int VX;\n public int VY;\n }\n\n [Table(Name = \"next_positions\", Public = true)]\n public partial struct NextPosition\n {\n [PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.entities.Insert(new Entity { Id = 1 });\n ctx.Db.positions.Insert(new Position { EntityId = 1, X = 0, Y = 0 });\n ctx.Db.velocities.Insert(new Velocity { EntityId = 1, VX = 1, VY = 0 });\n\n ctx.Db.entities.Insert(new Entity { Id = 2 });\n ctx.Db.positions.Insert(new Position { EntityId = 2, X = 10, Y = 0 });\n ctx.Db.velocities.Insert(new Velocity { EntityId = 2, VX = -2, VY = 3 });\n }\n\n [Reducer]\n public static void Step(ReducerContext ctx)\n {\n foreach (var position in ctx.Db.positions.Iter())\n {\n var velocity = ctx.Db.velocities.EntityId.Find(position.EntityId);\n if (velocity.HasValue)\n {\n var nextX = position.X + velocity.Value.VX;\n var nextY = position.Y + velocity.Value.VY;\n\n var existingNextPos = ctx.Db.next_positions.EntityId.Find(position.EntityId);\n if (existingNextPos.HasValue)\n {\n ctx.Db.next_positions.EntityId.Update(new NextPosition\n {\n EntityId = position.EntityId,\n X = nextX,\n Y = nextY\n });\n }\n else\n {\n ctx.Db.next_positions.Insert(new NextPosition\n {\n EntityId = position.EntityId,\n X = nextX,\n Y = nextY\n });\n }\n }\n }\n }\n}",
- "category": "schema",
- "route_api_model": "claude-haiku-4-5",
- "golden_db": "schema-t-020-ecs-golden",
- "llm_db": "schema-t-020-ecs-claude-4-5-haiku-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\claude-4-5-haiku\\llm",
- "scorer_details": {
- "ecs_next_pos_entity2": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM next_positions WHERE EntityId=2 AND X=8 AND Y=3"
- }
- },
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "golden_db": "schema-t-020-ecs-golden",
- "llm_db": "schema-t-020-ecs-claude-4-5-haiku-llm",
+ "golden_db": "basics-t-003-struct-in-table-golden",
+ "llm_db": "basics-t-003-struct-in-table-claude-4-5-haiku-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
"tables_diff": null,
"tables_equal": true
}
- },
- "ecs_step_next_positions_count": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 2,
- "expected": 2,
- "sql": "SELECT COUNT(*) AS n FROM next_positions"
- }
- },
- "ecs_next_pos_entity1": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM next_positions WHERE EntityId=1 AND X=1 AND Y=0"
- }
- },
- "ecs_seed_positions_count": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 2,
- "expected": 2,
- "sql": "SELECT COUNT(*) AS n FROM positions"
- }
}
},
"vendor": "anthropic",
- "started_at": "2025-10-21T21:21:59.768557Z",
- "finished_at": "2025-10-21T21:23:14.012132300Z"
+ "started_at": "2025-10-21T21:15:57.421580100Z",
+ "finished_at": "2025-10-21T21:17:35.897725Z"
},
"t_004_insert": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
@@ -10598,89 +10458,54 @@
"started_at": "2025-10-21T21:15:58.192811500Z",
"finished_at": "2025-10-21T21:17:36.937240800Z"
},
- "t_011_helper_function": {
+ "t_005_update": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_011_helper_function",
+ "task": "t_005_update",
"lang": "csharp",
"golden_published": true,
"model_name": "Claude 4.5 Haiku",
"total_tests": 3,
"passed_tests": 3,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"results\")]\n public partial struct Result\n {\n [PrimaryKey]\n public int Id;\n public int Sum;\n }\n\n private static int Add(int a, int b)\n {\n return a + b;\n }\n\n [Reducer]\n public static void ComputeSum(ReducerContext ctx, int id, int a, int b)\n {\n int sum = Add(a, b);\n ctx.Db.results.Insert(new Result { Id = id, Sum = sum });\n }\n}",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"users\", Public = true)]\n public partial struct User\n {\n [PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [Reducer]\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n var user = ctx.Db.users.Id.Find(id);\n if (user.HasValue)\n {\n ctx.Db.users.Id.Update(new User\n {\n Id = id,\n Name = name,\n Age = age,\n Active = active\n });\n }\n }\n}",
"category": "basics",
"route_api_model": "claude-haiku-4-5",
- "golden_db": "basics-t-011-helper-function-golden",
- "llm_db": "basics-t-011-helper-function-claude-4-5-haiku-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\claude-4-5-haiku\\llm",
+ "golden_db": "basics-t-005-update-golden",
+ "llm_db": "basics-t-005-update-claude-4-5-haiku-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_005_update\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_005_update\\csharp\\server\\claude-4-5-haiku\\llm",
"scorer_details": {
- "helper_func_sum_abs": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1 AND Sum=5"
- }
- },
- "schema_parity": {
+ "seed_users_row": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-011-helper-function-golden",
- "llm_db": "basics-t-011-helper-function-claude-4-5-haiku-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
+ "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
}
},
- "helper_func_sum_parity": {
+ "data_parity_update_user": {
"pass": true,
"partial": 1.0,
"notes": {
"args": [
1,
- 2,
- 3
+ "Alice2",
+ 31,
+ false
],
- "golden_db": "basics-t-011-helper-function-golden",
- "golden_out": "Id | Sum ----+----- 1 | 5",
- "llm_db": "basics-t-011-helper-function-claude-4-5-haiku-llm",
- "llm_out": "Id | Sum ----+----- 1 | 5",
- "query": "SELECT Id, Sum FROM results WHERE Id=1",
- "reducer": "ComputeSum",
+ "golden_db": "basics-t-005-update-golden",
+ "golden_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
+ "llm_db": "basics-t-005-update-claude-4-5-haiku-llm",
+ "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
+ "query": "SELECT Id, Name, Age, Active FROM users WHERE Id=1",
+ "reducer": "UpdateUser",
"server": "local"
}
- }
- },
- "vendor": "anthropic",
- "started_at": "2025-10-21T21:16:03.603944800Z",
- "finished_at": "2025-10-21T21:17:37.429004300Z"
- },
- "t_000_empty_reducers": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_000_empty_reducers",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "Claude 4.5 Haiku",
- "total_tests": 1,
- "passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Reducer]\n public static void EmptyReducer_NoArgs(ReducerContext ctx)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithInt(ReducerContext ctx, int count)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithString(ReducerContext ctx, string name)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithTwoArgs(ReducerContext ctx, int count, string name)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithThreeArgs(ReducerContext ctx, bool active, float ratio, string label)\n {\n }\n}",
- "category": "basics",
- "route_api_model": "claude-haiku-4-5",
- "golden_db": "basics-t-000-empty-reducers-golden",
- "llm_db": "basics-t-000-empty-reducers-claude-4-5-haiku-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\claude-4-5-haiku\\llm",
- "scorer_details": {
+ },
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-000-empty-reducers-golden",
- "llm_db": "basics-t-000-empty-reducers-claude-4-5-haiku-llm",
+ "golden_db": "basics-t-005-update-golden",
+ "llm_db": "basics-t-005-update-claude-4-5-haiku-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -10690,48 +10515,40 @@
}
},
"vendor": "anthropic",
- "started_at": "2025-10-21T21:15:55.039324800Z",
- "finished_at": "2025-10-21T21:17:35.767014400Z"
+ "started_at": "2025-10-21T21:15:58.946859500Z",
+ "finished_at": "2025-10-21T21:17:35.574459100Z"
},
- "t_013_spacetime_sum_type": {
+ "t_006_delete": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_013_spacetime_sum_type",
+ "task": "t_006_delete",
"lang": "csharp",
"golden_published": true,
"model_name": "Claude 4.5 Haiku",
"total_tests": 3,
"passed_tests": 3,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial record Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial record Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : SpacetimeDB.TaggedEnum<(Circle Circle, Rectangle Rectangle)>\n {\n }\n\n [Table(Name = \"results\")]\n public partial struct Result\n {\n [PrimaryKey]\n public int Id;\n public Shape Value;\n }\n\n [Reducer]\n public static void SetCircle(ReducerContext ctx, int id, int radius)\n {\n var circle = new Circle { Radius = radius };\n var shape = new Shape.Circle(circle);\n var result = new Result { Id = id, Value = shape };\n ctx.Db.results.Insert(result);\n }\n}",
- "category": "schema",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"users\", Public = true)]\n public partial struct User\n {\n [PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [Reducer]\n public static void DeleteUser(ReducerContext ctx, int id)\n {\n var user = ctx.Db.users.Id.Find(id);\n if (user.HasValue)\n {\n ctx.Db.users.Id.Delete(user.Value.Id);\n }\n }\n}",
+ "category": "basics",
"route_api_model": "claude-haiku-4-5",
- "golden_db": "schema-t-013-spacetime-sum-type-golden",
- "llm_db": "schema-t-013-spacetime-sum-type-claude-4-5-haiku-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\claude-4-5-haiku\\llm",
+ "golden_db": "basics-t-006-delete-golden",
+ "llm_db": "basics-t-006-delete-claude-4-5-haiku-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\claude-4-5-haiku\\llm",
"scorer_details": {
- "sum_type_row_parity": {
+ "delete_user_count_zero": {
"pass": true,
"partial": 1.0,
"notes": {
- "args": [
- 1,
- 10
- ],
- "golden_db": "schema-t-013-spacetime-sum-type-golden",
- "golden_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))",
- "llm_db": "schema-t-013-spacetime-sum-type-claude-4-5-haiku-llm",
- "llm_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))",
- "query": "SELECT Id, Value FROM results WHERE Id=1",
- "reducer": "SetCircle",
- "server": "local"
+ "actual": 0,
+ "expected": 0,
+ "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=1"
}
},
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-013-spacetime-sum-type-golden",
- "llm_db": "schema-t-013-spacetime-sum-type-claude-4-5-haiku-llm",
+ "golden_db": "basics-t-006-delete-golden",
+ "llm_db": "basics-t-006-delete-claude-4-5-haiku-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -10739,82 +10556,17 @@
"tables_equal": true
}
},
- "sum_type_row_count": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1"
- }
- }
- },
- "vendor": "anthropic",
- "started_at": "2025-10-21T21:21:54.694207Z",
- "finished_at": "2025-10-21T21:23:14.796865500Z"
- },
- "t_003_struct_in_table": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_003_struct_in_table",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "Claude 4.5 Haiku",
- "total_tests": 1,
- "passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"entities\", Public = true)]\n public partial struct Entity\n {\n [PrimaryKey]\n public int Id;\n public Position Pos;\n }\n}",
- "category": "basics",
- "route_api_model": "claude-haiku-4-5",
- "golden_db": "basics-t-003-struct-in-table-golden",
- "llm_db": "basics-t-003-struct-in-table-claude-4-5-haiku-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_003_struct_in_table\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_003_struct_in_table\\csharp\\server\\claude-4-5-haiku\\llm",
- "scorer_details": {
- "schema_parity": {
+ "seed_users_row": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-003-struct-in-table-golden",
- "llm_db": "basics-t-003-struct-in-table-claude-4-5-haiku-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
- }
- }
- },
- "vendor": "anthropic",
- "started_at": "2025-10-21T21:15:57.421580100Z",
- "finished_at": "2025-10-21T21:17:35.897725Z"
- },
- "t_015_product_type_columns": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_015_product_type_columns",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "Claude 4.5 Haiku",
- "total_tests": 3,
- "passed_tests": 0,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Address\n {\n [SpacetimeDB.Type]\n public string Street;\n [SpacetimeDB.Type]\n public int Zip;\n\n public Address(string street, int zip)\n {\n Street = street;\n Zip = zip;\n }\n }\n\n [SpacetimeDB.Type]\n public partial struct Position\n {\n [SpacetimeDB.Type]\n public int X;\n [SpacetimeDB.Type]\n public int Y;\n\n public Position(int x, int y)\n {\n X = x;\n Y = y;\n }\n }\n\n [Table(Name = \"profiles\", Public = true)]\n public partial struct Profile\n {\n [PrimaryKey]\n public int Id;\n public Address Home;\n public Address Work;\n public Position Pos;\n }\n\n [Reducer]\n public static void Seed(ReducerContext ctx)\n {\n var profile = new Profile\n {\n Id = 1,\n Home = new Address(Street: \"1 Main\", Zip: 11111),\n Work = new Address(Street: \"2 Broad\", Zip: 22222),\n Pos = new Position(X: 7, Y: 9)\n };\n\n ctx.Db.profiles.Insert(profile);\n }\n}",
- "category": "schema",
- "route_api_model": "claude-haiku-4-5",
- "golden_db": "schema-t-015-product-type-columns-golden",
- "llm_db": "schema-t-015-product-type-columns-claude-4-5-haiku-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\claude-4-5-haiku\\llm",
- "scorer_details": {
- "publish_error": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\claude-4-5-haiku\\llm\\Lib.cs(10,23): error STDBINT0001: An internal error occurred during codegen: Unable to cast object of type 'Microsoft.CodeAnalysis.CSharp.Syntax.VariableDeclaratorSyntax' to type 'Microsoft.CodeAnalysis.CSharp.Syntax.TypeDeclarationSyntax'. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\claude-4-5-haiku\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\claude-4-5-haiku\\llm\\Lib.cs(12,20): error STDBINT0001: An internal error occurred during codegen: Unable to cast object of type 'Microsoft.CodeAnalysis.CSharp.Syntax.VariableDeclaratorSyntax' to type 'Microsoft.CodeAnalysis.CSharp.Syntax.TypeDeclarationSyntax'. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\claude-4-5-haiku\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\claude-4-5-haiku\\llm\\Lib.cs(25,20): error STDBINT0001: An internal error occurred during codegen: Unable to cast object of type 'Microsoft.CodeAnalysis.CSharp.Syntax.VariableDeclaratorSyntax' to type 'Microsoft.CodeAnalysis.CSharp.Syntax.TypeDeclarationSyntax'. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\claude-4-5-haiku\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\claude-4-5-haiku\\llm\\Lib.cs(27,20): error STDBINT0001: An internal error occurred during codegen: Unable to cast object of type 'Microsoft.CodeAnalysis.CSharp.Syntax.VariableDeclaratorSyntax' to type 'Microsoft.CodeAnalysis.CSharp.Syntax.TypeDeclarationSyntax'. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\claude-4-5-haiku\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\claude-4-5-haiku\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'profiles' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\claude-4-5-haiku\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\claude-4-5-haiku\\llm\\Lib.cs(9,10): error CS0592: Attribute 'SpacetimeDB.Type' is not valid on this declaration type. It is only valid on 'class, struct, enum' declarations. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\claude-4-5-haiku\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\claude-4-5-haiku\\llm\\Lib.cs(11,10): error CS0592: Attribute 'SpacetimeDB.Type' is not valid on this declaration type. It is only valid on 'class, struct, enum' declarations. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\claude-4-5-haiku\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\claude-4-5-haiku\\llm\\Lib.cs(24,10): error CS0592: Attribute 'SpacetimeDB.Type' is not valid on this declaration type. It is only valid on 'class, struct, enum' declarations. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\claude-4-5-haiku\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\claude-4-5-haiku\\llm\\Lib.cs(26,10): error CS0592: Attribute 'SpacetimeDB.Type' is not valid on this declaration type. It is only valid on 'class, struct, enum' declarations. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\claude-4-5-haiku\\llm\\StdbModule.csproj]\r\n",
- "phase": "build_or_publish"
+ "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
}
}
},
"vendor": "anthropic",
- "started_at": "2025-10-21T21:21:56.203255200Z",
- "finished_at": "2025-10-21T21:22:10.125883400Z"
+ "started_at": "2025-10-21T21:15:59.720329100Z",
+ "finished_at": "2025-10-21T21:17:36.496186600Z"
},
"t_007_crud": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
@@ -10832,27 +10584,13 @@
"work_dir_golden": "target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\golden",
"work_dir_llm": "target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\claude-4-5-haiku\\llm",
"scorer_details": {
- "crud_row_id1_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "args": [],
- "golden_db": "basics-t-007-crud-golden",
- "golden_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
- "llm_db": "basics-t-007-crud-claude-4-5-haiku-llm",
- "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
- "query": "SELECT Id, Name, Age, Active FROM users WHERE Id=1",
- "reducer": "Crud",
- "server": "local"
- }
- },
- "crud_row_id2_deleted": {
+ "crud_total_count_one": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 0,
- "expected": 0,
- "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=2"
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM users"
}
},
"schema_parity": {
@@ -10868,13 +10606,27 @@
"tables_equal": true
}
},
- "crud_total_count_one": {
+ "crud_row_id2_deleted": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM users"
+ "actual": 0,
+ "expected": 0,
+ "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=2"
+ }
+ },
+ "crud_row_id1_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "args": [],
+ "golden_db": "basics-t-007-crud-golden",
+ "golden_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
+ "llm_db": "basics-t-007-crud-claude-4-5-haiku-llm",
+ "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
+ "query": "SELECT Id, Name, Age, Active FROM users WHERE Id=1",
+ "reducer": "Crud",
+ "server": "local"
}
}
},
@@ -10882,68 +10634,62 @@
"started_at": "2025-10-21T21:16:00.507473700Z",
"finished_at": "2025-10-21T21:17:34.144777300Z"
},
- "t_010_connect": {
+ "t_008_index_lookup": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_010_connect",
+ "task": "t_008_index_lookup",
"lang": "csharp",
"golden_published": true,
"model_name": "Claude 4.5 Haiku",
- "total_tests": 1,
- "passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"events\")]\n public partial struct Event\n {\n [PrimaryKey]\n [AutoInc]\n public int Id;\n public string Kind;\n }\n\n [Reducer(ReducerKind.ClientConnected)]\n public static void ClientConnected(ReducerContext ctx)\n {\n ctx.Db.events.Insert(new Event\n {\n Id = 0,\n Kind = \"connected\"\n });\n }\n\n [Reducer(ReducerKind.ClientDisconnected)]\n public static void ClientDisconnected(ReducerContext ctx)\n {\n ctx.Db.events.Insert(new Event\n {\n Id = 0,\n Kind = \"disconnected\"\n });\n }\n}",
+ "total_tests": 3,
+ "passed_tests": 3,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"users\", Public = true)]\n public partial struct User\n {\n [PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [Table(Name = \"results\", Public = true)]\n public partial struct Result\n {\n [PrimaryKey]\n public int Id;\n public string Name;\n }\n\n [Reducer]\n public static void LookupUserName(ReducerContext ctx, int id)\n {\n var user = ctx.Db.users.Id.Find(id);\n if (user.HasValue)\n {\n ctx.Db.results.Insert(new Result\n {\n Id = user.Value.Id,\n Name = user.Value.Name\n });\n }\n }\n}",
"category": "basics",
"route_api_model": "claude-haiku-4-5",
- "golden_db": "basics-t-010-connect-golden",
- "llm_db": "basics-t-010-connect-claude-4-5-haiku-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_010_connect\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_010_connect\\csharp\\server\\claude-4-5-haiku\\llm",
+ "golden_db": "basics-t-008-index-lookup-golden",
+ "llm_db": "basics-t-008-index-lookup-claude-4-5-haiku-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\claude-4-5-haiku\\llm",
"scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-010-connect-golden",
- "llm_db": "basics-t-010-connect-claude-4-5-haiku-llm",
+ "golden_db": "basics-t-008-index-lookup-golden",
+ "llm_db": "basics-t-008-index-lookup-claude-4-5-haiku-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
"tables_diff": null,
"tables_equal": true
}
- }
- },
- "vendor": "anthropic",
- "started_at": "2025-10-21T21:16:02.859193900Z",
- "finished_at": "2025-10-21T21:18:05.091192Z"
- },
- "t_021_multi_column_index": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_021_multi_column_index",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "Claude 4.5 Haiku",
- "total_tests": 4,
- "passed_tests": 0,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"logs\", Public = true)]\n [Index(Name = \"by_user_day\", BTree = [nameof(UserId), nameof(Day)])]\n public partial struct Log\n {\n [PrimaryKey]\n public int Id;\n public int UserId;\n public int Day;\n public string Message;\n }\n\n [Reducer(ReducerKind.Init)]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.logs.Insert(new Log { Id = 1, UserId = 7, Day = 1, Message = \"a\" });\n ctx.Db.logs.Insert(new Log { Id = 2, UserId = 7, Day = 2, Message = \"b\" });\n ctx.Db.logs.Insert(new Log { Id = 3, UserId = 9, Day = 1, Message = \"c\" });\n }\n}",
- "category": "schema",
- "route_api_model": "claude-haiku-4-5",
- "golden_db": "schema-t-021-multi-column-index-golden",
- "llm_db": "schema-t-021-multi-column-index-claude-4-5-haiku-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\claude-4-5-haiku\\llm",
- "scorer_details": {
- "publish_error": {
- "pass": false,
- "partial": 0.0,
+ },
+ "index_lookup_projection_parity": {
+ "pass": true,
+ "partial": 1.0,
"notes": {
- "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\claude-4-5-haiku\\llm\\Lib.cs(7,6): error CS0104: 'Index' is an ambiguous reference between 'SpacetimeDB.Index' and 'System.Index' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\claude-4-5-haiku\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\claude-4-5-haiku\\llm\\Lib.cs(7,42): error CS9176: There is no target type for the collection expression. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\claude-4-5-haiku\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\claude-4-5-haiku\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'logs' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\claude-4-5-haiku\\llm\\StdbModule.csproj]\r\n",
- "phase": "build_or_publish"
+ "args": [
+ 1
+ ],
+ "golden_db": "basics-t-008-index-lookup-golden",
+ "golden_out": "Id | Name ----+--------- 1 | \"Alice\"",
+ "llm_db": "basics-t-008-index-lookup-claude-4-5-haiku-llm",
+ "llm_out": "Id | Name ----+--------- 1 | \"Alice\"",
+ "query": "SELECT Id, Name FROM results WHERE Id=1",
+ "reducer": "LookupUserName",
+ "server": "local"
+ }
+ },
+ "seed_user_row": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
}
}
},
"vendor": "anthropic",
- "started_at": "2025-10-21T21:22:00.471862400Z",
- "finished_at": "2025-10-21T21:22:09.531652200Z"
+ "started_at": "2025-10-21T21:16:01.281757Z",
+ "finished_at": "2025-10-21T21:17:34.692711Z"
},
"t_009_init": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
@@ -10970,13 +10716,13 @@
"sql": "SELECT COUNT(*) AS n FROM users WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true"
}
},
- "init_total_two": {
+ "init_seed_bob": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 2,
- "expected": 2,
- "sql": "SELECT COUNT(*) AS n FROM users"
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false"
}
},
"schema_parity": {
@@ -10992,13 +10738,13 @@
"tables_equal": true
}
},
- "init_seed_bob": {
+ "init_total_two": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false"
+ "actual": 2,
+ "expected": 2,
+ "sql": "SELECT COUNT(*) AS n FROM users"
}
}
},
@@ -11006,28 +10752,28 @@
"started_at": "2025-10-21T21:16:02.085109500Z",
"finished_at": "2025-10-21T21:18:03.336602500Z"
},
- "t_001_basic_tables": {
+ "t_010_connect": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_001_basic_tables",
+ "task": "t_010_connect",
"lang": "csharp",
"golden_published": true,
"model_name": "Claude 4.5 Haiku",
"total_tests": 1,
"passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"users\", Public = true)]\n public partial struct Users\n {\n [PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [Table(Name = \"products\", Public = true)]\n public partial struct Products\n {\n [PrimaryKey]\n public int Id;\n public string Title;\n public float Price;\n public bool InStock;\n }\n\n [Table(Name = \"notes\", Public = true)]\n public partial struct Notes\n {\n [PrimaryKey]\n public int Id;\n public string Body;\n public long Rating;\n public bool Pinned;\n }\n}",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"events\")]\n public partial struct Event\n {\n [PrimaryKey]\n [AutoInc]\n public int Id;\n public string Kind;\n }\n\n [Reducer(ReducerKind.ClientConnected)]\n public static void ClientConnected(ReducerContext ctx)\n {\n ctx.Db.events.Insert(new Event\n {\n Id = 0,\n Kind = \"connected\"\n });\n }\n\n [Reducer(ReducerKind.ClientDisconnected)]\n public static void ClientDisconnected(ReducerContext ctx)\n {\n ctx.Db.events.Insert(new Event\n {\n Id = 0,\n Kind = \"disconnected\"\n });\n }\n}",
"category": "basics",
"route_api_model": "claude-haiku-4-5",
- "golden_db": "basics-t-001-basic-tables-golden",
- "llm_db": "basics-t-001-basic-tables-claude-4-5-haiku-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\claude-4-5-haiku\\llm",
+ "golden_db": "basics-t-010-connect-golden",
+ "llm_db": "basics-t-010-connect-claude-4-5-haiku-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_010_connect\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_010_connect\\csharp\\server\\claude-4-5-haiku\\llm",
"scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-001-basic-tables-golden",
- "llm_db": "basics-t-001-basic-tables-claude-4-5-haiku-llm",
+ "golden_db": "basics-t-010-connect-golden",
+ "llm_db": "basics-t-010-connect-claude-4-5-haiku-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -11037,31 +10783,31 @@
}
},
"vendor": "anthropic",
- "started_at": "2025-10-21T21:15:55.834612100Z",
- "finished_at": "2025-10-21T21:17:34.859654500Z"
+ "started_at": "2025-10-21T21:16:02.859193900Z",
+ "finished_at": "2025-10-21T21:18:05.091192Z"
},
- "t_016_sum_type_columns": {
+ "t_011_helper_function": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_016_sum_type_columns",
+ "task": "t_011_helper_function",
"lang": "csharp",
"golden_published": true,
"model_name": "Claude 4.5 Haiku",
"total_tests": 3,
"passed_tests": 3,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : SpacetimeDB.TaggedEnum<(Circle Circle, Rectangle Rectangle)>\n {\n }\n\n [Table(Name = \"drawings\", Public = true)]\n public partial struct Drawing\n {\n [PrimaryKey]\n public int Id;\n public Shape A;\n public Shape B;\n }\n\n [Reducer]\n public static void Seed(ReducerContext ctx)\n {\n var drawing = new Drawing\n {\n Id = 1,\n A = new Shape.Circle(new Circle { Radius = 10 }),\n B = new Shape.Rectangle(new Rectangle { Width = 4, Height = 6 })\n };\n ctx.Db.drawings.Insert(drawing);\n }\n}",
- "category": "schema",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"results\")]\n public partial struct Result\n {\n [PrimaryKey]\n public int Id;\n public int Sum;\n }\n\n private static int Add(int a, int b)\n {\n return a + b;\n }\n\n [Reducer]\n public static void ComputeSum(ReducerContext ctx, int id, int a, int b)\n {\n int sum = Add(a, b);\n ctx.Db.results.Insert(new Result { Id = id, Sum = sum });\n }\n}",
+ "category": "basics",
"route_api_model": "claude-haiku-4-5",
- "golden_db": "schema-t-016-sum-type-columns-golden",
- "llm_db": "schema-t-016-sum-type-columns-claude-4-5-haiku-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\claude-4-5-haiku\\llm",
+ "golden_db": "basics-t-011-helper-function-golden",
+ "llm_db": "basics-t-011-helper-function-claude-4-5-haiku-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\claude-4-5-haiku\\llm",
"scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-016-sum-type-columns-golden",
- "llm_db": "schema-t-016-sum-type-columns-claude-4-5-haiku-llm",
+ "golden_db": "basics-t-011-helper-function-golden",
+ "llm_db": "basics-t-011-helper-function-claude-4-5-haiku-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -11069,73 +10815,60 @@
"tables_equal": true
}
},
- "sum_type_columns_row_parity": {
+ "helper_func_sum_abs": {
"pass": true,
"partial": 1.0,
"notes": {
- "args": [],
- "golden_db": "schema-t-016-sum-type-columns-golden",
- "golden_out": "Id | A | B ----+--------------------------+--------------------------------------- 1 | (Circle = (Radius = 10)) | (Rectangle = (Width = 4, Height = 6))",
- "llm_db": "schema-t-016-sum-type-columns-claude-4-5-haiku-llm",
- "llm_out": "Id | A | B ----+--------------------------+--------------------------------------- 1 | (Circle = (Radius = 10)) | (Rectangle = (Width = 4, Height = 6))",
- "query": "SELECT Id, A, B FROM drawings WHERE Id=1",
- "reducer": "Seed",
- "server": "local"
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1 AND Sum=5"
}
},
- "sum_type_columns_row_count": {
+ "helper_func_sum_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM drawings WHERE Id=1"
+ "args": [
+ 1,
+ 2,
+ 3
+ ],
+ "golden_db": "basics-t-011-helper-function-golden",
+ "golden_out": "Id | Sum ----+----- 1 | 5",
+ "llm_db": "basics-t-011-helper-function-claude-4-5-haiku-llm",
+ "llm_out": "Id | Sum ----+----- 1 | 5",
+ "query": "SELECT Id, Sum FROM results WHERE Id=1",
+ "reducer": "ComputeSum",
+ "server": "local"
}
}
},
"vendor": "anthropic",
- "started_at": "2025-10-21T21:21:56.925366600Z",
- "finished_at": "2025-10-21T21:23:14.418688200Z"
+ "started_at": "2025-10-21T21:16:03.603944800Z",
+ "finished_at": "2025-10-21T21:17:37.429004300Z"
},
- "t_019_many_to_many": {
+ "t_012_spacetime_product_type": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_019_many_to_many",
+ "task": "t_012_spacetime_product_type",
"lang": "csharp",
"golden_published": true,
"model_name": "Claude 4.5 Haiku",
- "total_tests": 5,
- "passed_tests": 4,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"users\", Public = true)]\n public partial struct User\n {\n [PrimaryKey]\n public int UserId;\n public string Name;\n }\n\n [Table(Name = \"groups\", Public = true)]\n public partial struct Group\n {\n [PrimaryKey]\n public int GroupId;\n public string Title;\n }\n\n [Table(Name = \"memberships\", Public = true)]\n [SpacetimeDB.Index.BTree(Name = \"by_user\", Columns = [nameof(UserId)])]\n [SpacetimeDB.Index.BTree(Name = \"by_group\", Columns = [nameof(GroupId)])]\n public partial struct Membership\n {\n [PrimaryKey]\n public int Id;\n public int UserId;\n public int GroupId;\n }\n\n [Reducer(ReducerKind.Init)]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.users.Insert(new User { UserId = 1, Name = \"Alice\" });\n ctx.Db.users.Insert(new User { UserId = 2, Name = \"Bob\" });\n\n ctx.Db.groups.Insert(new Group { GroupId = 10, Title = \"Admin\" });\n ctx.Db.groups.Insert(new Group { GroupId = 20, Title = \"Dev\" });\n\n ctx.Db.memberships.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\n ctx.Db.memberships.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\n ctx.Db.memberships.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\n }\n}",
+ "total_tests": 3,
+ "passed_tests": 3,
+ "llm_output": "using SpacetimeDB;\n\n[SpacetimeDB.Type]\npublic partial struct Score\n{\n public int Left;\n public int Right;\n}\n\n[Table(Name = \"results\", Public = true)]\npublic partial struct Result\n{\n [PrimaryKey]\n public int Id;\n public Score Value;\n}\n\npublic static partial class Module\n{\n [Reducer]\n public static void SetScore(ReducerContext ctx, int id, int left, int right)\n {\n ctx.Db.results.Insert(new Result\n {\n Id = id,\n Value = new Score\n {\n Left = left,\n Right = right\n }\n });\n }\n}",
"category": "schema",
"route_api_model": "claude-haiku-4-5",
- "golden_db": "schema-t-019-many-to-many-golden",
- "llm_db": "schema-t-019-many-to-many-claude-4-5-haiku-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\claude-4-5-haiku\\llm",
+ "golden_db": "schema-t-012-spacetime-product-type-golden",
+ "llm_db": "schema-t-012-spacetime-product-type-claude-4-5-haiku-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\claude-4-5-haiku\\llm",
"scorer_details": {
- "m2m_has_1_20": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM memberships WHERE UserId=1 AND GroupId=20"
- }
- },
- "m2m_has_1_10": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: can't directly call special Init lifecycle reducer\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:3000/v1/database/c2000e5803df888c574fe79ecd2d746d1220d5e211bddcea7f2032877b524a9a/call/Seed)\n",
- "phase": "call_reducer"
- }
- },
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-019-many-to-many-golden",
- "llm_db": "schema-t-019-many-to-many-claude-4-5-haiku-llm",
+ "golden_db": "schema-t-012-spacetime-product-type-golden",
+ "llm_db": "schema-t-012-spacetime-product-type-claude-4-5-haiku-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -11143,51 +10876,60 @@
"tables_equal": true
}
},
- "memberships_three_rows": {
+ "product_type_row_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 3,
- "expected": 3,
- "sql": "SELECT COUNT(*) AS n FROM memberships"
+ "args": [
+ 1,
+ 2,
+ 3
+ ],
+ "golden_db": "schema-t-012-spacetime-product-type-golden",
+ "golden_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)",
+ "llm_db": "schema-t-012-spacetime-product-type-claude-4-5-haiku-llm",
+ "llm_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)",
+ "query": "SELECT Id, Value FROM results WHERE Id=1",
+ "reducer": "SetScore",
+ "server": "local"
}
},
- "m2m_has_2_20": {
+ "product_type_row_count": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM memberships WHERE UserId=2 AND GroupId=20"
+ "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1"
}
}
},
"vendor": "anthropic",
- "started_at": "2025-10-21T21:21:59.056624400Z",
- "finished_at": "2025-10-21T21:23:13.445114100Z"
+ "started_at": "2025-10-21T21:21:53.933934800Z",
+ "finished_at": "2025-10-21T21:23:15.187319300Z"
},
- "t_006_delete": {
+ "t_013_spacetime_sum_type": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_006_delete",
+ "task": "t_013_spacetime_sum_type",
"lang": "csharp",
"golden_published": true,
"model_name": "Claude 4.5 Haiku",
"total_tests": 3,
"passed_tests": 3,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"users\", Public = true)]\n public partial struct User\n {\n [PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [Reducer]\n public static void DeleteUser(ReducerContext ctx, int id)\n {\n var user = ctx.Db.users.Id.Find(id);\n if (user.HasValue)\n {\n ctx.Db.users.Id.Delete(user.Value.Id);\n }\n }\n}",
- "category": "basics",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial record Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial record Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : SpacetimeDB.TaggedEnum<(Circle Circle, Rectangle Rectangle)>\n {\n }\n\n [Table(Name = \"results\")]\n public partial struct Result\n {\n [PrimaryKey]\n public int Id;\n public Shape Value;\n }\n\n [Reducer]\n public static void SetCircle(ReducerContext ctx, int id, int radius)\n {\n var circle = new Circle { Radius = radius };\n var shape = new Shape.Circle(circle);\n var result = new Result { Id = id, Value = shape };\n ctx.Db.results.Insert(result);\n }\n}",
+ "category": "schema",
"route_api_model": "claude-haiku-4-5",
- "golden_db": "basics-t-006-delete-golden",
- "llm_db": "basics-t-006-delete-claude-4-5-haiku-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\claude-4-5-haiku\\llm",
+ "golden_db": "schema-t-013-spacetime-sum-type-golden",
+ "llm_db": "schema-t-013-spacetime-sum-type-claude-4-5-haiku-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\claude-4-5-haiku\\llm",
"scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-006-delete-golden",
- "llm_db": "basics-t-006-delete-claude-4-5-haiku-llm",
+ "golden_db": "schema-t-013-spacetime-sum-type-golden",
+ "llm_db": "schema-t-013-spacetime-sum-type-claude-4-5-haiku-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -11195,49 +10937,59 @@
"tables_equal": true
}
},
- "delete_user_count_zero": {
+ "sum_type_row_count": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 0,
- "expected": 0,
- "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=1"
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1"
}
},
- "seed_users_row": {
+ "sum_type_row_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
+ "args": [
+ 1,
+ 10
+ ],
+ "golden_db": "schema-t-013-spacetime-sum-type-golden",
+ "golden_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))",
+ "llm_db": "schema-t-013-spacetime-sum-type-claude-4-5-haiku-llm",
+ "llm_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))",
+ "query": "SELECT Id, Value FROM results WHERE Id=1",
+ "reducer": "SetCircle",
+ "server": "local"
}
}
},
"vendor": "anthropic",
- "started_at": "2025-10-21T21:15:59.720329100Z",
- "finished_at": "2025-10-21T21:17:36.496186600Z"
+ "started_at": "2025-10-21T21:21:54.694207Z",
+ "finished_at": "2025-10-21T21:23:14.796865500Z"
},
- "t_005_update": {
+ "t_014_elementary_columns": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_005_update",
+ "task": "t_014_elementary_columns",
"lang": "csharp",
"golden_published": true,
"model_name": "Claude 4.5 Haiku",
"total_tests": 3,
"passed_tests": 3,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"users\", Public = true)]\n public partial struct User\n {\n [PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [Reducer]\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n var user = ctx.Db.users.Id.Find(id);\n if (user.HasValue)\n {\n ctx.Db.users.Id.Update(new User\n {\n Id = id,\n Name = name,\n Age = age,\n Active = active\n });\n }\n }\n}",
- "category": "basics",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"primitives\")]\n public partial struct Primitive\n {\n [PrimaryKey]\n public int Id;\n public int Count;\n public long Total;\n public float Price;\n public double Ratio;\n public bool Active;\n public string Name;\n }\n\n [Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.primitives.Insert(new Primitive\n {\n Id = 1,\n Count = 2,\n Total = 3000000000,\n Price = 1.5f,\n Ratio = 2.25,\n Active = true,\n Name = \"Alice\"\n });\n }\n}",
+ "category": "schema",
"route_api_model": "claude-haiku-4-5",
- "golden_db": "basics-t-005-update-golden",
- "llm_db": "basics-t-005-update-claude-4-5-haiku-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_005_update\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_005_update\\csharp\\server\\claude-4-5-haiku\\llm",
+ "golden_db": "schema-t-014-elementary-columns-golden",
+ "llm_db": "schema-t-014-elementary-columns-claude-4-5-haiku-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_014_elementary_columns\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_014_elementary_columns\\csharp\\server\\claude-4-5-haiku\\llm",
"scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-005-update-golden",
- "llm_db": "basics-t-005-update-claude-4-5-haiku-llm",
+ "golden_db": "schema-t-014-elementary-columns-golden",
+ "llm_db": "schema-t-014-elementary-columns-claude-4-5-haiku-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -11245,135 +10997,94 @@
"tables_equal": true
}
},
- "seed_users_row": {
+ "elementary_columns_row_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
+ "args": [],
+ "golden_db": "schema-t-014-elementary-columns-golden",
+ "golden_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"",
+ "llm_db": "schema-t-014-elementary-columns-claude-4-5-haiku-llm",
+ "llm_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"",
+ "query": "SELECT Id, Count, Total, Price, Ratio, Active, Name FROM primitives WHERE Id=1",
+ "reducer": "Seed",
+ "server": "local"
}
},
- "data_parity_update_user": {
+ "elementary_columns_row_count": {
"pass": true,
"partial": 1.0,
"notes": {
- "args": [
- 1,
- "Alice2",
- 31,
- false
- ],
- "golden_db": "basics-t-005-update-golden",
- "golden_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
- "llm_db": "basics-t-005-update-claude-4-5-haiku-llm",
- "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
- "query": "SELECT Id, Name, Age, Active FROM users WHERE Id=1",
- "reducer": "UpdateUser",
- "server": "local"
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM primitives WHERE Id=1"
}
}
},
"vendor": "anthropic",
- "started_at": "2025-10-21T21:15:58.946859500Z",
- "finished_at": "2025-10-21T21:17:35.574459100Z"
+ "started_at": "2025-10-21T21:21:55.454990Z",
+ "finished_at": "2025-10-21T21:23:13.070138300Z"
},
- "t_012_spacetime_product_type": {
+ "t_015_product_type_columns": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_012_spacetime_product_type",
+ "task": "t_015_product_type_columns",
"lang": "csharp",
"golden_published": true,
"model_name": "Claude 4.5 Haiku",
"total_tests": 3,
- "passed_tests": 3,
- "llm_output": "using SpacetimeDB;\n\n[SpacetimeDB.Type]\npublic partial struct Score\n{\n public int Left;\n public int Right;\n}\n\n[Table(Name = \"results\", Public = true)]\npublic partial struct Result\n{\n [PrimaryKey]\n public int Id;\n public Score Value;\n}\n\npublic static partial class Module\n{\n [Reducer]\n public static void SetScore(ReducerContext ctx, int id, int left, int right)\n {\n ctx.Db.results.Insert(new Result\n {\n Id = id,\n Value = new Score\n {\n Left = left,\n Right = right\n }\n });\n }\n}",
+ "passed_tests": 0,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Address\n {\n [SpacetimeDB.Type]\n public string Street;\n [SpacetimeDB.Type]\n public int Zip;\n\n public Address(string street, int zip)\n {\n Street = street;\n Zip = zip;\n }\n }\n\n [SpacetimeDB.Type]\n public partial struct Position\n {\n [SpacetimeDB.Type]\n public int X;\n [SpacetimeDB.Type]\n public int Y;\n\n public Position(int x, int y)\n {\n X = x;\n Y = y;\n }\n }\n\n [Table(Name = \"profiles\", Public = true)]\n public partial struct Profile\n {\n [PrimaryKey]\n public int Id;\n public Address Home;\n public Address Work;\n public Position Pos;\n }\n\n [Reducer]\n public static void Seed(ReducerContext ctx)\n {\n var profile = new Profile\n {\n Id = 1,\n Home = new Address(Street: \"1 Main\", Zip: 11111),\n Work = new Address(Street: \"2 Broad\", Zip: 22222),\n Pos = new Position(X: 7, Y: 9)\n };\n\n ctx.Db.profiles.Insert(profile);\n }\n}",
"category": "schema",
"route_api_model": "claude-haiku-4-5",
- "golden_db": "schema-t-012-spacetime-product-type-golden",
- "llm_db": "schema-t-012-spacetime-product-type-claude-4-5-haiku-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\claude-4-5-haiku\\llm",
+ "golden_db": "schema-t-015-product-type-columns-golden",
+ "llm_db": "schema-t-015-product-type-columns-claude-4-5-haiku-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\claude-4-5-haiku\\llm",
"scorer_details": {
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "golden_db": "schema-t-012-spacetime-product-type-golden",
- "llm_db": "schema-t-012-spacetime-product-type-claude-4-5-haiku-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
- }
- },
- "product_type_row_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "args": [
- 1,
- 2,
- 3
- ],
- "golden_db": "schema-t-012-spacetime-product-type-golden",
- "golden_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)",
- "llm_db": "schema-t-012-spacetime-product-type-claude-4-5-haiku-llm",
- "llm_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)",
- "query": "SELECT Id, Value FROM results WHERE Id=1",
- "reducer": "SetScore",
- "server": "local"
- }
- },
- "product_type_row_count": {
- "pass": true,
- "partial": 1.0,
+ "publish_error": {
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1"
+ "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\claude-4-5-haiku\\llm\\Lib.cs(10,23): error STDBINT0001: An internal error occurred during codegen: Unable to cast object of type 'Microsoft.CodeAnalysis.CSharp.Syntax.VariableDeclaratorSyntax' to type 'Microsoft.CodeAnalysis.CSharp.Syntax.TypeDeclarationSyntax'. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\claude-4-5-haiku\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\claude-4-5-haiku\\llm\\Lib.cs(12,20): error STDBINT0001: An internal error occurred during codegen: Unable to cast object of type 'Microsoft.CodeAnalysis.CSharp.Syntax.VariableDeclaratorSyntax' to type 'Microsoft.CodeAnalysis.CSharp.Syntax.TypeDeclarationSyntax'. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\claude-4-5-haiku\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\claude-4-5-haiku\\llm\\Lib.cs(25,20): error STDBINT0001: An internal error occurred during codegen: Unable to cast object of type 'Microsoft.CodeAnalysis.CSharp.Syntax.VariableDeclaratorSyntax' to type 'Microsoft.CodeAnalysis.CSharp.Syntax.TypeDeclarationSyntax'. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\claude-4-5-haiku\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\claude-4-5-haiku\\llm\\Lib.cs(27,20): error STDBINT0001: An internal error occurred during codegen: Unable to cast object of type 'Microsoft.CodeAnalysis.CSharp.Syntax.VariableDeclaratorSyntax' to type 'Microsoft.CodeAnalysis.CSharp.Syntax.TypeDeclarationSyntax'. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\claude-4-5-haiku\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\claude-4-5-haiku\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'profiles' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\claude-4-5-haiku\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\claude-4-5-haiku\\llm\\Lib.cs(9,10): error CS0592: Attribute 'SpacetimeDB.Type' is not valid on this declaration type. It is only valid on 'class, struct, enum' declarations. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\claude-4-5-haiku\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\claude-4-5-haiku\\llm\\Lib.cs(11,10): error CS0592: Attribute 'SpacetimeDB.Type' is not valid on this declaration type. It is only valid on 'class, struct, enum' declarations. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\claude-4-5-haiku\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\claude-4-5-haiku\\llm\\Lib.cs(24,10): error CS0592: Attribute 'SpacetimeDB.Type' is not valid on this declaration type. It is only valid on 'class, struct, enum' declarations. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\claude-4-5-haiku\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\claude-4-5-haiku\\llm\\Lib.cs(26,10): error CS0592: Attribute 'SpacetimeDB.Type' is not valid on this declaration type. It is only valid on 'class, struct, enum' declarations. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\claude-4-5-haiku\\llm\\StdbModule.csproj]\r\n",
+ "phase": "build_or_publish"
}
}
},
"vendor": "anthropic",
- "started_at": "2025-10-21T21:21:53.933934800Z",
- "finished_at": "2025-10-21T21:23:15.187319300Z"
- }
- }
- },
- {
- "name": "DeepSeek R1",
- "route_api_model": "deepseek-reasoner",
- "tasks": {
- "t_012_spacetime_product_type": {
+ "started_at": "2025-10-21T21:21:56.203255200Z",
+ "finished_at": "2025-10-21T21:22:10.125883400Z"
+ },
+ "t_016_sum_type_columns": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_012_spacetime_product_type",
+ "task": "t_016_sum_type_columns",
"lang": "csharp",
"golden_published": true,
- "model_name": "DeepSeek R1",
+ "model_name": "Claude 4.5 Haiku",
"total_tests": 3,
"passed_tests": 3,
- "llm_output": "using SpacetimeDB;\n\n[SpacetimeDB.Type]\npublic partial struct Score\n{\n public int Left;\n public int Right;\n}\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"results\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Score Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetScore(ReducerContext ctx, int id, int left, int right)\n {\n ctx.Db.results.Insert(new Result\n {\n Id = id,\n Value = new Score { Left = left, Right = right }\n });\n }\n}",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : SpacetimeDB.TaggedEnum<(Circle Circle, Rectangle Rectangle)>\n {\n }\n\n [Table(Name = \"drawings\", Public = true)]\n public partial struct Drawing\n {\n [PrimaryKey]\n public int Id;\n public Shape A;\n public Shape B;\n }\n\n [Reducer]\n public static void Seed(ReducerContext ctx)\n {\n var drawing = new Drawing\n {\n Id = 1,\n A = new Shape.Circle(new Circle { Radius = 10 }),\n B = new Shape.Rectangle(new Rectangle { Width = 4, Height = 6 })\n };\n ctx.Db.drawings.Insert(drawing);\n }\n}",
"category": "schema",
- "route_api_model": "deepseek-reasoner",
- "golden_db": "schema-t-012-spacetime-product-type-golden",
- "llm_db": "schema-t-012-spacetime-product-type-deepseek-r1-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\deepseek-r1\\llm",
+ "route_api_model": "claude-haiku-4-5",
+ "golden_db": "schema-t-016-sum-type-columns-golden",
+ "llm_db": "schema-t-016-sum-type-columns-claude-4-5-haiku-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\claude-4-5-haiku\\llm",
"scorer_details": {
- "product_type_row_count": {
+ "sum_type_columns_row_count": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1"
+ "sql": "SELECT COUNT(*) AS n FROM drawings WHERE Id=1"
}
},
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-012-spacetime-product-type-golden",
- "llm_db": "schema-t-012-spacetime-product-type-deepseek-r1-llm",
+ "golden_db": "schema-t-016-sum-type-columns-golden",
+ "llm_db": "schema-t-016-sum-type-columns-claude-4-5-haiku-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -11381,144 +11092,257 @@
"tables_equal": true
}
},
- "product_type_row_parity": {
+ "sum_type_columns_row_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "args": [
- 1,
- 2,
- 3
- ],
- "golden_db": "schema-t-012-spacetime-product-type-golden",
- "golden_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)",
- "llm_db": "schema-t-012-spacetime-product-type-deepseek-r1-llm",
- "llm_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)",
- "query": "SELECT Id, Value FROM results WHERE Id=1",
- "reducer": "SetScore",
+ "args": [],
+ "golden_db": "schema-t-016-sum-type-columns-golden",
+ "golden_out": "Id | A | B ----+--------------------------+--------------------------------------- 1 | (Circle = (Radius = 10)) | (Rectangle = (Width = 4, Height = 6))",
+ "llm_db": "schema-t-016-sum-type-columns-claude-4-5-haiku-llm",
+ "llm_out": "Id | A | B ----+--------------------------+--------------------------------------- 1 | (Circle = (Radius = 10)) | (Rectangle = (Width = 4, Height = 6))",
+ "query": "SELECT Id, A, B FROM drawings WHERE Id=1",
+ "reducer": "Seed",
"server": "local"
}
}
},
- "vendor": "deepseek",
- "started_at": "2025-10-21T21:33:42.685896400Z",
- "finished_at": "2025-10-21T21:35:28.769962700Z"
+ "vendor": "anthropic",
+ "started_at": "2025-10-21T21:21:56.925366600Z",
+ "finished_at": "2025-10-21T21:23:14.418688200Z"
},
- "t_013_spacetime_sum_type": {
+ "t_017_scheduled_columns": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_013_spacetime_sum_type",
+ "task": "t_017_scheduled_columns",
"lang": "csharp",
"golden_published": true,
- "model_name": "DeepSeek R1",
- "total_tests": 3,
+ "model_name": "Claude 4.5 Haiku",
+ "total_tests": 2,
"passed_tests": 0,
- "llm_output": "using SpacetimeDB;\n\n[SpacetimeDB.Type]\npublic partial struct Circle\n{\n public int Radius;\n}\n\n[SpacetimeDB.Type]\npublic partial struct Rectangle\n{\n public int Width;\n public int Height;\n}\n\n[SpacetimeDB.Type]\npublic partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"results\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetCircle(ReducerContext ctx, int id, int radius)\n {\n ctx.Db.results.Insert(new Result\n {\n Id = id,\n Value = new Shape.Circle(new Circle { Radius = radius })\n });\n }\n}",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"tick_timer\", Scheduled = nameof(Tick), ScheduledAt = nameof(scheduled_at))]\n public partial struct TickTimer\n {\n [PrimaryKey, AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.tick_timer.Insert(new TickTimer\n {\n ScheduledId = 0,\n ScheduledAt = new ScheduleAt.Interval(new TimeDuration { Microseconds = 50_000 })\n });\n }\n\n [Reducer]\n public static void Tick(ReducerContext ctx, TickTimer timer)\n {\n }\n}",
"category": "schema",
- "route_api_model": "deepseek-reasoner",
- "golden_db": "schema-t-013-spacetime-sum-type-golden",
- "llm_db": "schema-t-013-spacetime-sum-type-deepseek-r1-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\deepseek-r1\\llm",
+ "route_api_model": "claude-haiku-4-5",
+ "golden_db": "schema-t-017-scheduled-columns-golden",
+ "llm_db": "schema-t-017-scheduled-columns-claude-4-5-haiku-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_017_scheduled_columns\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_017_scheduled_columns\\csharp\\server\\claude-4-5-haiku\\llm",
"scorer_details": {
"publish_error": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\deepseek-r1\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'results' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\deepseek-r1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\deepseek-r1\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.BSATN.Codegen\\SpacetimeDB.Codegen.Type\\Shape.cs(6,26): error CS8910: The primary constructor conflicts with the synthesized copy constructor. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\deepseek-r1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\deepseek-r1\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.BSATN.Codegen\\SpacetimeDB.Codegen.Type\\Shape.cs(12,26): error CS8910: The primary constructor conflicts with the synthesized copy constructor. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\deepseek-r1\\llm\\StdbModule.csproj]\r\n",
+ "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_017_scheduled_columns\\csharp\\server\\claude-4-5-haiku\\llm\\Lib.cs(6,6): error STDB0014: Could not find the specified column scheduled_at in TickTimer. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_017_scheduled_columns\\csharp\\server\\claude-4-5-haiku\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_017_scheduled_columns\\csharp\\server\\claude-4-5-haiku\\llm\\Lib.cs(6,80): error CS0103: The name 'scheduled_at' does not exist in the current context [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_017_scheduled_columns\\csharp\\server\\claude-4-5-haiku\\llm\\StdbModule.csproj]\r\n",
"phase": "build_or_publish"
}
}
},
- "vendor": "deepseek",
- "started_at": "2025-10-21T21:33:43.223808Z",
- "finished_at": "2025-10-21T21:34:32.511591800Z"
+ "vendor": "anthropic",
+ "started_at": "2025-10-21T21:21:57.636347400Z",
+ "finished_at": "2025-10-21T21:22:10.168663100Z"
},
- "t_015_product_type_columns": {
+ "t_018_constraints": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_015_product_type_columns",
+ "task": "t_018_constraints",
"lang": "csharp",
"golden_published": true,
- "model_name": "DeepSeek R1",
+ "model_name": "Claude 4.5 Haiku",
"total_tests": 3,
- "passed_tests": 3,
- "llm_output": "using SpacetimeDB;\n\n[SpacetimeDB.Type]\npublic partial struct Address\n{\n public string Street;\n public int Zip;\n}\n\n[SpacetimeDB.Type]\npublic partial struct Position\n{\n public int X;\n public int Y;\n}\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"profiles\")]\n public partial struct Profile\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Address Home;\n public Address Work;\n public Position Pos;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.profiles.Insert(new Profile\n {\n Id = 1,\n Home = new Address { Street = \"1 Main\", Zip = 11111 },\n Work = new Address { Street = \"2 Broad\", Zip = 22222 },\n Pos = new Position { X = 7, Y = 9 }\n });\n }\n}",
+ "passed_tests": 0,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"accounts\", Public = true)]\n [Index(Name = \"by_name\", BTree = [nameof(Name)])]\n public partial struct Account\n {\n [PrimaryKey]\n public int Id;\n\n [Unique]\n public string Email;\n\n public string Name;\n }\n\n [Reducer(ReducerKind.Init)]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.accounts.Insert(new Account\n {\n Id = 1,\n Email = \"a@example.com\",\n Name = \"Alice\"\n });\n\n ctx.Db.accounts.Insert(new Account\n {\n Id = 2,\n Email = \"b@example.com\",\n Name = \"Bob\"\n });\n }\n}",
"category": "schema",
- "route_api_model": "deepseek-reasoner",
- "golden_db": "schema-t-015-product-type-columns-golden",
- "llm_db": "schema-t-015-product-type-columns-deepseek-r1-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\deepseek-r1\\llm",
+ "route_api_model": "claude-haiku-4-5",
+ "golden_db": "schema-t-018-constraints-golden",
+ "llm_db": "schema-t-018-constraints-claude-4-5-haiku-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\claude-4-5-haiku\\llm",
"scorer_details": {
- "product_type_columns_row_count": {
+ "publish_error": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\claude-4-5-haiku\\llm\\Lib.cs(7,6): error CS0104: 'Index' is an ambiguous reference between 'SpacetimeDB.Index' and 'System.Index' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\claude-4-5-haiku\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\claude-4-5-haiku\\llm\\Lib.cs(7,38): error CS9176: There is no target type for the collection expression. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\claude-4-5-haiku\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\claude-4-5-haiku\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'accounts' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\claude-4-5-haiku\\llm\\StdbModule.csproj]\r\n",
+ "phase": "build_or_publish"
+ }
+ }
+ },
+ "vendor": "anthropic",
+ "started_at": "2025-10-21T21:21:58.345339Z",
+ "finished_at": "2025-10-21T21:22:11.208616800Z"
+ },
+ "t_019_many_to_many": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_019_many_to_many",
+ "lang": "csharp",
+ "golden_published": true,
+ "model_name": "Claude 4.5 Haiku",
+ "total_tests": 5,
+ "passed_tests": 4,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"users\", Public = true)]\n public partial struct User\n {\n [PrimaryKey]\n public int UserId;\n public string Name;\n }\n\n [Table(Name = \"groups\", Public = true)]\n public partial struct Group\n {\n [PrimaryKey]\n public int GroupId;\n public string Title;\n }\n\n [Table(Name = \"memberships\", Public = true)]\n [SpacetimeDB.Index.BTree(Name = \"by_user\", Columns = [nameof(UserId)])]\n [SpacetimeDB.Index.BTree(Name = \"by_group\", Columns = [nameof(GroupId)])]\n public partial struct Membership\n {\n [PrimaryKey]\n public int Id;\n public int UserId;\n public int GroupId;\n }\n\n [Reducer(ReducerKind.Init)]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.users.Insert(new User { UserId = 1, Name = \"Alice\" });\n ctx.Db.users.Insert(new User { UserId = 2, Name = \"Bob\" });\n\n ctx.Db.groups.Insert(new Group { GroupId = 10, Title = \"Admin\" });\n ctx.Db.groups.Insert(new Group { GroupId = 20, Title = \"Dev\" });\n\n ctx.Db.memberships.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\n ctx.Db.memberships.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\n ctx.Db.memberships.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\n }\n}",
+ "category": "schema",
+ "route_api_model": "claude-haiku-4-5",
+ "golden_db": "schema-t-019-many-to-many-golden",
+ "llm_db": "schema-t-019-many-to-many-claude-4-5-haiku-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\claude-4-5-haiku\\llm",
+ "scorer_details": {
+ "m2m_has_2_20": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM profiles WHERE Id=1"
+ "sql": "SELECT COUNT(*) AS n FROM memberships WHERE UserId=2 AND GroupId=20"
}
},
- "product_type_columns_row_parity": {
+ "m2m_has_1_10": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: can't directly call special Init lifecycle reducer\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:3000/v1/database/c2000e5803df888c574fe79ecd2d746d1220d5e211bddcea7f2032877b524a9a/call/Seed)\n",
+ "phase": "call_reducer"
+ }
+ },
+ "schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "args": [],
- "golden_db": "schema-t-015-product-type-columns-golden",
- "golden_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)",
- "llm_db": "schema-t-015-product-type-columns-deepseek-r1-llm",
- "llm_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)",
- "query": "SELECT Id, Home, Work, Pos FROM profiles WHERE Id=1",
- "reducer": "Seed",
- "server": "local"
+ "golden_db": "schema-t-019-many-to-many-golden",
+ "llm_db": "schema-t-019-many-to-many-claude-4-5-haiku-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
+ }
+ },
+ "memberships_three_rows": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 3,
+ "expected": 3,
+ "sql": "SELECT COUNT(*) AS n FROM memberships"
}
},
+ "m2m_has_1_20": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM memberships WHERE UserId=1 AND GroupId=20"
+ }
+ }
+ },
+ "vendor": "anthropic",
+ "started_at": "2025-10-21T21:21:59.056624400Z",
+ "finished_at": "2025-10-21T21:23:13.445114100Z"
+ },
+ "t_020_ecs": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_020_ecs",
+ "lang": "csharp",
+ "golden_published": true,
+ "model_name": "Claude 4.5 Haiku",
+ "total_tests": 5,
+ "passed_tests": 5,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"entities\", Public = true)]\n public partial struct Entity\n {\n [PrimaryKey]\n public int Id;\n }\n\n [Table(Name = \"positions\", Public = true)]\n public partial struct Position\n {\n [PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [Table(Name = \"velocities\", Public = true)]\n public partial struct Velocity\n {\n [PrimaryKey]\n public int EntityId;\n public int VX;\n public int VY;\n }\n\n [Table(Name = \"next_positions\", Public = true)]\n public partial struct NextPosition\n {\n [PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.entities.Insert(new Entity { Id = 1 });\n ctx.Db.positions.Insert(new Position { EntityId = 1, X = 0, Y = 0 });\n ctx.Db.velocities.Insert(new Velocity { EntityId = 1, VX = 1, VY = 0 });\n\n ctx.Db.entities.Insert(new Entity { Id = 2 });\n ctx.Db.positions.Insert(new Position { EntityId = 2, X = 10, Y = 0 });\n ctx.Db.velocities.Insert(new Velocity { EntityId = 2, VX = -2, VY = 3 });\n }\n\n [Reducer]\n public static void Step(ReducerContext ctx)\n {\n foreach (var position in ctx.Db.positions.Iter())\n {\n var velocity = ctx.Db.velocities.EntityId.Find(position.EntityId);\n if (velocity.HasValue)\n {\n var nextX = position.X + velocity.Value.VX;\n var nextY = position.Y + velocity.Value.VY;\n\n var existingNextPos = ctx.Db.next_positions.EntityId.Find(position.EntityId);\n if (existingNextPos.HasValue)\n {\n ctx.Db.next_positions.EntityId.Update(new NextPosition\n {\n EntityId = position.EntityId,\n X = nextX,\n Y = nextY\n });\n }\n else\n {\n ctx.Db.next_positions.Insert(new NextPosition\n {\n EntityId = position.EntityId,\n X = nextX,\n Y = nextY\n });\n }\n }\n }\n }\n}",
+ "category": "schema",
+ "route_api_model": "claude-haiku-4-5",
+ "golden_db": "schema-t-020-ecs-golden",
+ "llm_db": "schema-t-020-ecs-claude-4-5-haiku-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\claude-4-5-haiku\\llm",
+ "scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-015-product-type-columns-golden",
- "llm_db": "schema-t-015-product-type-columns-deepseek-r1-llm",
+ "golden_db": "schema-t-020-ecs-golden",
+ "llm_db": "schema-t-020-ecs-claude-4-5-haiku-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
"tables_diff": null,
"tables_equal": true
}
+ },
+ "ecs_seed_positions_count": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 2,
+ "expected": 2,
+ "sql": "SELECT COUNT(*) AS n FROM positions"
+ }
+ },
+ "ecs_next_pos_entity2": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM next_positions WHERE EntityId=2 AND X=8 AND Y=3"
+ }
+ },
+ "ecs_step_next_positions_count": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 2,
+ "expected": 2,
+ "sql": "SELECT COUNT(*) AS n FROM next_positions"
+ }
+ },
+ "ecs_next_pos_entity1": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM next_positions WHERE EntityId=1 AND X=1 AND Y=0"
+ }
}
},
- "vendor": "deepseek",
- "started_at": "2025-10-21T21:33:44.211931300Z",
- "finished_at": "2025-10-21T21:35:27.125850100Z"
+ "vendor": "anthropic",
+ "started_at": "2025-10-21T21:21:59.768557Z",
+ "finished_at": "2025-10-21T21:23:14.012132300Z"
},
"t_021_multi_column_index": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
"task": "t_021_multi_column_index",
"lang": "csharp",
"golden_published": true,
- "model_name": "DeepSeek R1",
+ "model_name": "Claude 4.5 Haiku",
"total_tests": 4,
"passed_tests": 0,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"logs\")]\n public partial struct Log\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int Day;\n public string Message;\n }\n\n [SpacetimeDB.Index.BTree(Name = \"by_user_day\", Columns = [nameof(UserId), nameof(Day)])]\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.logs.Insert(new Log { Id = 1, UserId = 7, Day = 1, Message = \"a\" });\n ctx.Db.logs.Insert(new Log { Id = 2, UserId = 7, Day = 2, Message = \"b\" });\n ctx.Db.logs.Insert(new Log { Id = 3, UserId = 9, Day = 1, Message = \"c\" });\n }\n}",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"logs\", Public = true)]\n [Index(Name = \"by_user_day\", BTree = [nameof(UserId), nameof(Day)])]\n public partial struct Log\n {\n [PrimaryKey]\n public int Id;\n public int UserId;\n public int Day;\n public string Message;\n }\n\n [Reducer(ReducerKind.Init)]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.logs.Insert(new Log { Id = 1, UserId = 7, Day = 1, Message = \"a\" });\n ctx.Db.logs.Insert(new Log { Id = 2, UserId = 7, Day = 2, Message = \"b\" });\n ctx.Db.logs.Insert(new Log { Id = 3, UserId = 9, Day = 1, Message = \"c\" });\n }\n}",
"category": "schema",
- "route_api_model": "deepseek-reasoner",
+ "route_api_model": "claude-haiku-4-5",
"golden_db": "schema-t-021-multi-column-index-golden",
- "llm_db": "schema-t-021-multi-column-index-deepseek-r1-llm",
+ "llm_db": "schema-t-021-multi-column-index-claude-4-5-haiku-llm",
"work_dir_golden": "target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\deepseek-r1\\llm",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\claude-4-5-haiku\\llm",
"scorer_details": {
"publish_error": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\deepseek-r1\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'logs' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\deepseek-r1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\deepseek-r1\\llm\\Lib.cs(16,70): error CS0103: The name 'UserId' does not exist in the current context [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\deepseek-r1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\deepseek-r1\\llm\\Lib.cs(16,86): error CS0103: The name 'Day' does not exist in the current context [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\deepseek-r1\\llm\\StdbModule.csproj]\r\n",
+ "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\claude-4-5-haiku\\llm\\Lib.cs(7,6): error CS0104: 'Index' is an ambiguous reference between 'SpacetimeDB.Index' and 'System.Index' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\claude-4-5-haiku\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\claude-4-5-haiku\\llm\\Lib.cs(7,42): error CS9176: There is no target type for the collection expression. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\claude-4-5-haiku\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\claude-4-5-haiku\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'logs' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\claude-4-5-haiku\\llm\\StdbModule.csproj]\r\n",
"phase": "build_or_publish"
}
}
},
- "vendor": "deepseek",
- "started_at": "2025-10-21T21:33:47.372781600Z",
- "finished_at": "2025-10-21T21:34:32.333574Z"
- },
+ "vendor": "anthropic",
+ "started_at": "2025-10-21T21:22:00.471862400Z",
+ "finished_at": "2025-10-21T21:22:09.531652200Z"
+ }
+ }
+ },
+ {
+ "name": "DeepSeek R1",
+ "route_api_model": "deepseek-reasoner",
+ "tasks": {
"t_000_empty_reducers": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
"task": "t_000_empty_reducers",
@@ -11553,28 +11377,28 @@
"started_at": "2025-10-21T21:39:14.757563400Z",
"finished_at": "2025-10-21T21:41:14.343622600Z"
},
- "t_002_scheduled_table": {
+ "t_001_basic_tables": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_002_scheduled_table",
+ "task": "t_001_basic_tables",
"lang": "csharp",
"golden_published": true,
"model_name": "DeepSeek R1",
"total_tests": 1,
"passed_tests": 1,
- "llm_output": "```csharp\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"tick_timer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [SpacetimeDB.Reducer]\n public static void Tick(ReducerContext ctx, TickTimer timer)\n {\n // Scheduled reducer logic would go here\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.tick_timer.Insert(new TickTimer\n {\n ScheduledId = 0,\n ScheduledAt = new ScheduleAt.Interval(new TimeDuration { Microseconds = 50_000 })\n });\n }\n}\n```",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\")]\n public partial struct Users\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"products\")]\n public partial struct Products\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Title;\n public float Price;\n public bool InStock;\n }\n\n [SpacetimeDB.Table(Name = \"notes\")]\n public partial struct Notes\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Body;\n public long Rating;\n public bool Pinned;\n }\n}",
"category": "basics",
"route_api_model": "deepseek-reasoner",
- "golden_db": "basics-t-002-scheduled-table-golden",
- "llm_db": "basics-t-002-scheduled-table-deepseek-r1-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_002_scheduled_table\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_002_scheduled_table\\csharp\\server\\deepseek-r1\\llm",
+ "golden_db": "basics-t-001-basic-tables-golden",
+ "llm_db": "basics-t-001-basic-tables-deepseek-r1-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\deepseek-r1\\llm",
"scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-002-scheduled-table-golden",
- "llm_db": "basics-t-002-scheduled-table-deepseek-r1-llm",
+ "golden_db": "basics-t-001-basic-tables-golden",
+ "llm_db": "basics-t-001-basic-tables-deepseek-r1-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -11584,58 +11408,31 @@
}
},
"vendor": "deepseek",
- "started_at": "2025-10-21T21:39:16.119052700Z",
- "finished_at": "2025-10-21T21:41:21.784824300Z"
+ "started_at": "2025-10-21T21:39:15.441826600Z",
+ "finished_at": "2025-10-21T21:41:20.741222100Z"
},
- "t_009_init": {
+ "t_002_scheduled_table": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_009_init",
+ "task": "t_002_scheduled_table",
"lang": "csharp",
"golden_published": true,
"model_name": "DeepSeek R1",
- "total_tests": 4,
- "passed_tests": 4,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.users.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.users.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n }\n}",
+ "total_tests": 1,
+ "passed_tests": 1,
+ "llm_output": "```csharp\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"tick_timer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [SpacetimeDB.Reducer]\n public static void Tick(ReducerContext ctx, TickTimer timer)\n {\n // Scheduled reducer logic would go here\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.tick_timer.Insert(new TickTimer\n {\n ScheduledId = 0,\n ScheduledAt = new ScheduleAt.Interval(new TimeDuration { Microseconds = 50_000 })\n });\n }\n}\n```",
"category": "basics",
"route_api_model": "deepseek-reasoner",
- "golden_db": "basics-t-009-init-golden",
- "llm_db": "basics-t-009-init-deepseek-r1-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_009_init\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_009_init\\csharp\\server\\deepseek-r1\\llm",
+ "golden_db": "basics-t-002-scheduled-table-golden",
+ "llm_db": "basics-t-002-scheduled-table-deepseek-r1-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_002_scheduled_table\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_002_scheduled_table\\csharp\\server\\deepseek-r1\\llm",
"scorer_details": {
- "init_seed_alice": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true"
- }
- },
- "init_seed_bob": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false"
- }
- },
- "init_total_two": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 2,
- "expected": 2,
- "sql": "SELECT COUNT(*) AS n FROM users"
- }
- },
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-009-init-golden",
- "llm_db": "basics-t-009-init-deepseek-r1-llm",
+ "golden_db": "basics-t-002-scheduled-table-golden",
+ "llm_db": "basics-t-002-scheduled-table-deepseek-r1-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -11645,31 +11442,31 @@
}
},
"vendor": "deepseek",
- "started_at": "2025-10-21T21:39:21.144559Z",
- "finished_at": "2025-10-21T21:41:21.096123900Z"
+ "started_at": "2025-10-21T21:39:16.119052700Z",
+ "finished_at": "2025-10-21T21:41:21.784824300Z"
},
- "t_010_connect": {
+ "t_003_struct_in_table": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_010_connect",
+ "task": "t_003_struct_in_table",
"lang": "csharp",
"golden_published": true,
"model_name": "DeepSeek R1",
"total_tests": 1,
"passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"events\")]\n public partial struct Event\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public int Id;\n public string Kind;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientConnected)]\n public static void ClientConnected(ReducerContext ctx)\n {\n ctx.Db.events.Insert(new Event { Id = 0, Kind = \"connected\" });\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientDisconnected)]\n public static void ClientDisconnected(ReducerContext ctx)\n {\n ctx.Db.events.Insert(new Event { Id = 0, Kind = \"disconnected\" });\n }\n}",
+ "llm_output": "```csharp\nusing SpacetimeDB;\n\n[SpacetimeDB.Type]\npublic partial struct Position\n{\n public int X;\n public int Y;\n}\n\n[SpacetimeDB.Table(Name = \"entities\")]\npublic partial struct Entity\n{\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Position Pos;\n}\n\npublic static partial class Module\n{\n}\n```",
"category": "basics",
"route_api_model": "deepseek-reasoner",
- "golden_db": "basics-t-010-connect-golden",
- "llm_db": "basics-t-010-connect-deepseek-r1-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_010_connect\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_010_connect\\csharp\\server\\deepseek-r1\\llm",
+ "golden_db": "basics-t-003-struct-in-table-golden",
+ "llm_db": "basics-t-003-struct-in-table-deepseek-r1-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_003_struct_in_table\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_003_struct_in_table\\csharp\\server\\deepseek-r1\\llm",
"scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-010-connect-golden",
- "llm_db": "basics-t-010-connect-deepseek-r1-llm",
+ "golden_db": "basics-t-003-struct-in-table-golden",
+ "llm_db": "basics-t-003-struct-in-table-deepseek-r1-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -11679,54 +11476,31 @@
}
},
"vendor": "deepseek",
- "started_at": "2025-10-21T21:39:21.827123300Z",
- "finished_at": "2025-10-21T21:41:14.998070200Z"
+ "started_at": "2025-10-21T21:39:16.798048900Z",
+ "finished_at": "2025-10-21T21:41:09.488033900Z"
},
- "t_007_crud": {
+ "t_004_insert": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_007_crud",
+ "task": "t_004_insert",
"lang": "csharp",
"golden_published": true,
"model_name": "DeepSeek R1",
- "total_tests": 4,
- "passed_tests": 4,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void Crud(ReducerContext ctx)\n {\n ctx.Db.users.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.users.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n ctx.Db.users.Id.Update(new User { Id = 1, Name = \"Alice2\", Age = 31, Active = false });\n ctx.Db.users.Id.Delete(2);\n }\n}",
+ "total_tests": 2,
+ "passed_tests": 2,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void InsertUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n ctx.Db.users.Insert(new User { Id = id, Name = name, Age = age, Active = active });\n }\n}",
"category": "basics",
"route_api_model": "deepseek-reasoner",
- "golden_db": "basics-t-007-crud-golden",
- "llm_db": "basics-t-007-crud-deepseek-r1-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\deepseek-r1\\llm",
+ "golden_db": "basics-t-004-insert-golden",
+ "llm_db": "basics-t-004-insert-deepseek-r1-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_004_insert\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_004_insert\\csharp\\server\\deepseek-r1\\llm",
"scorer_details": {
- "crud_total_count_one": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM users"
- }
- },
- "crud_row_id1_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "args": [],
- "golden_db": "basics-t-007-crud-golden",
- "golden_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
- "llm_db": "basics-t-007-crud-deepseek-r1-llm",
- "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
- "query": "SELECT Id, Name, Age, Active FROM users WHERE Id=1",
- "reducer": "Crud",
- "server": "local"
- }
- },
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-007-crud-golden",
- "llm_db": "basics-t-007-crud-deepseek-r1-llm",
+ "golden_db": "basics-t-004-insert-golden",
+ "llm_db": "basics-t-004-insert-deepseek-r1-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -11734,19 +11508,29 @@
"tables_equal": true
}
},
- "crud_row_id2_deleted": {
+ "data_parity_insert_user": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 0,
- "expected": 0,
- "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=2"
+ "args": [
+ 1,
+ "Alice",
+ 30,
+ true
+ ],
+ "golden_db": "basics-t-004-insert-golden",
+ "golden_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true",
+ "llm_db": "basics-t-004-insert-deepseek-r1-llm",
+ "llm_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true",
+ "query": "SELECT Id, Name, Age, Active FROM users WHERE Id=1",
+ "reducer": "InsertUser",
+ "server": "local"
}
}
},
"vendor": "deepseek",
- "started_at": "2025-10-21T21:39:19.677357800Z",
- "finished_at": "2025-10-21T21:41:22.347482800Z"
+ "started_at": "2025-10-21T21:39:17.531864Z",
+ "finished_at": "2025-10-21T21:41:14.830810500Z"
},
"t_005_update": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
@@ -11764,13 +11548,6 @@
"work_dir_golden": "target\\llm-runs\\basics\\t_005_update\\csharp\\server\\golden",
"work_dir_llm": "target\\llm-runs\\basics\\t_005_update\\csharp\\server\\deepseek-r1\\llm",
"scorer_details": {
- "seed_users_row": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
- }
- },
"data_parity_update_user": {
"pass": true,
"partial": 1.0,
@@ -11790,128 +11567,19 @@
"server": "local"
}
},
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "golden_db": "basics-t-005-update-golden",
- "llm_db": "basics-t-005-update-deepseek-r1-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
- }
- }
- },
- "vendor": "deepseek",
- "started_at": "2025-10-21T21:39:18.300869400Z",
- "finished_at": "2025-10-21T21:41:20.576102100Z"
- },
- "t_004_insert": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_004_insert",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "DeepSeek R1",
- "total_tests": 2,
- "passed_tests": 2,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void InsertUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n ctx.Db.users.Insert(new User { Id = id, Name = name, Age = age, Active = active });\n }\n}",
- "category": "basics",
- "route_api_model": "deepseek-reasoner",
- "golden_db": "basics-t-004-insert-golden",
- "llm_db": "basics-t-004-insert-deepseek-r1-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_004_insert\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_004_insert\\csharp\\server\\deepseek-r1\\llm",
- "scorer_details": {
- "schema_parity": {
+ "seed_users_row": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-004-insert-golden",
- "llm_db": "basics-t-004-insert-deepseek-r1-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
+ "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
}
},
- "data_parity_insert_user": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "args": [
- 1,
- "Alice",
- 30,
- true
- ],
- "golden_db": "basics-t-004-insert-golden",
- "golden_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true",
- "llm_db": "basics-t-004-insert-deepseek-r1-llm",
- "llm_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true",
- "query": "SELECT Id, Name, Age, Active FROM users WHERE Id=1",
- "reducer": "InsertUser",
- "server": "local"
- }
- }
- },
- "vendor": "deepseek",
- "started_at": "2025-10-21T21:39:17.531864Z",
- "finished_at": "2025-10-21T21:41:14.830810500Z"
- },
- "t_008_index_lookup": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_008_index_lookup",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "DeepSeek R1",
- "total_tests": 3,
- "passed_tests": 0,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"results\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void LookupUserName(ReducerContext ctx, int id)\n {\n var user = ctx.Db.users.Id.Find(id);\n if (user != null)\n {\n ctx.Db.results.Insert(new Result { Id = user.Id, Name = user.Name });\n }\n }\n}",
- "category": "basics",
- "route_api_model": "deepseek-reasoner",
- "golden_db": "basics-t-008-index-lookup-golden",
- "llm_db": "basics-t-008-index-lookup-deepseek-r1-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\deepseek-r1\\llm",
- "scorer_details": {
- "publish_error": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\deepseek-r1\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'results' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\deepseek-r1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\deepseek-r1\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(70,24): warning CS8981: The type name 'users' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\deepseek-r1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\deepseek-r1\\llm\\Lib.cs(30,58): error CS1061: 'Module.User?' does not contain a definition for 'Id' and no accessible extension method 'Id' accepting a first argument of type 'Module.User?' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\deepseek-r1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\deepseek-r1\\llm\\Lib.cs(30,74): error CS1061: 'Module.User?' does not contain a definition for 'Name' and no accessible extension method 'Name' accepting a first argument of type 'Module.User?' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\deepseek-r1\\llm\\StdbModule.csproj]\r\n",
- "phase": "build_or_publish"
- }
- }
- },
- "vendor": "deepseek",
- "started_at": "2025-10-21T21:39:20.345751Z",
- "finished_at": "2025-10-21T21:39:39.519758900Z"
- },
- "t_003_struct_in_table": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_003_struct_in_table",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "DeepSeek R1",
- "total_tests": 1,
- "passed_tests": 1,
- "llm_output": "```csharp\nusing SpacetimeDB;\n\n[SpacetimeDB.Type]\npublic partial struct Position\n{\n public int X;\n public int Y;\n}\n\n[SpacetimeDB.Table(Name = \"entities\")]\npublic partial struct Entity\n{\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Position Pos;\n}\n\npublic static partial class Module\n{\n}\n```",
- "category": "basics",
- "route_api_model": "deepseek-reasoner",
- "golden_db": "basics-t-003-struct-in-table-golden",
- "llm_db": "basics-t-003-struct-in-table-deepseek-r1-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_003_struct_in_table\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_003_struct_in_table\\csharp\\server\\deepseek-r1\\llm",
- "scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-003-struct-in-table-golden",
- "llm_db": "basics-t-003-struct-in-table-deepseek-r1-llm",
+ "golden_db": "basics-t-005-update-golden",
+ "llm_db": "basics-t-005-update-deepseek-r1-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -11921,8 +11589,8 @@
}
},
"vendor": "deepseek",
- "started_at": "2025-10-21T21:39:16.798048900Z",
- "finished_at": "2025-10-21T21:41:09.488033900Z"
+ "started_at": "2025-10-21T21:39:18.300869400Z",
+ "finished_at": "2025-10-21T21:41:20.576102100Z"
},
"t_006_delete": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
@@ -11974,51 +11642,60 @@
"started_at": "2025-10-21T21:39:18.965740900Z",
"finished_at": "2025-10-21T21:41:10.233366700Z"
},
- "t_014_elementary_columns": {
+ "t_007_crud": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_014_elementary_columns",
+ "task": "t_007_crud",
"lang": "csharp",
"golden_published": true,
"model_name": "DeepSeek R1",
- "total_tests": 3,
- "passed_tests": 3,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"primitives\")]\n public partial struct Primitive\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Count;\n public long Total;\n public float Price;\n public double Ratio;\n public bool Active;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.primitives.Insert(new Primitive\n {\n Id = 1,\n Count = 2,\n Total = 3000000000,\n Price = 1.5f,\n Ratio = 2.25,\n Active = true,\n Name = \"Alice\"\n });\n }\n}",
- "category": "schema",
+ "total_tests": 4,
+ "passed_tests": 4,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void Crud(ReducerContext ctx)\n {\n ctx.Db.users.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.users.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n ctx.Db.users.Id.Update(new User { Id = 1, Name = \"Alice2\", Age = 31, Active = false });\n ctx.Db.users.Id.Delete(2);\n }\n}",
+ "category": "basics",
"route_api_model": "deepseek-reasoner",
- "golden_db": "schema-t-014-elementary-columns-golden",
- "llm_db": "schema-t-014-elementary-columns-deepseek-r1-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_014_elementary_columns\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_014_elementary_columns\\csharp\\server\\deepseek-r1\\llm",
+ "golden_db": "basics-t-007-crud-golden",
+ "llm_db": "basics-t-007-crud-deepseek-r1-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\deepseek-r1\\llm",
"scorer_details": {
- "elementary_columns_row_parity": {
+ "crud_row_id1_parity": {
"pass": true,
"partial": 1.0,
"notes": {
"args": [],
- "golden_db": "schema-t-014-elementary-columns-golden",
- "golden_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"",
- "llm_db": "schema-t-014-elementary-columns-deepseek-r1-llm",
- "llm_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"",
- "query": "SELECT Id, Count, Total, Price, Ratio, Active, Name FROM primitives WHERE Id=1",
- "reducer": "Seed",
+ "golden_db": "basics-t-007-crud-golden",
+ "golden_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
+ "llm_db": "basics-t-007-crud-deepseek-r1-llm",
+ "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
+ "query": "SELECT Id, Name, Age, Active FROM users WHERE Id=1",
+ "reducer": "Crud",
"server": "local"
}
},
- "elementary_columns_row_count": {
+ "crud_row_id2_deleted": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 0,
+ "expected": 0,
+ "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=2"
+ }
+ },
+ "crud_total_count_one": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM primitives WHERE Id=1"
+ "sql": "SELECT COUNT(*) AS n FROM users"
}
},
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-014-elementary-columns-golden",
- "llm_db": "schema-t-014-elementary-columns-deepseek-r1-llm",
+ "golden_db": "basics-t-007-crud-golden",
+ "llm_db": "basics-t-007-crud-deepseek-r1-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -12028,68 +11705,60 @@
}
},
"vendor": "deepseek",
- "started_at": "2025-10-21T21:33:43.710607Z",
- "finished_at": "2025-10-21T21:35:27.658519100Z"
+ "started_at": "2025-10-21T21:39:19.677357800Z",
+ "finished_at": "2025-10-21T21:41:22.347482800Z"
},
- "t_018_constraints": {
+ "t_008_index_lookup": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_018_constraints",
+ "task": "t_008_index_lookup",
"lang": "csharp",
"golden_published": true,
"model_name": "DeepSeek R1",
"total_tests": 3,
"passed_tests": 0,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"accounts\")]\n public partial struct Account\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n \n [SpacetimeDB.Unique]\n public string Email;\n \n public string Name;\n \n [SpacetimeDB.Index.BTree(Name = \"by_name\")]\n public string NameIndex => Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.accounts.Insert(new Account { Id = 1, Email = \"a@example.com\", Name = \"Alice\" });\n ctx.Db.accounts.Insert(new Account { Id = 2, Email = \"b@example.com\", Name = \"Bob\" });\n }\n}",
- "category": "schema",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"results\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void LookupUserName(ReducerContext ctx, int id)\n {\n var user = ctx.Db.users.Id.Find(id);\n if (user != null)\n {\n ctx.Db.results.Insert(new Result { Id = user.Id, Name = user.Name });\n }\n }\n}",
+ "category": "basics",
"route_api_model": "deepseek-reasoner",
- "golden_db": "schema-t-018-constraints-golden",
- "llm_db": "schema-t-018-constraints-deepseek-r1-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\deepseek-r1\\llm",
+ "golden_db": "basics-t-008-index-lookup-golden",
+ "llm_db": "basics-t-008-index-lookup-deepseek-r1-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\deepseek-r1\\llm",
"scorer_details": {
"publish_error": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\deepseek-r1\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'accounts' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\deepseek-r1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\deepseek-r1\\llm\\Lib.cs(17,10): error CS0592: Attribute 'SpacetimeDB.Index.BTree' is not valid on this declaration type. It is only valid on 'class, struct, field' declarations. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\deepseek-r1\\llm\\StdbModule.csproj]\r\n",
+ "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\deepseek-r1\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'results' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\deepseek-r1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\deepseek-r1\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(70,24): warning CS8981: The type name 'users' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\deepseek-r1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\deepseek-r1\\llm\\Lib.cs(30,58): error CS1061: 'Module.User?' does not contain a definition for 'Id' and no accessible extension method 'Id' accepting a first argument of type 'Module.User?' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\deepseek-r1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\deepseek-r1\\llm\\Lib.cs(30,74): error CS1061: 'Module.User?' does not contain a definition for 'Name' and no accessible extension method 'Name' accepting a first argument of type 'Module.User?' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\deepseek-r1\\llm\\StdbModule.csproj]\r\n",
"phase": "build_or_publish"
}
}
},
"vendor": "deepseek",
- "started_at": "2025-10-21T21:33:45.804288800Z",
- "finished_at": "2025-10-21T21:34:32.188544700Z"
+ "started_at": "2025-10-21T21:39:20.345751Z",
+ "finished_at": "2025-10-21T21:39:39.519758900Z"
},
- "t_020_ecs": {
+ "t_009_init": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_020_ecs",
+ "task": "t_009_init",
"lang": "csharp",
"golden_published": true,
"model_name": "DeepSeek R1",
- "total_tests": 5,
- "passed_tests": 2,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"entities\")]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n }\n\n [SpacetimeDB.Table(Name = \"positions\")]\n public partial struct Position\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"velocities\")]\n public partial struct Velocity\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int VX;\n public int VY;\n }\n\n [SpacetimeDB.Table(Name = \"next_positions\")]\n public partial struct NextPosition\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.entities.Insert(new Entity { Id = 1 });\n ctx.Db.positions.Insert(new Position { EntityId = 1, X = 0, Y = 0 });\n ctx.Db.velocities.Insert(new Velocity { EntityId = 1, VX = 1, VY = 0 });\n\n ctx.Db.entities.Insert(new Entity { Id = 2 });\n ctx.Db.positions.Insert(new Position { EntityId = 2, X = 10, Y = 0 });\n ctx.Db.velocities.Insert(new Velocity { EntityId = 2, VX = -2, VY = 3 });\n }\n\n [SpacetimeDB.Reducer]\n public static void Step(ReducerContext ctx)\n {\n foreach (var pos in ctx.Db.positions.Iter())\n {\n var vel = ctx.Db.velocities.EntityId.Find(pos.EntityId);\n if (vel.HasValue)\n {\n ctx.Db.next_positions.EntityId.Update(new NextPosition\n {\n EntityId = pos.EntityId,\n X = pos.X + vel.Value.VX,\n Y = pos.Y + vel.Value.VY\n });\n }\n }\n }\n}",
- "category": "schema",
+ "total_tests": 4,
+ "passed_tests": 4,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.users.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.users.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n }\n}",
+ "category": "basics",
"route_api_model": "deepseek-reasoner",
- "golden_db": "schema-t-020-ecs-golden",
- "llm_db": "schema-t-020-ecs-deepseek-r1-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\deepseek-r1\\llm",
+ "golden_db": "basics-t-009-init-golden",
+ "llm_db": "basics-t-009-init-deepseek-r1-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_009_init\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_009_init\\csharp\\server\\deepseek-r1\\llm",
"scorer_details": {
- "ecs_step_next_positions_count": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: SpacetimeDB.NoSuchRowException: The row was not found, e.g., in an update call\n at SpacetimeDB.Internal.FFI.CheckedStatus.Marshaller.ConvertToManaged(Errno )\n at SpacetimeDB.Internal.FFI.datastore_update_bsatn(TableId , IndexId , Span`1 , UInt32& )\n at SpacetimeDB.Internal.UniqueIndex`4[[SpacetimeDB.Internal.TableHandles.next_positions, StdbModule, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null],[Module.NextPosition, StdbModule, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null],[System.Int32, System.Private.CoreLib, Version=8.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e],[SpacetimeDB.BSATN.I32, SpacetimeDB.BSATN.Runtime, Version=1.5.0.0, Culture=neutral, PublicKeyToken=null]].DoUpdate(NextPosition )\n at SpacetimeDB.Internal.TableHandles.next_positions.EntityIdUniqueIndex.Update(NextPosition )\n at Module.Step(ReducerContext )\n at ModuleRegistration.Step.Invoke(BinaryReader , IReducerContext )\n at SpacetimeDB.Internal.Module.__call_reducer__(UInt32 id, UInt64 sender_0, UInt64 sender_1, UInt64 sender_2, UInt64 sender_3, UInt64 conn_id_0, UInt64 conn_id_1, Timestamp timestamp, BytesSource args, BytesSink error)\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:3000/v1/database/c200815c605205023ca92d57e9fb13bd132838935a208007ae3bff05885703a8/call/Step)\n",
- "phase": "call_reducer"
- }
- },
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-020-ecs-golden",
- "llm_db": "schema-t-020-ecs-deepseek-r1-llm",
+ "golden_db": "basics-t-009-init-golden",
+ "llm_db": "basics-t-009-init-deepseek-r1-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -12097,35 +11766,71 @@
"tables_equal": true
}
},
- "ecs_next_pos_entity1": {
- "pass": false,
- "partial": 0.0,
+ "init_seed_alice": {
+ "pass": true,
+ "partial": 1.0,
"notes": {
- "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: SpacetimeDB.NoSuchRowException: The row was not found, e.g., in an update call\n at SpacetimeDB.Internal.FFI.CheckedStatus.Marshaller.ConvertToManaged(Errno )\n at SpacetimeDB.Internal.FFI.datastore_update_bsatn(TableId , IndexId , Span`1 , UInt32& )\n at SpacetimeDB.Internal.UniqueIndex`4[[SpacetimeDB.Internal.TableHandles.next_positions, StdbModule, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null],[Module.NextPosition, StdbModule, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null],[System.Int32, System.Private.CoreLib, Version=8.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e],[SpacetimeDB.BSATN.I32, SpacetimeDB.BSATN.Runtime, Version=1.5.0.0, Culture=neutral, PublicKeyToken=null]].DoUpdate(NextPosition )\n at SpacetimeDB.Internal.TableHandles.next_positions.EntityIdUniqueIndex.Update(NextPosition )\n at Module.Step(ReducerContext )\n at ModuleRegistration.Step.Invoke(BinaryReader , IReducerContext )\n at SpacetimeDB.Internal.Module.__call_reducer__(UInt32 id, UInt64 sender_0, UInt64 sender_1, UInt64 sender_2, UInt64 sender_3, UInt64 conn_id_0, UInt64 conn_id_1, Timestamp timestamp, BytesSource args, BytesSink error)\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:3000/v1/database/c200815c605205023ca92d57e9fb13bd132838935a208007ae3bff05885703a8/call/Step)\n",
- "phase": "call_reducer"
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true"
}
},
- "ecs_seed_positions_count": {
+ "init_total_two": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 2,
"expected": 2,
- "sql": "SELECT COUNT(*) AS n FROM positions"
+ "sql": "SELECT COUNT(*) AS n FROM users"
}
},
- "ecs_next_pos_entity2": {
- "pass": false,
- "partial": 0.0,
+ "init_seed_bob": {
+ "pass": true,
+ "partial": 1.0,
"notes": {
- "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: SpacetimeDB.NoSuchRowException: The row was not found, e.g., in an update call\n at SpacetimeDB.Internal.FFI.CheckedStatus.Marshaller.ConvertToManaged(Errno )\n at SpacetimeDB.Internal.FFI.datastore_update_bsatn(TableId , IndexId , Span`1 , UInt32& )\n at SpacetimeDB.Internal.UniqueIndex`4[[SpacetimeDB.Internal.TableHandles.next_positions, StdbModule, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null],[Module.NextPosition, StdbModule, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null],[System.Int32, System.Private.CoreLib, Version=8.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e],[SpacetimeDB.BSATN.I32, SpacetimeDB.BSATN.Runtime, Version=1.5.0.0, Culture=neutral, PublicKeyToken=null]].DoUpdate(NextPosition )\n at SpacetimeDB.Internal.TableHandles.next_positions.EntityIdUniqueIndex.Update(NextPosition )\n at Module.Step(ReducerContext )\n at ModuleRegistration.Step.Invoke(BinaryReader , IReducerContext )\n at SpacetimeDB.Internal.Module.__call_reducer__(UInt32 id, UInt64 sender_0, UInt64 sender_1, UInt64 sender_2, UInt64 sender_3, UInt64 conn_id_0, UInt64 conn_id_1, Timestamp timestamp, BytesSource args, BytesSink error)\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:3000/v1/database/c200815c605205023ca92d57e9fb13bd132838935a208007ae3bff05885703a8/call/Step)\n",
- "phase": "call_reducer"
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false"
}
}
},
"vendor": "deepseek",
- "started_at": "2025-10-21T21:33:46.869823300Z",
- "finished_at": "2025-10-21T21:35:44.603745Z"
+ "started_at": "2025-10-21T21:39:21.144559Z",
+ "finished_at": "2025-10-21T21:41:21.096123900Z"
+ },
+ "t_010_connect": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_010_connect",
+ "lang": "csharp",
+ "golden_published": true,
+ "model_name": "DeepSeek R1",
+ "total_tests": 1,
+ "passed_tests": 1,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"events\")]\n public partial struct Event\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public int Id;\n public string Kind;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientConnected)]\n public static void ClientConnected(ReducerContext ctx)\n {\n ctx.Db.events.Insert(new Event { Id = 0, Kind = \"connected\" });\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientDisconnected)]\n public static void ClientDisconnected(ReducerContext ctx)\n {\n ctx.Db.events.Insert(new Event { Id = 0, Kind = \"disconnected\" });\n }\n}",
+ "category": "basics",
+ "route_api_model": "deepseek-reasoner",
+ "golden_db": "basics-t-010-connect-golden",
+ "llm_db": "basics-t-010-connect-deepseek-r1-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_010_connect\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_010_connect\\csharp\\server\\deepseek-r1\\llm",
+ "scorer_details": {
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "golden_db": "basics-t-010-connect-golden",
+ "llm_db": "basics-t-010-connect-deepseek-r1-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
+ }
+ }
+ },
+ "vendor": "deepseek",
+ "started_at": "2025-10-21T21:39:21.827123300Z",
+ "finished_at": "2025-10-21T21:41:14.998070200Z"
},
"t_011_helper_function": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
@@ -12143,22 +11848,13 @@
"work_dir_golden": "target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\golden",
"work_dir_llm": "target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\deepseek-r1\\llm",
"scorer_details": {
- "helper_func_sum_parity": {
+ "helper_func_sum_abs": {
"pass": true,
"partial": 1.0,
"notes": {
- "args": [
- 1,
- 2,
- 3
- ],
- "golden_db": "basics-t-011-helper-function-golden",
- "golden_out": "Id | Sum ----+----- 1 | 5",
- "llm_db": "basics-t-011-helper-function-deepseek-r1-llm",
- "llm_out": "Id | Sum ----+----- 1 | 5",
- "query": "SELECT Id, Sum FROM results WHERE Id=1",
- "reducer": "ComputeSum",
- "server": "local"
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1 AND Sum=5"
}
},
"schema_parity": {
@@ -12174,13 +11870,22 @@
"tables_equal": true
}
},
- "helper_func_sum_abs": {
+ "helper_func_sum_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1 AND Sum=5"
+ "args": [
+ 1,
+ 2,
+ 3
+ ],
+ "golden_db": "basics-t-011-helper-function-golden",
+ "golden_out": "Id | Sum ----+----- 1 | 5",
+ "llm_db": "basics-t-011-helper-function-deepseek-r1-llm",
+ "llm_out": "Id | Sum ----+----- 1 | 5",
+ "query": "SELECT Id, Sum FROM results WHERE Id=1",
+ "reducer": "ComputeSum",
+ "server": "local"
}
}
},
@@ -12188,213 +11893,22 @@
"started_at": "2025-10-21T21:39:22.504027600Z",
"finished_at": "2025-10-21T21:41:21.651539700Z"
},
- "t_019_many_to_many": {
+ "t_012_spacetime_product_type": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_019_many_to_many",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "DeepSeek R1",
- "total_tests": 5,
- "passed_tests": 0,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int UserId;\n public string Name;\n }\n\n [SpacetimeDB.Table]\n public partial struct Group\n {\n [SpacetimeDB.PrimaryKey]\n public int GroupId;\n public string Title;\n }\n\n [SpacetimeDB.Table]\n public partial struct Membership\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int GroupId;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.user.Insert(new User { UserId = 1, Name = \"Alice\" });\n ctx.Db.user.Insert(new User { UserId = 2, Name = \"Bob\" });\n\n ctx.Db.group.Insert(new Group { GroupId = 10, Title = \"Admin\" });\n ctx.Db.group.Insert(new Group { GroupId = 20, Title = \"Dev\" });\n\n ctx.Db.membership.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\n ctx.Db.membership.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\n ctx.Db.membership.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\n }\n}",
- "category": "schema",
- "route_api_model": "deepseek-reasoner",
- "golden_db": "schema-t-019-many-to-many-golden",
- "llm_db": "schema-t-019-many-to-many-deepseek-r1-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\deepseek-r1\\llm",
- "scorer_details": {
- "publish_error": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\deepseek-r1\\llm\\Lib.cs(34,16): error CS1061: 'Local' does not contain a definition for 'user' and no accessible extension method 'user' accepting a first argument of type 'Local' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\deepseek-r1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\deepseek-r1\\llm\\Lib.cs(35,16): error CS1061: 'Local' does not contain a definition for 'user' and no accessible extension method 'user' accepting a first argument of type 'Local' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\deepseek-r1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\deepseek-r1\\llm\\Lib.cs(37,16): error CS1061: 'Local' does not contain a definition for 'group' and no accessible extension method 'group' accepting a first argument of type 'Local' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\deepseek-r1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\deepseek-r1\\llm\\Lib.cs(38,16): error CS1061: 'Local' does not contain a definition for 'group' and no accessible extension method 'group' accepting a first argument of type 'Local' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\deepseek-r1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\deepseek-r1\\llm\\Lib.cs(40,16): error CS1061: 'Local' does not contain a definition for 'membership' and no accessible extension method 'membership' accepting a first argument of type 'Local' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\deepseek-r1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\deepseek-r1\\llm\\Lib.cs(41,16): error CS1061: 'Local' does not contain a definition for 'membership' and no accessible extension method 'membership' accepting a first argument of type 'Local' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\deepseek-r1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\deepseek-r1\\llm\\Lib.cs(42,16): error CS1061: 'Local' does not contain a definition for 'membership' and no accessible extension method 'membership' accepting a first argument of type 'Local' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\deepseek-r1\\llm\\StdbModule.csproj]\r\n",
- "phase": "build_or_publish"
- }
- }
- },
- "vendor": "deepseek",
- "started_at": "2025-10-21T21:33:46.352762300Z",
- "finished_at": "2025-10-21T21:34:40.791806900Z"
- },
- "t_016_sum_type_columns": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_016_sum_type_columns",
+ "task": "t_012_spacetime_product_type",
"lang": "csharp",
"golden_published": true,
"model_name": "DeepSeek R1",
"total_tests": 3,
"passed_tests": 3,
- "llm_output": "```csharp\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"drawings\")]\n public partial struct Drawing\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape A;\n public Shape B;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.drawings.Insert(new Drawing\n {\n Id = 1,\n A = new Shape.Circle(new Circle { Radius = 10 }),\n B = new Shape.Rectangle(new Rectangle { Width = 4, Height = 6 })\n });\n }\n}\n```",
- "category": "schema",
- "route_api_model": "deepseek-reasoner",
- "golden_db": "schema-t-016-sum-type-columns-golden",
- "llm_db": "schema-t-016-sum-type-columns-deepseek-r1-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\deepseek-r1\\llm",
- "scorer_details": {
- "sum_type_columns_row_count": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM drawings WHERE Id=1"
- }
- },
- "sum_type_columns_row_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "args": [],
- "golden_db": "schema-t-016-sum-type-columns-golden",
- "golden_out": "Id | A | B ----+--------------------------+--------------------------------------- 1 | (Circle = (Radius = 10)) | (Rectangle = (Width = 4, Height = 6))",
- "llm_db": "schema-t-016-sum-type-columns-deepseek-r1-llm",
- "llm_out": "Id | A | B ----+--------------------------+--------------------------------------- 1 | (Circle = (Radius = 10)) | (Rectangle = (Width = 4, Height = 6))",
- "query": "SELECT Id, A, B FROM drawings WHERE Id=1",
- "reducer": "Seed",
- "server": "local"
- }
- },
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "golden_db": "schema-t-016-sum-type-columns-golden",
- "llm_db": "schema-t-016-sum-type-columns-deepseek-r1-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
- }
- }
- },
- "vendor": "deepseek",
- "started_at": "2025-10-21T21:33:44.740329400Z",
- "finished_at": "2025-10-21T21:35:28.238303500Z"
- },
- "t_001_basic_tables": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_001_basic_tables",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "DeepSeek R1",
- "total_tests": 1,
- "passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\")]\n public partial struct Users\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"products\")]\n public partial struct Products\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Title;\n public float Price;\n public bool InStock;\n }\n\n [SpacetimeDB.Table(Name = \"notes\")]\n public partial struct Notes\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Body;\n public long Rating;\n public bool Pinned;\n }\n}",
- "category": "basics",
- "route_api_model": "deepseek-reasoner",
- "golden_db": "basics-t-001-basic-tables-golden",
- "llm_db": "basics-t-001-basic-tables-deepseek-r1-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\deepseek-r1\\llm",
- "scorer_details": {
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "golden_db": "basics-t-001-basic-tables-golden",
- "llm_db": "basics-t-001-basic-tables-deepseek-r1-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
- }
- }
- },
- "vendor": "deepseek",
- "started_at": "2025-10-21T21:39:15.441826600Z",
- "finished_at": "2025-10-21T21:41:20.741222100Z"
- },
- "t_017_scheduled_columns": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_017_scheduled_columns",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "DeepSeek R1",
- "total_tests": 2,
- "passed_tests": 2,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"tick_timer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [SpacetimeDB.Reducer]\n public static void Tick(ReducerContext ctx, TickTimer schedule)\n {\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.tick_timer.Insert(new TickTimer\n {\n ScheduledId = 0,\n ScheduledAt = new ScheduleAt.Interval(new TimeDuration { Microseconds = 50_000 })\n });\n }\n}",
+ "llm_output": "using SpacetimeDB;\n\n[SpacetimeDB.Type]\npublic partial struct Score\n{\n public int Left;\n public int Right;\n}\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"results\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Score Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetScore(ReducerContext ctx, int id, int left, int right)\n {\n ctx.Db.results.Insert(new Result\n {\n Id = id,\n Value = new Score { Left = left, Right = right }\n });\n }\n}",
"category": "schema",
"route_api_model": "deepseek-reasoner",
- "golden_db": "schema-t-017-scheduled-columns-golden",
- "llm_db": "schema-t-017-scheduled-columns-deepseek-r1-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_017_scheduled_columns\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_017_scheduled_columns\\csharp\\server\\deepseek-r1\\llm",
- "scorer_details": {
- "scheduled_seeded_one_row": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM tick_timer WHERE ScheduledId>=0"
- }
- },
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "golden_db": "schema-t-017-scheduled-columns-golden",
- "llm_db": "schema-t-017-scheduled-columns-deepseek-r1-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
- }
- }
- },
- "vendor": "deepseek",
- "started_at": "2025-10-21T21:33:45.255546Z",
- "finished_at": "2025-10-21T21:35:30.065368Z"
- }
- }
- },
- {
- "name": "DeepSeek V3",
- "route_api_model": "deepseek-chat",
- "tasks": {
- "t_012_spacetime_product_type": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_012_spacetime_product_type",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "DeepSeek V3",
- "total_tests": 3,
- "passed_tests": 3,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Score\n {\n public int Left;\n public int Right;\n }\n\n [SpacetimeDB.Table(Name = \"results\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Score Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetScore(ReducerContext ctx, int id, int left, int right)\n {\n ctx.Db.results.Insert(new Result\n {\n Id = id,\n Value = new Score { Left = left, Right = right }\n });\n }\n}",
- "category": "schema",
- "route_api_model": "deepseek-chat",
"golden_db": "schema-t-012-spacetime-product-type-golden",
- "llm_db": "schema-t-012-spacetime-product-type-deepseek-v3-llm",
+ "llm_db": "schema-t-012-spacetime-product-type-deepseek-r1-llm",
"work_dir_golden": "target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\deepseek-v3\\llm",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\deepseek-r1\\llm",
"scorer_details": {
- "product_type_row_count": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1"
- }
- },
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "golden_db": "schema-t-012-spacetime-product-type-golden",
- "llm_db": "schema-t-012-spacetime-product-type-deepseek-v3-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
- }
- },
"product_type_row_parity": {
"pass": true,
"partial": 1.0,
@@ -12406,131 +11920,84 @@
],
"golden_db": "schema-t-012-spacetime-product-type-golden",
"golden_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)",
- "llm_db": "schema-t-012-spacetime-product-type-deepseek-v3-llm",
+ "llm_db": "schema-t-012-spacetime-product-type-deepseek-r1-llm",
"llm_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)",
"query": "SELECT Id, Value FROM results WHERE Id=1",
"reducer": "SetScore",
"server": "local"
}
- }
- },
- "vendor": "deepseek",
- "started_at": "2025-10-21T21:47:24.300697500Z",
- "finished_at": "2025-10-21T21:48:58.320135Z"
- },
- "t_019_many_to_many": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_019_many_to_many",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "DeepSeek V3",
- "total_tests": 5,
- "passed_tests": 4,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int UserId;\n public string Name;\n }\n\n [SpacetimeDB.Table(Name = \"groups\", Public = true)]\n public partial struct Group\n {\n [SpacetimeDB.PrimaryKey]\n public int GroupId;\n public string Title;\n }\n\n [SpacetimeDB.Table(Name = \"memberships\", Public = true)]\n [SpacetimeDB.Index.BTree(Name = \"by_user\", Columns = [nameof(UserId)])]\n [SpacetimeDB.Index.BTree(Name = \"by_group\", Columns = [nameof(GroupId)])]\n public partial struct Membership\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int GroupId;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.users.Insert(new User { UserId = 1, Name = \"Alice\" });\n ctx.Db.users.Insert(new User { UserId = 2, Name = \"Bob\" });\n\n ctx.Db.groups.Insert(new Group { GroupId = 10, Title = \"Admin\" });\n ctx.Db.groups.Insert(new Group { GroupId = 20, Title = \"Dev\" });\n\n ctx.Db.memberships.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\n ctx.Db.memberships.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\n ctx.Db.memberships.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\n }\n}",
- "category": "schema",
- "route_api_model": "deepseek-chat",
- "golden_db": "schema-t-019-many-to-many-golden",
- "llm_db": "schema-t-019-many-to-many-deepseek-v3-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\deepseek-v3\\llm",
- "scorer_details": {
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "golden_db": "schema-t-019-many-to-many-golden",
- "llm_db": "schema-t-019-many-to-many-deepseek-v3-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
- }
- },
- "m2m_has_1_10": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: can't directly call special Init lifecycle reducer\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:3000/v1/database/c2006709be6e83ff9679cb5896e4dcfd226fc63c542008337dfa50ce18aa3869/call/Seed)\n",
- "phase": "call_reducer"
- }
- },
- "memberships_three_rows": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 3,
- "expected": 3,
- "sql": "SELECT COUNT(*) AS n FROM memberships"
- }
},
- "m2m_has_2_20": {
+ "product_type_row_count": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM memberships WHERE UserId=2 AND GroupId=20"
+ "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1"
}
},
- "m2m_has_1_20": {
+ "schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM memberships WHERE UserId=1 AND GroupId=20"
+ "golden_db": "schema-t-012-spacetime-product-type-golden",
+ "llm_db": "schema-t-012-spacetime-product-type-deepseek-r1-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
}
}
},
"vendor": "deepseek",
- "started_at": "2025-10-21T21:47:28.582794500Z",
- "finished_at": "2025-10-21T21:49:10.680876500Z"
+ "started_at": "2025-10-21T21:33:42.685896400Z",
+ "finished_at": "2025-10-21T21:35:28.769962700Z"
},
- "t_008_index_lookup": {
+ "t_013_spacetime_sum_type": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_008_index_lookup",
+ "task": "t_013_spacetime_sum_type",
"lang": "csharp",
"golden_published": true,
- "model_name": "DeepSeek V3",
+ "model_name": "DeepSeek R1",
"total_tests": 3,
"passed_tests": 0,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"results\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void LookupUserName(ReducerContext ctx, int id)\n {\n var user = ctx.Db.users.Id.Find(id);\n if (user != null)\n {\n ctx.Db.results.Insert(new Result { Id = user.Id, Name = user.Name });\n }\n }\n}",
- "category": "basics",
- "route_api_model": "deepseek-chat",
- "golden_db": "basics-t-008-index-lookup-golden",
- "llm_db": "basics-t-008-index-lookup-deepseek-v3-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\deepseek-v3\\llm",
+ "llm_output": "using SpacetimeDB;\n\n[SpacetimeDB.Type]\npublic partial struct Circle\n{\n public int Radius;\n}\n\n[SpacetimeDB.Type]\npublic partial struct Rectangle\n{\n public int Width;\n public int Height;\n}\n\n[SpacetimeDB.Type]\npublic partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"results\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetCircle(ReducerContext ctx, int id, int radius)\n {\n ctx.Db.results.Insert(new Result\n {\n Id = id,\n Value = new Shape.Circle(new Circle { Radius = radius })\n });\n }\n}",
+ "category": "schema",
+ "route_api_model": "deepseek-reasoner",
+ "golden_db": "schema-t-013-spacetime-sum-type-golden",
+ "llm_db": "schema-t-013-spacetime-sum-type-deepseek-r1-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\deepseek-r1\\llm",
"scorer_details": {
"publish_error": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\deepseek-v3\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(70,24): warning CS8981: The type name 'users' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\deepseek-v3\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\deepseek-v3\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'results' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\deepseek-v3\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\deepseek-v3\\llm\\Lib.cs(30,58): error CS1061: 'Module.User?' does not contain a definition for 'Id' and no accessible extension method 'Id' accepting a first argument of type 'Module.User?' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\deepseek-v3\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\deepseek-v3\\llm\\Lib.cs(30,74): error CS1061: 'Module.User?' does not contain a definition for 'Name' and no accessible extension method 'Name' accepting a first argument of type 'Module.User?' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\deepseek-v3\\llm\\StdbModule.csproj]\r\n",
+ "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\deepseek-r1\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'results' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\deepseek-r1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\deepseek-r1\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.BSATN.Codegen\\SpacetimeDB.Codegen.Type\\Shape.cs(6,26): error CS8910: The primary constructor conflicts with the synthesized copy constructor. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\deepseek-r1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\deepseek-r1\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.BSATN.Codegen\\SpacetimeDB.Codegen.Type\\Shape.cs(12,26): error CS8910: The primary constructor conflicts with the synthesized copy constructor. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\deepseek-r1\\llm\\StdbModule.csproj]\r\n",
"phase": "build_or_publish"
}
}
},
"vendor": "deepseek",
- "started_at": "2025-10-21T21:43:19.478413200Z",
- "finished_at": "2025-10-21T21:43:35.441778500Z"
+ "started_at": "2025-10-21T21:33:43.223808Z",
+ "finished_at": "2025-10-21T21:34:32.511591800Z"
},
"t_014_elementary_columns": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
"task": "t_014_elementary_columns",
"lang": "csharp",
"golden_published": true,
- "model_name": "DeepSeek V3",
+ "model_name": "DeepSeek R1",
"total_tests": 3,
"passed_tests": 3,
"llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"primitives\")]\n public partial struct Primitive\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Count;\n public long Total;\n public float Price;\n public double Ratio;\n public bool Active;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.primitives.Insert(new Primitive\n {\n Id = 1,\n Count = 2,\n Total = 3000000000,\n Price = 1.5f,\n Ratio = 2.25,\n Active = true,\n Name = \"Alice\"\n });\n }\n}",
"category": "schema",
- "route_api_model": "deepseek-chat",
+ "route_api_model": "deepseek-reasoner",
"golden_db": "schema-t-014-elementary-columns-golden",
- "llm_db": "schema-t-014-elementary-columns-deepseek-v3-llm",
+ "llm_db": "schema-t-014-elementary-columns-deepseek-r1-llm",
"work_dir_golden": "target\\llm-runs\\schema\\t_014_elementary_columns\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_014_elementary_columns\\csharp\\server\\deepseek-v3\\llm",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_014_elementary_columns\\csharp\\server\\deepseek-r1\\llm",
"scorer_details": {
"elementary_columns_row_parity": {
"pass": true,
@@ -12539,7 +12006,7 @@
"args": [],
"golden_db": "schema-t-014-elementary-columns-golden",
"golden_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"",
- "llm_db": "schema-t-014-elementary-columns-deepseek-v3-llm",
+ "llm_db": "schema-t-014-elementary-columns-deepseek-r1-llm",
"llm_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"",
"query": "SELECT Id, Count, Total, Price, Ratio, Active, Name FROM primitives WHERE Id=1",
"reducer": "Seed",
@@ -12560,7 +12027,7 @@
"partial": 1.0,
"notes": {
"golden_db": "schema-t-014-elementary-columns-golden",
- "llm_db": "schema-t-014-elementary-columns-deepseek-v3-llm",
+ "llm_db": "schema-t-014-elementary-columns-deepseek-r1-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -12570,52 +12037,25 @@
}
},
"vendor": "deepseek",
- "started_at": "2025-10-21T21:47:25.560295400Z",
- "finished_at": "2025-10-21T21:49:06.926867300Z"
+ "started_at": "2025-10-21T21:33:43.710607Z",
+ "finished_at": "2025-10-21T21:35:27.658519100Z"
},
"t_015_product_type_columns": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
"task": "t_015_product_type_columns",
"lang": "csharp",
"golden_published": true,
- "model_name": "DeepSeek V3",
+ "model_name": "DeepSeek R1",
"total_tests": 3,
"passed_tests": 3,
"llm_output": "using SpacetimeDB;\n\n[SpacetimeDB.Type]\npublic partial struct Address\n{\n public string Street;\n public int Zip;\n}\n\n[SpacetimeDB.Type]\npublic partial struct Position\n{\n public int X;\n public int Y;\n}\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"profiles\")]\n public partial struct Profile\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Address Home;\n public Address Work;\n public Position Pos;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.profiles.Insert(new Profile\n {\n Id = 1,\n Home = new Address { Street = \"1 Main\", Zip = 11111 },\n Work = new Address { Street = \"2 Broad\", Zip = 22222 },\n Pos = new Position { X = 7, Y = 9 }\n });\n }\n}",
"category": "schema",
- "route_api_model": "deepseek-chat",
+ "route_api_model": "deepseek-reasoner",
"golden_db": "schema-t-015-product-type-columns-golden",
- "llm_db": "schema-t-015-product-type-columns-deepseek-v3-llm",
+ "llm_db": "schema-t-015-product-type-columns-deepseek-r1-llm",
"work_dir_golden": "target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\deepseek-v3\\llm",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\deepseek-r1\\llm",
"scorer_details": {
- "product_type_columns_row_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "args": [],
- "golden_db": "schema-t-015-product-type-columns-golden",
- "golden_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)",
- "llm_db": "schema-t-015-product-type-columns-deepseek-v3-llm",
- "llm_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)",
- "query": "SELECT Id, Home, Work, Pos FROM profiles WHERE Id=1",
- "reducer": "Seed",
- "server": "local"
- }
- },
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "golden_db": "schema-t-015-product-type-columns-golden",
- "llm_db": "schema-t-015-product-type-columns-deepseek-v3-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
- }
- },
"product_type_columns_row_count": {
"pass": true,
"partial": 1.0,
@@ -12624,111 +12064,70 @@
"expected": 1,
"sql": "SELECT COUNT(*) AS n FROM profiles WHERE Id=1"
}
- }
- },
- "vendor": "deepseek",
- "started_at": "2025-10-21T21:47:26.187633700Z",
- "finished_at": "2025-10-21T21:49:08.548904400Z"
- },
- "t_000_empty_reducers": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_000_empty_reducers",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "DeepSeek V3",
- "total_tests": 1,
- "passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_NoArgs(ReducerContext ctx)\n {\n }\n\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_WithInt(ReducerContext ctx, int count)\n {\n }\n\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_WithString(ReducerContext ctx, string name)\n {\n }\n\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_WithTwoArgs(ReducerContext ctx, int count, string name)\n {\n }\n\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_WithThreeArgs(ReducerContext ctx, bool active, float ratio, string label)\n {\n }\n}",
- "category": "basics",
- "route_api_model": "deepseek-chat",
- "golden_db": "basics-t-000-empty-reducers-golden",
- "llm_db": "basics-t-000-empty-reducers-deepseek-v3-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\deepseek-v3\\llm",
- "scorer_details": {
+ },
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-000-empty-reducers-golden",
- "llm_db": "basics-t-000-empty-reducers-deepseek-v3-llm",
+ "golden_db": "schema-t-015-product-type-columns-golden",
+ "llm_db": "schema-t-015-product-type-columns-deepseek-r1-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
"tables_diff": null,
"tables_equal": true
}
- }
- },
- "vendor": "deepseek",
- "started_at": "2025-10-21T21:43:15.268555100Z",
- "finished_at": "2025-10-21T21:44:58.653818500Z"
- },
- "t_001_basic_tables": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_001_basic_tables",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "DeepSeek V3",
- "total_tests": 1,
- "passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\", Public = true)]\n public partial struct Users\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"products\", Public = true)]\n public partial struct Products\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Title;\n public float Price;\n public bool InStock;\n }\n\n [SpacetimeDB.Table(Name = \"notes\", Public = true)]\n public partial struct Notes\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Body;\n public long Rating;\n public bool Pinned;\n }\n}",
- "category": "basics",
- "route_api_model": "deepseek-chat",
- "golden_db": "basics-t-001-basic-tables-golden",
- "llm_db": "basics-t-001-basic-tables-deepseek-v3-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\deepseek-v3\\llm",
- "scorer_details": {
- "schema_parity": {
+ },
+ "product_type_columns_row_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-001-basic-tables-golden",
- "llm_db": "basics-t-001-basic-tables-deepseek-v3-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
+ "args": [],
+ "golden_db": "schema-t-015-product-type-columns-golden",
+ "golden_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)",
+ "llm_db": "schema-t-015-product-type-columns-deepseek-r1-llm",
+ "llm_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)",
+ "query": "SELECT Id, Home, Work, Pos FROM profiles WHERE Id=1",
+ "reducer": "Seed",
+ "server": "local"
}
}
},
"vendor": "deepseek",
- "started_at": "2025-10-21T21:43:15.819316200Z",
- "finished_at": "2025-10-21T21:44:59.876292400Z"
+ "started_at": "2025-10-21T21:33:44.211931300Z",
+ "finished_at": "2025-10-21T21:35:27.125850100Z"
},
- "t_013_spacetime_sum_type": {
+ "t_016_sum_type_columns": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_013_spacetime_sum_type",
+ "task": "t_016_sum_type_columns",
"lang": "csharp",
"golden_published": true,
- "model_name": "DeepSeek V3",
+ "model_name": "DeepSeek R1",
"total_tests": 3,
"passed_tests": 3,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"results\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetCircle(ReducerContext ctx, int id, int radius)\n {\n ctx.Db.results.Insert(new Result\n {\n Id = id,\n Value = new Shape.Circle(new Circle { Radius = radius })\n });\n }\n}",
+ "llm_output": "```csharp\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"drawings\")]\n public partial struct Drawing\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape A;\n public Shape B;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.drawings.Insert(new Drawing\n {\n Id = 1,\n A = new Shape.Circle(new Circle { Radius = 10 }),\n B = new Shape.Rectangle(new Rectangle { Width = 4, Height = 6 })\n });\n }\n}\n```",
"category": "schema",
- "route_api_model": "deepseek-chat",
- "golden_db": "schema-t-013-spacetime-sum-type-golden",
- "llm_db": "schema-t-013-spacetime-sum-type-deepseek-v3-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\deepseek-v3\\llm",
+ "route_api_model": "deepseek-reasoner",
+ "golden_db": "schema-t-016-sum-type-columns-golden",
+ "llm_db": "schema-t-016-sum-type-columns-deepseek-r1-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\deepseek-r1\\llm",
"scorer_details": {
- "sum_type_row_count": {
+ "sum_type_columns_row_count": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1"
+ "sql": "SELECT COUNT(*) AS n FROM drawings WHERE Id=1"
}
},
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-013-spacetime-sum-type-golden",
- "llm_db": "schema-t-013-spacetime-sum-type-deepseek-v3-llm",
+ "golden_db": "schema-t-016-sum-type-columns-golden",
+ "llm_db": "schema-t-016-sum-type-columns-deepseek-r1-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -12736,102 +12135,164 @@
"tables_equal": true
}
},
- "sum_type_row_parity": {
+ "sum_type_columns_row_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "args": [
- 1,
- 10
- ],
- "golden_db": "schema-t-013-spacetime-sum-type-golden",
- "golden_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))",
- "llm_db": "schema-t-013-spacetime-sum-type-deepseek-v3-llm",
- "llm_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))",
- "query": "SELECT Id, Value FROM results WHERE Id=1",
- "reducer": "SetCircle",
+ "args": [],
+ "golden_db": "schema-t-016-sum-type-columns-golden",
+ "golden_out": "Id | A | B ----+--------------------------+--------------------------------------- 1 | (Circle = (Radius = 10)) | (Rectangle = (Width = 4, Height = 6))",
+ "llm_db": "schema-t-016-sum-type-columns-deepseek-r1-llm",
+ "llm_out": "Id | A | B ----+--------------------------+--------------------------------------- 1 | (Circle = (Radius = 10)) | (Rectangle = (Width = 4, Height = 6))",
+ "query": "SELECT Id, A, B FROM drawings WHERE Id=1",
+ "reducer": "Seed",
"server": "local"
}
}
},
"vendor": "deepseek",
- "started_at": "2025-10-21T21:47:24.962875700Z",
- "finished_at": "2025-10-21T21:49:10.062838600Z"
+ "started_at": "2025-10-21T21:33:44.740329400Z",
+ "finished_at": "2025-10-21T21:35:28.238303500Z"
},
"t_017_scheduled_columns": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
"task": "t_017_scheduled_columns",
"lang": "csharp",
"golden_published": true,
- "model_name": "DeepSeek V3",
+ "model_name": "DeepSeek R1",
"total_tests": 2,
"passed_tests": 2,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"tick_timer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public ulong ScheduledId;\n\n public ScheduleAt ScheduledAt;\n }\n\n [SpacetimeDB.Reducer]\n public static void Tick(ReducerContext ctx, TickTimer schedule)\n {\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.tick_timer.Insert(new TickTimer\n {\n ScheduledId = 0,\n ScheduledAt = new ScheduleAt.Interval(new TimeDuration { Microseconds = 50_000 })\n });\n }\n}",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"tick_timer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [SpacetimeDB.Reducer]\n public static void Tick(ReducerContext ctx, TickTimer schedule)\n {\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.tick_timer.Insert(new TickTimer\n {\n ScheduledId = 0,\n ScheduledAt = new ScheduleAt.Interval(new TimeDuration { Microseconds = 50_000 })\n });\n }\n}",
"category": "schema",
- "route_api_model": "deepseek-chat",
+ "route_api_model": "deepseek-reasoner",
"golden_db": "schema-t-017-scheduled-columns-golden",
- "llm_db": "schema-t-017-scheduled-columns-deepseek-v3-llm",
+ "llm_db": "schema-t-017-scheduled-columns-deepseek-r1-llm",
"work_dir_golden": "target\\llm-runs\\schema\\t_017_scheduled_columns\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_017_scheduled_columns\\csharp\\server\\deepseek-v3\\llm",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_017_scheduled_columns\\csharp\\server\\deepseek-r1\\llm",
"scorer_details": {
- "scheduled_seeded_one_row": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM tick_timer WHERE ScheduledId>=0"
- }
- },
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
"golden_db": "schema-t-017-scheduled-columns-golden",
- "llm_db": "schema-t-017-scheduled-columns-deepseek-v3-llm",
+ "llm_db": "schema-t-017-scheduled-columns-deepseek-r1-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
"tables_diff": null,
"tables_equal": true
}
+ },
+ "scheduled_seeded_one_row": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM tick_timer WHERE ScheduledId>=0"
+ }
}
},
"vendor": "deepseek",
- "started_at": "2025-10-21T21:47:27.382264500Z",
- "finished_at": "2025-10-21T21:49:09.480877Z"
+ "started_at": "2025-10-21T21:33:45.255546Z",
+ "finished_at": "2025-10-21T21:35:30.065368Z"
},
- "t_011_helper_function": {
+ "t_018_constraints": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_011_helper_function",
+ "task": "t_018_constraints",
"lang": "csharp",
"golden_published": true,
- "model_name": "DeepSeek V3",
+ "model_name": "DeepSeek R1",
"total_tests": 3,
- "passed_tests": 3,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"results\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Sum;\n }\n\n private static int Add(int a, int b)\n {\n return a + b;\n }\n\n [SpacetimeDB.Reducer]\n public static void ComputeSum(ReducerContext ctx, int id, int a, int b)\n {\n ctx.Db.results.Insert(new Result { Id = id, Sum = Add(a, b) });\n }\n}",
- "category": "basics",
- "route_api_model": "deepseek-chat",
- "golden_db": "basics-t-011-helper-function-golden",
- "llm_db": "basics-t-011-helper-function-deepseek-v3-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\deepseek-v3\\llm",
+ "passed_tests": 0,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"accounts\")]\n public partial struct Account\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n \n [SpacetimeDB.Unique]\n public string Email;\n \n public string Name;\n \n [SpacetimeDB.Index.BTree(Name = \"by_name\")]\n public string NameIndex => Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.accounts.Insert(new Account { Id = 1, Email = \"a@example.com\", Name = \"Alice\" });\n ctx.Db.accounts.Insert(new Account { Id = 2, Email = \"b@example.com\", Name = \"Bob\" });\n }\n}",
+ "category": "schema",
+ "route_api_model": "deepseek-reasoner",
+ "golden_db": "schema-t-018-constraints-golden",
+ "llm_db": "schema-t-018-constraints-deepseek-r1-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\deepseek-r1\\llm",
"scorer_details": {
- "helper_func_sum_abs": {
- "pass": true,
- "partial": 1.0,
+ "publish_error": {
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1 AND Sum=5"
+ "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\deepseek-r1\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'accounts' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\deepseek-r1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\deepseek-r1\\llm\\Lib.cs(17,10): error CS0592: Attribute 'SpacetimeDB.Index.BTree' is not valid on this declaration type. It is only valid on 'class, struct, field' declarations. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\deepseek-r1\\llm\\StdbModule.csproj]\r\n",
+ "phase": "build_or_publish"
+ }
+ }
+ },
+ "vendor": "deepseek",
+ "started_at": "2025-10-21T21:33:45.804288800Z",
+ "finished_at": "2025-10-21T21:34:32.188544700Z"
+ },
+ "t_019_many_to_many": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_019_many_to_many",
+ "lang": "csharp",
+ "golden_published": true,
+ "model_name": "DeepSeek R1",
+ "total_tests": 5,
+ "passed_tests": 0,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int UserId;\n public string Name;\n }\n\n [SpacetimeDB.Table]\n public partial struct Group\n {\n [SpacetimeDB.PrimaryKey]\n public int GroupId;\n public string Title;\n }\n\n [SpacetimeDB.Table]\n public partial struct Membership\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int GroupId;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.user.Insert(new User { UserId = 1, Name = \"Alice\" });\n ctx.Db.user.Insert(new User { UserId = 2, Name = \"Bob\" });\n\n ctx.Db.group.Insert(new Group { GroupId = 10, Title = \"Admin\" });\n ctx.Db.group.Insert(new Group { GroupId = 20, Title = \"Dev\" });\n\n ctx.Db.membership.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\n ctx.Db.membership.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\n ctx.Db.membership.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\n }\n}",
+ "category": "schema",
+ "route_api_model": "deepseek-reasoner",
+ "golden_db": "schema-t-019-many-to-many-golden",
+ "llm_db": "schema-t-019-many-to-many-deepseek-r1-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\deepseek-r1\\llm",
+ "scorer_details": {
+ "publish_error": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\deepseek-r1\\llm\\Lib.cs(34,16): error CS1061: 'Local' does not contain a definition for 'user' and no accessible extension method 'user' accepting a first argument of type 'Local' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\deepseek-r1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\deepseek-r1\\llm\\Lib.cs(35,16): error CS1061: 'Local' does not contain a definition for 'user' and no accessible extension method 'user' accepting a first argument of type 'Local' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\deepseek-r1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\deepseek-r1\\llm\\Lib.cs(37,16): error CS1061: 'Local' does not contain a definition for 'group' and no accessible extension method 'group' accepting a first argument of type 'Local' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\deepseek-r1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\deepseek-r1\\llm\\Lib.cs(38,16): error CS1061: 'Local' does not contain a definition for 'group' and no accessible extension method 'group' accepting a first argument of type 'Local' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\deepseek-r1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\deepseek-r1\\llm\\Lib.cs(40,16): error CS1061: 'Local' does not contain a definition for 'membership' and no accessible extension method 'membership' accepting a first argument of type 'Local' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\deepseek-r1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\deepseek-r1\\llm\\Lib.cs(41,16): error CS1061: 'Local' does not contain a definition for 'membership' and no accessible extension method 'membership' accepting a first argument of type 'Local' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\deepseek-r1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\deepseek-r1\\llm\\Lib.cs(42,16): error CS1061: 'Local' does not contain a definition for 'membership' and no accessible extension method 'membership' accepting a first argument of type 'Local' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\deepseek-r1\\llm\\StdbModule.csproj]\r\n",
+ "phase": "build_or_publish"
+ }
+ }
+ },
+ "vendor": "deepseek",
+ "started_at": "2025-10-21T21:33:46.352762300Z",
+ "finished_at": "2025-10-21T21:34:40.791806900Z"
+ },
+ "t_020_ecs": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_020_ecs",
+ "lang": "csharp",
+ "golden_published": true,
+ "model_name": "DeepSeek R1",
+ "total_tests": 5,
+ "passed_tests": 2,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"entities\")]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n }\n\n [SpacetimeDB.Table(Name = \"positions\")]\n public partial struct Position\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"velocities\")]\n public partial struct Velocity\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int VX;\n public int VY;\n }\n\n [SpacetimeDB.Table(Name = \"next_positions\")]\n public partial struct NextPosition\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.entities.Insert(new Entity { Id = 1 });\n ctx.Db.positions.Insert(new Position { EntityId = 1, X = 0, Y = 0 });\n ctx.Db.velocities.Insert(new Velocity { EntityId = 1, VX = 1, VY = 0 });\n\n ctx.Db.entities.Insert(new Entity { Id = 2 });\n ctx.Db.positions.Insert(new Position { EntityId = 2, X = 10, Y = 0 });\n ctx.Db.velocities.Insert(new Velocity { EntityId = 2, VX = -2, VY = 3 });\n }\n\n [SpacetimeDB.Reducer]\n public static void Step(ReducerContext ctx)\n {\n foreach (var pos in ctx.Db.positions.Iter())\n {\n var vel = ctx.Db.velocities.EntityId.Find(pos.EntityId);\n if (vel.HasValue)\n {\n ctx.Db.next_positions.EntityId.Update(new NextPosition\n {\n EntityId = pos.EntityId,\n X = pos.X + vel.Value.VX,\n Y = pos.Y + vel.Value.VY\n });\n }\n }\n }\n}",
+ "category": "schema",
+ "route_api_model": "deepseek-reasoner",
+ "golden_db": "schema-t-020-ecs-golden",
+ "llm_db": "schema-t-020-ecs-deepseek-r1-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\deepseek-r1\\llm",
+ "scorer_details": {
+ "ecs_next_pos_entity1": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: SpacetimeDB.NoSuchRowException: The row was not found, e.g., in an update call\n at SpacetimeDB.Internal.FFI.CheckedStatus.Marshaller.ConvertToManaged(Errno )\n at SpacetimeDB.Internal.FFI.datastore_update_bsatn(TableId , IndexId , Span`1 , UInt32& )\n at SpacetimeDB.Internal.UniqueIndex`4[[SpacetimeDB.Internal.TableHandles.next_positions, StdbModule, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null],[Module.NextPosition, StdbModule, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null],[System.Int32, System.Private.CoreLib, Version=8.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e],[SpacetimeDB.BSATN.I32, SpacetimeDB.BSATN.Runtime, Version=1.5.0.0, Culture=neutral, PublicKeyToken=null]].DoUpdate(NextPosition )\n at SpacetimeDB.Internal.TableHandles.next_positions.EntityIdUniqueIndex.Update(NextPosition )\n at Module.Step(ReducerContext )\n at ModuleRegistration.Step.Invoke(BinaryReader , IReducerContext )\n at SpacetimeDB.Internal.Module.__call_reducer__(UInt32 id, UInt64 sender_0, UInt64 sender_1, UInt64 sender_2, UInt64 sender_3, UInt64 conn_id_0, UInt64 conn_id_1, Timestamp timestamp, BytesSource args, BytesSink error)\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:3000/v1/database/c200815c605205023ca92d57e9fb13bd132838935a208007ae3bff05885703a8/call/Step)\n",
+ "phase": "call_reducer"
+ }
+ },
+ "ecs_step_next_positions_count": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: SpacetimeDB.NoSuchRowException: The row was not found, e.g., in an update call\n at SpacetimeDB.Internal.FFI.CheckedStatus.Marshaller.ConvertToManaged(Errno )\n at SpacetimeDB.Internal.FFI.datastore_update_bsatn(TableId , IndexId , Span`1 , UInt32& )\n at SpacetimeDB.Internal.UniqueIndex`4[[SpacetimeDB.Internal.TableHandles.next_positions, StdbModule, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null],[Module.NextPosition, StdbModule, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null],[System.Int32, System.Private.CoreLib, Version=8.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e],[SpacetimeDB.BSATN.I32, SpacetimeDB.BSATN.Runtime, Version=1.5.0.0, Culture=neutral, PublicKeyToken=null]].DoUpdate(NextPosition )\n at SpacetimeDB.Internal.TableHandles.next_positions.EntityIdUniqueIndex.Update(NextPosition )\n at Module.Step(ReducerContext )\n at ModuleRegistration.Step.Invoke(BinaryReader , IReducerContext )\n at SpacetimeDB.Internal.Module.__call_reducer__(UInt32 id, UInt64 sender_0, UInt64 sender_1, UInt64 sender_2, UInt64 sender_3, UInt64 conn_id_0, UInt64 conn_id_1, Timestamp timestamp, BytesSource args, BytesSink error)\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:3000/v1/database/c200815c605205023ca92d57e9fb13bd132838935a208007ae3bff05885703a8/call/Step)\n",
+ "phase": "call_reducer"
}
},
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-011-helper-function-golden",
- "llm_db": "basics-t-011-helper-function-deepseek-v3-llm",
+ "golden_db": "schema-t-020-ecs-golden",
+ "llm_db": "schema-t-020-ecs-deepseek-r1-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -12839,187 +12300,221 @@
"tables_equal": true
}
},
- "helper_func_sum_parity": {
+ "ecs_next_pos_entity2": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: SpacetimeDB.NoSuchRowException: The row was not found, e.g., in an update call\n at SpacetimeDB.Internal.FFI.CheckedStatus.Marshaller.ConvertToManaged(Errno )\n at SpacetimeDB.Internal.FFI.datastore_update_bsatn(TableId , IndexId , Span`1 , UInt32& )\n at SpacetimeDB.Internal.UniqueIndex`4[[SpacetimeDB.Internal.TableHandles.next_positions, StdbModule, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null],[Module.NextPosition, StdbModule, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null],[System.Int32, System.Private.CoreLib, Version=8.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e],[SpacetimeDB.BSATN.I32, SpacetimeDB.BSATN.Runtime, Version=1.5.0.0, Culture=neutral, PublicKeyToken=null]].DoUpdate(NextPosition )\n at SpacetimeDB.Internal.TableHandles.next_positions.EntityIdUniqueIndex.Update(NextPosition )\n at Module.Step(ReducerContext )\n at ModuleRegistration.Step.Invoke(BinaryReader , IReducerContext )\n at SpacetimeDB.Internal.Module.__call_reducer__(UInt32 id, UInt64 sender_0, UInt64 sender_1, UInt64 sender_2, UInt64 sender_3, UInt64 conn_id_0, UInt64 conn_id_1, Timestamp timestamp, BytesSource args, BytesSink error)\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:3000/v1/database/c200815c605205023ca92d57e9fb13bd132838935a208007ae3bff05885703a8/call/Step)\n",
+ "phase": "call_reducer"
+ }
+ },
+ "ecs_seed_positions_count": {
"pass": true,
"partial": 1.0,
"notes": {
- "args": [
- 1,
- 2,
- 3
- ],
- "golden_db": "basics-t-011-helper-function-golden",
- "golden_out": "Id | Sum ----+----- 1 | 5",
- "llm_db": "basics-t-011-helper-function-deepseek-v3-llm",
- "llm_out": "Id | Sum ----+----- 1 | 5",
- "query": "SELECT Id, Sum FROM results WHERE Id=1",
- "reducer": "ComputeSum",
- "server": "local"
+ "actual": 2,
+ "expected": 2,
+ "sql": "SELECT COUNT(*) AS n FROM positions"
}
}
},
"vendor": "deepseek",
- "started_at": "2025-10-21T21:43:20.988640Z",
- "finished_at": "2025-10-21T21:44:50.833236100Z"
+ "started_at": "2025-10-21T21:33:46.869823300Z",
+ "finished_at": "2025-10-21T21:35:44.603745Z"
},
- "t_009_init": {
+ "t_021_multi_column_index": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_009_init",
+ "task": "t_021_multi_column_index",
"lang": "csharp",
"golden_published": true,
- "model_name": "DeepSeek V3",
+ "model_name": "DeepSeek R1",
"total_tests": 4,
- "passed_tests": 4,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.users.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.users.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n }\n}",
- "category": "basics",
- "route_api_model": "deepseek-chat",
- "golden_db": "basics-t-009-init-golden",
- "llm_db": "basics-t-009-init-deepseek-v3-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_009_init\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_009_init\\csharp\\server\\deepseek-v3\\llm",
+ "passed_tests": 0,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"logs\")]\n public partial struct Log\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int Day;\n public string Message;\n }\n\n [SpacetimeDB.Index.BTree(Name = \"by_user_day\", Columns = [nameof(UserId), nameof(Day)])]\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.logs.Insert(new Log { Id = 1, UserId = 7, Day = 1, Message = \"a\" });\n ctx.Db.logs.Insert(new Log { Id = 2, UserId = 7, Day = 2, Message = \"b\" });\n ctx.Db.logs.Insert(new Log { Id = 3, UserId = 9, Day = 1, Message = \"c\" });\n }\n}",
+ "category": "schema",
+ "route_api_model": "deepseek-reasoner",
+ "golden_db": "schema-t-021-multi-column-index-golden",
+ "llm_db": "schema-t-021-multi-column-index-deepseek-r1-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\deepseek-r1\\llm",
"scorer_details": {
- "init_seed_bob": {
- "pass": true,
- "partial": 1.0,
+ "publish_error": {
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false"
+ "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\deepseek-r1\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'logs' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\deepseek-r1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\deepseek-r1\\llm\\Lib.cs(16,70): error CS0103: The name 'UserId' does not exist in the current context [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\deepseek-r1\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\deepseek-r1\\llm\\Lib.cs(16,86): error CS0103: The name 'Day' does not exist in the current context [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\deepseek-r1\\llm\\StdbModule.csproj]\r\n",
+ "phase": "build_or_publish"
}
- },
+ }
+ },
+ "vendor": "deepseek",
+ "started_at": "2025-10-21T21:33:47.372781600Z",
+ "finished_at": "2025-10-21T21:34:32.333574Z"
+ }
+ }
+ },
+ {
+ "name": "DeepSeek V3",
+ "route_api_model": "deepseek-chat",
+ "tasks": {
+ "t_000_empty_reducers": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_000_empty_reducers",
+ "lang": "csharp",
+ "golden_published": true,
+ "model_name": "DeepSeek V3",
+ "total_tests": 1,
+ "passed_tests": 1,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_NoArgs(ReducerContext ctx)\n {\n }\n\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_WithInt(ReducerContext ctx, int count)\n {\n }\n\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_WithString(ReducerContext ctx, string name)\n {\n }\n\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_WithTwoArgs(ReducerContext ctx, int count, string name)\n {\n }\n\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_WithThreeArgs(ReducerContext ctx, bool active, float ratio, string label)\n {\n }\n}",
+ "category": "basics",
+ "route_api_model": "deepseek-chat",
+ "golden_db": "basics-t-000-empty-reducers-golden",
+ "llm_db": "basics-t-000-empty-reducers-deepseek-v3-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\deepseek-v3\\llm",
+ "scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-009-init-golden",
- "llm_db": "basics-t-009-init-deepseek-v3-llm",
+ "golden_db": "basics-t-000-empty-reducers-golden",
+ "llm_db": "basics-t-000-empty-reducers-deepseek-v3-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
"tables_diff": null,
"tables_equal": true
}
- },
- "init_total_two": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 2,
- "expected": 2,
- "sql": "SELECT COUNT(*) AS n FROM users"
- }
- },
- "init_seed_alice": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true"
- }
}
},
"vendor": "deepseek",
- "started_at": "2025-10-21T21:43:19.982065600Z",
- "finished_at": "2025-10-21T21:44:58.956936700Z"
+ "started_at": "2025-10-21T21:43:15.268555100Z",
+ "finished_at": "2025-10-21T21:44:58.653818500Z"
},
- "t_007_crud": {
+ "t_001_basic_tables": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_007_crud",
+ "task": "t_001_basic_tables",
"lang": "csharp",
"golden_published": true,
"model_name": "DeepSeek V3",
- "total_tests": 4,
- "passed_tests": 4,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void Crud(ReducerContext ctx)\n {\n ctx.Db.users.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.users.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n \n var user1 = ctx.Db.users.Id.Find(1);\n if (user1 != null)\n {\n ctx.Db.users.Id.Update(new User { Id = 1, Name = \"Alice2\", Age = 31, Active = false });\n }\n \n ctx.Db.users.Id.Delete(2);\n }\n}",
+ "total_tests": 1,
+ "passed_tests": 1,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\", Public = true)]\n public partial struct Users\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"products\", Public = true)]\n public partial struct Products\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Title;\n public float Price;\n public bool InStock;\n }\n\n [SpacetimeDB.Table(Name = \"notes\", Public = true)]\n public partial struct Notes\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Body;\n public long Rating;\n public bool Pinned;\n }\n}",
"category": "basics",
"route_api_model": "deepseek-chat",
- "golden_db": "basics-t-007-crud-golden",
- "llm_db": "basics-t-007-crud-deepseek-v3-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\deepseek-v3\\llm",
+ "golden_db": "basics-t-001-basic-tables-golden",
+ "llm_db": "basics-t-001-basic-tables-deepseek-v3-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_001_basic_tables\\csharp\\server\\deepseek-v3\\llm",
"scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-007-crud-golden",
- "llm_db": "basics-t-007-crud-deepseek-v3-llm",
+ "golden_db": "basics-t-001-basic-tables-golden",
+ "llm_db": "basics-t-001-basic-tables-deepseek-v3-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
"tables_diff": null,
"tables_equal": true
}
- },
- "crud_total_count_one": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM users"
- }
- },
- "crud_row_id2_deleted": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 0,
- "expected": 0,
- "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=2"
- }
- },
- "crud_row_id1_parity": {
+ }
+ },
+ "vendor": "deepseek",
+ "started_at": "2025-10-21T21:43:15.819316200Z",
+ "finished_at": "2025-10-21T21:44:59.876292400Z"
+ },
+ "t_002_scheduled_table": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_002_scheduled_table",
+ "lang": "csharp",
+ "golden_published": true,
+ "model_name": "DeepSeek V3",
+ "total_tests": 1,
+ "passed_tests": 1,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"tick_timer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public ulong ScheduledId;\n\n public ScheduleAt ScheduledAt;\n }\n\n [SpacetimeDB.Reducer]\n public static void Tick(ReducerContext ctx, TickTimer schedule)\n {\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.tick_timer.Insert(new TickTimer\n {\n ScheduledId = 0,\n ScheduledAt = new ScheduleAt.Interval(new TimeDuration { Microseconds = 50_000 })\n });\n }\n}",
+ "category": "basics",
+ "route_api_model": "deepseek-chat",
+ "golden_db": "basics-t-002-scheduled-table-golden",
+ "llm_db": "basics-t-002-scheduled-table-deepseek-v3-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_002_scheduled_table\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_002_scheduled_table\\csharp\\server\\deepseek-v3\\llm",
+ "scorer_details": {
+ "schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "args": [],
- "golden_db": "basics-t-007-crud-golden",
- "golden_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
- "llm_db": "basics-t-007-crud-deepseek-v3-llm",
- "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
- "query": "SELECT Id, Name, Age, Active FROM users WHERE Id=1",
- "reducer": "Crud",
- "server": "local"
+ "golden_db": "basics-t-002-scheduled-table-golden",
+ "llm_db": "basics-t-002-scheduled-table-deepseek-v3-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
}
}
},
"vendor": "deepseek",
- "started_at": "2025-10-21T21:43:18.974162900Z",
- "finished_at": "2025-10-21T21:44:58.489258700Z"
+ "started_at": "2025-10-21T21:43:16.350094900Z",
+ "finished_at": "2025-10-21T21:45:00.415735Z"
},
- "t_006_delete": {
+ "t_003_struct_in_table": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_006_delete",
+ "task": "t_003_struct_in_table",
"lang": "csharp",
"golden_published": true,
"model_name": "DeepSeek V3",
- "total_tests": 3,
- "passed_tests": 3,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void DeleteUser(ReducerContext ctx, int id)\n {\n if (ctx.Db.users.Id.Find(id) is User user)\n {\n ctx.Db.users.Id.Delete(id);\n }\n }\n}",
+ "total_tests": 1,
+ "passed_tests": 1,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"entities\")]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Position Pos;\n }\n}",
"category": "basics",
"route_api_model": "deepseek-chat",
- "golden_db": "basics-t-006-delete-golden",
- "llm_db": "basics-t-006-delete-deepseek-v3-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\deepseek-v3\\llm",
+ "golden_db": "basics-t-003-struct-in-table-golden",
+ "llm_db": "basics-t-003-struct-in-table-deepseek-v3-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_003_struct_in_table\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_003_struct_in_table\\csharp\\server\\deepseek-v3\\llm",
"scorer_details": {
- "delete_user_count_zero": {
+ "schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 0,
- "expected": 0,
- "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=1"
+ "golden_db": "basics-t-003-struct-in-table-golden",
+ "llm_db": "basics-t-003-struct-in-table-deepseek-v3-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
}
- },
+ }
+ },
+ "vendor": "deepseek",
+ "started_at": "2025-10-21T21:43:16.874441900Z",
+ "finished_at": "2025-10-21T21:44:50.111591900Z"
+ },
+ "t_004_insert": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_004_insert",
+ "lang": "csharp",
+ "golden_published": true,
+ "model_name": "DeepSeek V3",
+ "total_tests": 2,
+ "passed_tests": 2,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void InsertUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n ctx.Db.users.Insert(new User { Id = id, Name = name, Age = age, Active = active });\n }\n}",
+ "category": "basics",
+ "route_api_model": "deepseek-chat",
+ "golden_db": "basics-t-004-insert-golden",
+ "llm_db": "basics-t-004-insert-deepseek-v3-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_004_insert\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_004_insert\\csharp\\server\\deepseek-v3\\llm",
+ "scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-006-delete-golden",
- "llm_db": "basics-t-006-delete-deepseek-v3-llm",
+ "golden_db": "basics-t-004-insert-golden",
+ "llm_db": "basics-t-004-insert-deepseek-v3-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -13027,17 +12522,29 @@
"tables_equal": true
}
},
- "seed_users_row": {
+ "data_parity_insert_user": {
"pass": true,
"partial": 1.0,
"notes": {
- "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
+ "args": [
+ 1,
+ "Alice",
+ 30,
+ true
+ ],
+ "golden_db": "basics-t-004-insert-golden",
+ "golden_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true",
+ "llm_db": "basics-t-004-insert-deepseek-v3-llm",
+ "llm_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true",
+ "query": "SELECT Id, Name, Age, Active FROM users WHERE Id=1",
+ "reducer": "InsertUser",
+ "server": "local"
}
}
},
"vendor": "deepseek",
- "started_at": "2025-10-21T21:43:18.439088600Z",
- "finished_at": "2025-10-21T21:44:51.489757900Z"
+ "started_at": "2025-10-21T21:43:17.407052700Z",
+ "finished_at": "2025-10-21T21:45:00.254993100Z"
},
"t_005_update": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
@@ -13055,13 +12562,6 @@
"work_dir_golden": "target\\llm-runs\\basics\\t_005_update\\csharp\\server\\golden",
"work_dir_llm": "target\\llm-runs\\basics\\t_005_update\\csharp\\server\\deepseek-v3\\llm",
"scorer_details": {
- "seed_users_row": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
- }
- },
"schema_parity": {
"pass": true,
"partial": 1.0,
@@ -13093,97 +12593,100 @@
"reducer": "UpdateUser",
"server": "local"
}
+ },
+ "seed_users_row": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
+ }
}
},
"vendor": "deepseek",
"started_at": "2025-10-21T21:43:17.917737100Z",
"finished_at": "2025-10-21T21:44:59.712059200Z"
},
- "t_010_connect": {
+ "t_006_delete": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_010_connect",
+ "task": "t_006_delete",
"lang": "csharp",
"golden_published": true,
"model_name": "DeepSeek V3",
- "total_tests": 1,
- "passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"events\")]\n public partial struct Event\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public int Id;\n public string Kind;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientConnected)]\n public static void ClientConnected(ReducerContext ctx)\n {\n ctx.Db.events.Insert(new Event { Id = 0, Kind = \"connected\" });\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientDisconnected)]\n public static void ClientDisconnected(ReducerContext ctx)\n {\n ctx.Db.events.Insert(new Event { Id = 0, Kind = \"disconnected\" });\n }\n}",
+ "total_tests": 3,
+ "passed_tests": 3,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void DeleteUser(ReducerContext ctx, int id)\n {\n if (ctx.Db.users.Id.Find(id) is User user)\n {\n ctx.Db.users.Id.Delete(id);\n }\n }\n}",
"category": "basics",
"route_api_model": "deepseek-chat",
- "golden_db": "basics-t-010-connect-golden",
- "llm_db": "basics-t-010-connect-deepseek-v3-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_010_connect\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_010_connect\\csharp\\server\\deepseek-v3\\llm",
+ "golden_db": "basics-t-006-delete-golden",
+ "llm_db": "basics-t-006-delete-deepseek-v3-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\deepseek-v3\\llm",
"scorer_details": {
+ "seed_users_row": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
+ }
+ },
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-010-connect-golden",
- "llm_db": "basics-t-010-connect-deepseek-v3-llm",
+ "golden_db": "basics-t-006-delete-golden",
+ "llm_db": "basics-t-006-delete-deepseek-v3-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
"tables_diff": null,
"tables_equal": true
}
- }
- },
- "vendor": "deepseek",
- "started_at": "2025-10-21T21:43:20.488470300Z",
- "finished_at": "2025-10-21T21:44:59.087263200Z"
- },
- "t_020_ecs": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_020_ecs",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "DeepSeek V3",
- "total_tests": 5,
- "passed_tests": 0,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"entities\")]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n }\n\n [SpacetimeDB.Table(Name = \"positions\")]\n public partial struct Position\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"velocities\")]\n public partial struct Velocity\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int VX;\n public int VY;\n }\n\n [SpacetimeDB.Table(Name = \"next_positions\")]\n public partial struct NextPosition\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.entities.Insert(new Entity { Id = 1 });\n ctx.Db.positions.Insert(new Position { EntityId = 1, X = 0, Y = 0 });\n ctx.Db.velocities.Insert(new Velocity { EntityId = 1, VX = 1, VY = 0 });\n\n ctx.Db.entities.Insert(new Entity { Id = 2 });\n ctx.Db.positions.Insert(new Position { EntityId = 2, X = 10, Y = 0 });\n ctx.Db.velocities.Insert(new Velocity { EntityId = 2, VX = -2, VY = 3 });\n }\n\n [SpacetimeDB.Reducer]\n public static void Step(ReducerContext ctx)\n {\n foreach (var pos in ctx.Db.positions.Iter())\n {\n var vel = ctx.Db.velocities.EntityId.Find(pos.EntityId);\n if (vel != null)\n {\n var nextPos = new NextPosition\n {\n EntityId = pos.EntityId,\n X = pos.X + vel.VX,\n Y = pos.Y + vel.VY\n };\n ctx.Db.next_positions.EntityId.Update(nextPos);\n }\n }\n }\n}",
- "category": "schema",
- "route_api_model": "deepseek-chat",
- "golden_db": "schema-t-020-ecs-golden",
- "llm_db": "schema-t-020-ecs-deepseek-v3-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\deepseek-v3\\llm",
- "scorer_details": {
- "publish_error": {
- "pass": false,
- "partial": 0.0,
+ },
+ "delete_user_count_zero": {
+ "pass": true,
+ "partial": 1.0,
"notes": {
- "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\deepseek-v3\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'entities' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\deepseek-v3\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\deepseek-v3\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(113,24): warning CS8981: The type name 'positions' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\deepseek-v3\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\deepseek-v3\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(156,24): warning CS8981: The type name 'velocities' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\deepseek-v3\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\deepseek-v3\\llm\\Lib.cs(63,37): error CS1061: 'Module.Velocity?' does not contain a definition for 'VX' and no accessible extension method 'VX' accepting a first argument of type 'Module.Velocity?' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\deepseek-v3\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\deepseek-v3\\llm\\Lib.cs(64,37): error CS1061: 'Module.Velocity?' does not contain a definition for 'VY' and no accessible extension method 'VY' accepting a first argument of type 'Module.Velocity?' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\deepseek-v3\\llm\\StdbModule.csproj]\r\n",
- "phase": "build_or_publish"
+ "actual": 0,
+ "expected": 0,
+ "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=1"
}
}
},
"vendor": "deepseek",
- "started_at": "2025-10-21T21:47:29.171789600Z",
- "finished_at": "2025-10-21T21:47:53.751440300Z"
+ "started_at": "2025-10-21T21:43:18.439088600Z",
+ "finished_at": "2025-10-21T21:44:51.489757900Z"
},
- "t_016_sum_type_columns": {
+ "t_007_crud": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_016_sum_type_columns",
+ "task": "t_007_crud",
"lang": "csharp",
"golden_published": true,
"model_name": "DeepSeek V3",
- "total_tests": 3,
- "passed_tests": 3,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"drawings\")]\n public partial struct Drawing\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape A;\n public Shape B;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.drawings.Insert(new Drawing\n {\n Id = 1,\n A = new Shape.Circle(new Circle { Radius = 10 }),\n B = new Shape.Rectangle(new Rectangle { Width = 4, Height = 6 })\n });\n }\n}",
- "category": "schema",
+ "total_tests": 4,
+ "passed_tests": 4,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void Crud(ReducerContext ctx)\n {\n ctx.Db.users.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.users.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n \n var user1 = ctx.Db.users.Id.Find(1);\n if (user1 != null)\n {\n ctx.Db.users.Id.Update(new User { Id = 1, Name = \"Alice2\", Age = 31, Active = false });\n }\n \n ctx.Db.users.Id.Delete(2);\n }\n}",
+ "category": "basics",
"route_api_model": "deepseek-chat",
- "golden_db": "schema-t-016-sum-type-columns-golden",
- "llm_db": "schema-t-016-sum-type-columns-deepseek-v3-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\deepseek-v3\\llm",
+ "golden_db": "basics-t-007-crud-golden",
+ "llm_db": "basics-t-007-crud-deepseek-v3-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\deepseek-v3\\llm",
"scorer_details": {
+ "crud_row_id2_deleted": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 0,
+ "expected": 0,
+ "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=2"
+ }
+ },
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-016-sum-type-columns-golden",
- "llm_db": "schema-t-016-sum-type-columns-deepseek-v3-llm",
+ "golden_db": "basics-t-007-crud-golden",
+ "llm_db": "basics-t-007-crud-deepseek-v3-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -13191,168 +12694,146 @@
"tables_equal": true
}
},
- "sum_type_columns_row_count": {
+ "crud_total_count_one": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM drawings WHERE Id=1"
+ "sql": "SELECT COUNT(*) AS n FROM users"
}
},
- "sum_type_columns_row_parity": {
+ "crud_row_id1_parity": {
"pass": true,
"partial": 1.0,
"notes": {
"args": [],
- "golden_db": "schema-t-016-sum-type-columns-golden",
- "golden_out": "Id | A | B ----+--------------------------+--------------------------------------- 1 | (Circle = (Radius = 10)) | (Rectangle = (Width = 4, Height = 6))",
- "llm_db": "schema-t-016-sum-type-columns-deepseek-v3-llm",
- "llm_out": "Id | A | B ----+--------------------------+--------------------------------------- 1 | (Circle = (Radius = 10)) | (Rectangle = (Width = 4, Height = 6))",
- "query": "SELECT Id, A, B FROM drawings WHERE Id=1",
- "reducer": "Seed",
+ "golden_db": "basics-t-007-crud-golden",
+ "golden_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
+ "llm_db": "basics-t-007-crud-deepseek-v3-llm",
+ "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
+ "query": "SELECT Id, Name, Age, Active FROM users WHERE Id=1",
+ "reducer": "Crud",
"server": "local"
}
}
},
"vendor": "deepseek",
- "started_at": "2025-10-21T21:47:26.781535900Z",
- "finished_at": "2025-10-21T21:49:07.572308Z"
+ "started_at": "2025-10-21T21:43:18.974162900Z",
+ "finished_at": "2025-10-21T21:44:58.489258700Z"
},
- "t_018_constraints": {
+ "t_008_index_lookup": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_018_constraints",
+ "task": "t_008_index_lookup",
"lang": "csharp",
"golden_published": true,
"model_name": "DeepSeek V3",
"total_tests": 3,
- "passed_tests": 2,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"accounts\", Public = true)]\n [SpacetimeDB.Index.BTree(Name = \"by_name\", Columns = [nameof(Name)])]\n public partial struct Account\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n [SpacetimeDB.Unique]\n public string Email;\n public string Name;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.accounts.Insert(new Account { Id = 1, Email = \"a@example.com\", Name = \"Alice\" });\n ctx.Db.accounts.Insert(new Account { Id = 2, Email = \"b@example.com\", Name = \"Bob\" });\n }\n}",
- "category": "schema",
+ "passed_tests": 0,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"results\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void LookupUserName(ReducerContext ctx, int id)\n {\n var user = ctx.Db.users.Id.Find(id);\n if (user != null)\n {\n ctx.Db.results.Insert(new Result { Id = user.Id, Name = user.Name });\n }\n }\n}",
+ "category": "basics",
"route_api_model": "deepseek-chat",
- "golden_db": "schema-t-018-constraints-golden",
- "llm_db": "schema-t-018-constraints-deepseek-v3-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\deepseek-v3\\llm",
+ "golden_db": "basics-t-008-index-lookup-golden",
+ "llm_db": "basics-t-008-index-lookup-deepseek-v3-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\deepseek-v3\\llm",
"scorer_details": {
- "constraints_row_parity_after_seed": {
+ "publish_error": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: can't directly call special Init lifecycle reducer\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:3000/v1/database/c20079919ed90a64ecacb120b3bdd78cba0f54c43555b600e7f7c763eb2d1274/call/Seed)\n",
- "phase": "call_reducer_llm"
- }
- },
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "golden_db": "schema-t-018-constraints-golden",
- "llm_db": "schema-t-018-constraints-deepseek-v3-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
- }
- },
- "constraints_seed_two_rows": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM accounts WHERE Id=2"
+ "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\deepseek-v3\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(70,24): warning CS8981: The type name 'users' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\deepseek-v3\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\deepseek-v3\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'results' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\deepseek-v3\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\deepseek-v3\\llm\\Lib.cs(30,58): error CS1061: 'Module.User?' does not contain a definition for 'Id' and no accessible extension method 'Id' accepting a first argument of type 'Module.User?' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\deepseek-v3\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\deepseek-v3\\llm\\Lib.cs(30,74): error CS1061: 'Module.User?' does not contain a definition for 'Name' and no accessible extension method 'Name' accepting a first argument of type 'Module.User?' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\deepseek-v3\\llm\\StdbModule.csproj]\r\n",
+ "phase": "build_or_publish"
}
}
},
"vendor": "deepseek",
- "started_at": "2025-10-21T21:47:27.983142300Z",
- "finished_at": "2025-10-21T21:49:08.019543700Z"
+ "started_at": "2025-10-21T21:43:19.478413200Z",
+ "finished_at": "2025-10-21T21:43:35.441778500Z"
},
- "t_021_multi_column_index": {
+ "t_009_init": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_021_multi_column_index",
+ "task": "t_009_init",
"lang": "csharp",
"golden_published": true,
"model_name": "DeepSeek V3",
"total_tests": 4,
"passed_tests": 4,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"logs\")]\n [SpacetimeDB.Index.BTree(Name = \"by_user_day\", Columns = [nameof(UserId), nameof(Day)])]\n public partial struct Log\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int Day;\n public string Message;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.logs.Insert(new Log { Id = 1, UserId = 7, Day = 1, Message = \"a\" });\n ctx.Db.logs.Insert(new Log { Id = 2, UserId = 7, Day = 2, Message = \"b\" });\n ctx.Db.logs.Insert(new Log { Id = 3, UserId = 9, Day = 1, Message = \"c\" });\n }\n}",
- "category": "schema",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.users.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.users.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n }\n}",
+ "category": "basics",
"route_api_model": "deepseek-chat",
- "golden_db": "schema-t-021-multi-column-index-golden",
- "llm_db": "schema-t-021-multi-column-index-deepseek-v3-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\deepseek-v3\\llm",
+ "golden_db": "basics-t-009-init-golden",
+ "llm_db": "basics-t-009-init-deepseek-v3-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_009_init\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_009_init\\csharp\\server\\deepseek-v3\\llm",
"scorer_details": {
- "mcindex_lookup_u7_d2": {
+ "init_total_two": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 2,
+ "expected": 2,
+ "sql": "SELECT COUNT(*) AS n FROM users"
+ }
+ },
+ "init_seed_bob": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM logs WHERE UserId=7 AND Day=2"
+ "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false"
}
},
- "mcindex_lookup_u7_d1": {
+ "init_seed_alice": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM logs WHERE UserId=7 AND Day=1"
+ "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true"
}
},
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-021-multi-column-index-golden",
- "llm_db": "schema-t-021-multi-column-index-deepseek-v3-llm",
+ "golden_db": "basics-t-009-init-golden",
+ "llm_db": "basics-t-009-init-deepseek-v3-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
"tables_diff": null,
"tables_equal": true
}
- },
- "mcindex_seed_count": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 3,
- "expected": 3,
- "sql": "SELECT COUNT(*) AS n FROM logs"
- }
}
},
"vendor": "deepseek",
- "started_at": "2025-10-21T21:47:29.875295400Z",
- "finished_at": "2025-10-21T21:49:09.219649600Z"
+ "started_at": "2025-10-21T21:43:19.982065600Z",
+ "finished_at": "2025-10-21T21:44:58.956936700Z"
},
- "t_002_scheduled_table": {
+ "t_010_connect": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_002_scheduled_table",
+ "task": "t_010_connect",
"lang": "csharp",
"golden_published": true,
"model_name": "DeepSeek V3",
"total_tests": 1,
"passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"tick_timer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public ulong ScheduledId;\n\n public ScheduleAt ScheduledAt;\n }\n\n [SpacetimeDB.Reducer]\n public static void Tick(ReducerContext ctx, TickTimer schedule)\n {\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.tick_timer.Insert(new TickTimer\n {\n ScheduledId = 0,\n ScheduledAt = new ScheduleAt.Interval(new TimeDuration { Microseconds = 50_000 })\n });\n }\n}",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"events\")]\n public partial struct Event\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public int Id;\n public string Kind;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientConnected)]\n public static void ClientConnected(ReducerContext ctx)\n {\n ctx.Db.events.Insert(new Event { Id = 0, Kind = \"connected\" });\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientDisconnected)]\n public static void ClientDisconnected(ReducerContext ctx)\n {\n ctx.Db.events.Insert(new Event { Id = 0, Kind = \"disconnected\" });\n }\n}",
"category": "basics",
"route_api_model": "deepseek-chat",
- "golden_db": "basics-t-002-scheduled-table-golden",
- "llm_db": "basics-t-002-scheduled-table-deepseek-v3-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_002_scheduled_table\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_002_scheduled_table\\csharp\\server\\deepseek-v3\\llm",
+ "golden_db": "basics-t-010-connect-golden",
+ "llm_db": "basics-t-010-connect-deepseek-v3-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_010_connect\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_010_connect\\csharp\\server\\deepseek-v3\\llm",
"scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-002-scheduled-table-golden",
- "llm_db": "basics-t-002-scheduled-table-deepseek-v3-llm",
+ "golden_db": "basics-t-010-connect-golden",
+ "llm_db": "basics-t-010-connect-deepseek-v3-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -13362,31 +12843,58 @@
}
},
"vendor": "deepseek",
- "started_at": "2025-10-21T21:43:16.350094900Z",
- "finished_at": "2025-10-21T21:45:00.415735Z"
+ "started_at": "2025-10-21T21:43:20.488470300Z",
+ "finished_at": "2025-10-21T21:44:59.087263200Z"
},
- "t_003_struct_in_table": {
+ "t_011_helper_function": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_003_struct_in_table",
+ "task": "t_011_helper_function",
"lang": "csharp",
"golden_published": true,
"model_name": "DeepSeek V3",
- "total_tests": 1,
- "passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"entities\")]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Position Pos;\n }\n}",
+ "total_tests": 3,
+ "passed_tests": 3,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"results\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Sum;\n }\n\n private static int Add(int a, int b)\n {\n return a + b;\n }\n\n [SpacetimeDB.Reducer]\n public static void ComputeSum(ReducerContext ctx, int id, int a, int b)\n {\n ctx.Db.results.Insert(new Result { Id = id, Sum = Add(a, b) });\n }\n}",
"category": "basics",
"route_api_model": "deepseek-chat",
- "golden_db": "basics-t-003-struct-in-table-golden",
- "llm_db": "basics-t-003-struct-in-table-deepseek-v3-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_003_struct_in_table\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_003_struct_in_table\\csharp\\server\\deepseek-v3\\llm",
+ "golden_db": "basics-t-011-helper-function-golden",
+ "llm_db": "basics-t-011-helper-function-deepseek-v3-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\deepseek-v3\\llm",
"scorer_details": {
+ "helper_func_sum_abs": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1 AND Sum=5"
+ }
+ },
+ "helper_func_sum_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "args": [
+ 1,
+ 2,
+ 3
+ ],
+ "golden_db": "basics-t-011-helper-function-golden",
+ "golden_out": "Id | Sum ----+----- 1 | 5",
+ "llm_db": "basics-t-011-helper-function-deepseek-v3-llm",
+ "llm_out": "Id | Sum ----+----- 1 | 5",
+ "query": "SELECT Id, Sum FROM results WHERE Id=1",
+ "reducer": "ComputeSum",
+ "server": "local"
+ }
+ },
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-003-struct-in-table-golden",
- "llm_db": "basics-t-003-struct-in-table-deepseek-v3-llm",
+ "golden_db": "basics-t-011-helper-function-golden",
+ "llm_db": "basics-t-011-helper-function-deepseek-v3-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -13396,90 +12904,109 @@
}
},
"vendor": "deepseek",
- "started_at": "2025-10-21T21:43:16.874441900Z",
- "finished_at": "2025-10-21T21:44:50.111591900Z"
+ "started_at": "2025-10-21T21:43:20.988640Z",
+ "finished_at": "2025-10-21T21:44:50.833236100Z"
},
- "t_004_insert": {
+ "t_012_spacetime_product_type": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_004_insert",
+ "task": "t_012_spacetime_product_type",
"lang": "csharp",
"golden_published": true,
"model_name": "DeepSeek V3",
- "total_tests": 2,
- "passed_tests": 2,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void InsertUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n ctx.Db.users.Insert(new User { Id = id, Name = name, Age = age, Active = active });\n }\n}",
- "category": "basics",
+ "total_tests": 3,
+ "passed_tests": 3,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Score\n {\n public int Left;\n public int Right;\n }\n\n [SpacetimeDB.Table(Name = \"results\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Score Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetScore(ReducerContext ctx, int id, int left, int right)\n {\n ctx.Db.results.Insert(new Result\n {\n Id = id,\n Value = new Score { Left = left, Right = right }\n });\n }\n}",
+ "category": "schema",
"route_api_model": "deepseek-chat",
- "golden_db": "basics-t-004-insert-golden",
- "llm_db": "basics-t-004-insert-deepseek-v3-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_004_insert\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_004_insert\\csharp\\server\\deepseek-v3\\llm",
+ "golden_db": "schema-t-012-spacetime-product-type-golden",
+ "llm_db": "schema-t-012-spacetime-product-type-deepseek-v3-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\deepseek-v3\\llm",
"scorer_details": {
- "data_parity_insert_user": {
+ "product_type_row_count": {
"pass": true,
"partial": 1.0,
"notes": {
- "args": [
- 1,
- "Alice",
- 30,
- true
- ],
- "golden_db": "basics-t-004-insert-golden",
- "golden_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true",
- "llm_db": "basics-t-004-insert-deepseek-v3-llm",
- "llm_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true",
- "query": "SELECT Id, Name, Age, Active FROM users WHERE Id=1",
- "reducer": "InsertUser",
- "server": "local"
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1"
}
},
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-004-insert-golden",
- "llm_db": "basics-t-004-insert-deepseek-v3-llm",
+ "golden_db": "schema-t-012-spacetime-product-type-golden",
+ "llm_db": "schema-t-012-spacetime-product-type-deepseek-v3-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
"tables_diff": null,
"tables_equal": true
}
+ },
+ "product_type_row_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "args": [
+ 1,
+ 2,
+ 3
+ ],
+ "golden_db": "schema-t-012-spacetime-product-type-golden",
+ "golden_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)",
+ "llm_db": "schema-t-012-spacetime-product-type-deepseek-v3-llm",
+ "llm_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)",
+ "query": "SELECT Id, Value FROM results WHERE Id=1",
+ "reducer": "SetScore",
+ "server": "local"
+ }
}
},
"vendor": "deepseek",
- "started_at": "2025-10-21T21:43:17.407052700Z",
- "finished_at": "2025-10-21T21:45:00.254993100Z"
- }
- }
- },
- {
- "name": "Meta Llama 3.1 405B",
- "route_api_model": "meta-llama/llama-3.1-405b-instruct",
- "tasks": {
- "t_017_scheduled_columns": {
+ "started_at": "2025-10-21T21:47:24.300697500Z",
+ "finished_at": "2025-10-21T21:48:58.320135Z"
+ },
+ "t_013_spacetime_sum_type": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_017_scheduled_columns",
+ "task": "t_013_spacetime_sum_type",
"lang": "csharp",
"golden_published": true,
- "model_name": "Meta Llama 3.1 405B",
- "total_tests": 2,
- "passed_tests": 2,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"tick_timer\", Scheduled = nameof(Tick), ScheduledAt = nameof(TickTimer.ScheduledAt))]\n public partial struct TickTimer\n {\n [PrimaryKey]\n [AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [Reducer]\n public static void Tick(ReducerContext ctx, TickTimer schedule)\n {\n Log.Info(\"Tick reducer called\");\n }\n\n [Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n var currentTime = ctx.Timestamp;\n var interval = new TimeDuration { Microseconds = +50_000 };\n var futureTimestamp = currentTime + interval;\n\n ctx.Db.tick_timer.Insert(new()\n {\n ScheduledId = 0, // Have [AutoInc] assign an Id.\n ScheduledAt = new ScheduleAt.Interval(interval),\n });\n }\n}",
+ "model_name": "DeepSeek V3",
+ "total_tests": 3,
+ "passed_tests": 3,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"results\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetCircle(ReducerContext ctx, int id, int radius)\n {\n ctx.Db.results.Insert(new Result\n {\n Id = id,\n Value = new Shape.Circle(new Circle { Radius = radius })\n });\n }\n}",
"category": "schema",
- "route_api_model": "meta-llama/llama-3.1-405b-instruct",
- "golden_db": "schema-t-017-scheduled-columns-golden",
- "llm_db": "schema-t-017-scheduled-columns-meta-llama-3-1-405b-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_017_scheduled_columns\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_017_scheduled_columns\\csharp\\server\\meta-llama-3-1-405b\\llm",
+ "route_api_model": "deepseek-chat",
+ "golden_db": "schema-t-013-spacetime-sum-type-golden",
+ "llm_db": "schema-t-013-spacetime-sum-type-deepseek-v3-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\deepseek-v3\\llm",
"scorer_details": {
+ "sum_type_row_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "args": [
+ 1,
+ 10
+ ],
+ "golden_db": "schema-t-013-spacetime-sum-type-golden",
+ "golden_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))",
+ "llm_db": "schema-t-013-spacetime-sum-type-deepseek-v3-llm",
+ "llm_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))",
+ "query": "SELECT Id, Value FROM results WHERE Id=1",
+ "reducer": "SetCircle",
+ "server": "local"
+ }
+ },
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-017-scheduled-columns-golden",
- "llm_db": "schema-t-017-scheduled-columns-meta-llama-3-1-405b-llm",
+ "golden_db": "schema-t-013-spacetime-sum-type-golden",
+ "llm_db": "schema-t-013-spacetime-sum-type-deepseek-v3-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -13487,186 +13014,156 @@
"tables_equal": true
}
},
- "scheduled_seeded_one_row": {
+ "sum_type_row_count": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM tick_timer WHERE ScheduledId>=0"
+ "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1"
}
}
},
- "vendor": "meta",
- "started_at": "2025-10-22T02:37:51.377249200Z",
- "finished_at": "2025-10-22T02:40:16.461659400Z"
+ "vendor": "deepseek",
+ "started_at": "2025-10-21T21:47:24.962875700Z",
+ "finished_at": "2025-10-21T21:49:10.062838600Z"
},
- "t_015_product_type_columns": {
+ "t_014_elementary_columns": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_015_product_type_columns",
+ "task": "t_014_elementary_columns",
"lang": "csharp",
"golden_published": true,
- "model_name": "Meta Llama 3.1 405B",
+ "model_name": "DeepSeek V3",
"total_tests": 3,
"passed_tests": 3,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Address\n {\n public string Street;\n public int Zip;\n }\n\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"profiles\")]\n public partial struct Profile\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Address Home;\n public Address Work;\n public Position Pos;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.profiles.Insert(new Profile\n {\n Id = 1,\n Home = new Address { Street = \"1 Main\", Zip = 11111 },\n Work = new Address { Street = \"2 Broad\", Zip = 22222 },\n Pos = new Position { X = 7, Y = 9 }\n });\n }\n}",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"primitives\")]\n public partial struct Primitive\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Count;\n public long Total;\n public float Price;\n public double Ratio;\n public bool Active;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.primitives.Insert(new Primitive\n {\n Id = 1,\n Count = 2,\n Total = 3000000000,\n Price = 1.5f,\n Ratio = 2.25,\n Active = true,\n Name = \"Alice\"\n });\n }\n}",
"category": "schema",
- "route_api_model": "meta-llama/llama-3.1-405b-instruct",
- "golden_db": "schema-t-015-product-type-columns-golden",
- "llm_db": "schema-t-015-product-type-columns-meta-llama-3-1-405b-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\meta-llama-3-1-405b\\llm",
+ "route_api_model": "deepseek-chat",
+ "golden_db": "schema-t-014-elementary-columns-golden",
+ "llm_db": "schema-t-014-elementary-columns-deepseek-v3-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_014_elementary_columns\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_014_elementary_columns\\csharp\\server\\deepseek-v3\\llm",
"scorer_details": {
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "golden_db": "schema-t-015-product-type-columns-golden",
- "llm_db": "schema-t-015-product-type-columns-meta-llama-3-1-405b-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
- }
- },
- "product_type_columns_row_parity": {
+ "elementary_columns_row_parity": {
"pass": true,
"partial": 1.0,
"notes": {
"args": [],
- "golden_db": "schema-t-015-product-type-columns-golden",
- "golden_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)",
- "llm_db": "schema-t-015-product-type-columns-meta-llama-3-1-405b-llm",
- "llm_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)",
- "query": "SELECT Id, Home, Work, Pos FROM profiles WHERE Id=1",
+ "golden_db": "schema-t-014-elementary-columns-golden",
+ "golden_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"",
+ "llm_db": "schema-t-014-elementary-columns-deepseek-v3-llm",
+ "llm_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"",
+ "query": "SELECT Id, Count, Total, Price, Ratio, Active, Name FROM primitives WHERE Id=1",
"reducer": "Seed",
"server": "local"
}
},
- "product_type_columns_row_count": {
+ "elementary_columns_row_count": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM profiles WHERE Id=1"
+ "sql": "SELECT COUNT(*) AS n FROM primitives WHERE Id=1"
}
- }
- },
- "vendor": "meta",
- "started_at": "2025-10-22T02:37:50.454758900Z",
- "finished_at": "2025-10-22T02:40:17.776456700Z"
- },
- "t_020_ecs": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_020_ecs",
- "lang": "csharp",
- "golden_published": false,
- "model_name": "Meta Llama 3.1 405B",
- "total_tests": 5,
- "passed_tests": 0,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"entities\", Public = true)]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n }\n\n [SpacetimeDB.Table(Name = \"positions\", Public = true)]\n public partial struct Position\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"velocities\", Public = true)]\n public partial struct Velocity\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int VX;\n public int VY;\n }\n\n [SpacetimeDB.Table(Name = \"next_positions\", Public = true)]\n public partial struct NextPosition\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.entities.Insert(new Entity { Id = 1 });\n ctx.Db.entities.Insert(new Entity { Id = 2 });\n\n ctx.Db.positions.Insert(new Position { EntityId = 1, X = 0, Y = 0 });\n ctx.Db.positions.Insert(new Position { EntityId = 2, X = 10, Y = 0 });\n\n ctx.Db.velocities.Insert(new Velocity { EntityId = 1, VX = 1, VY = 0 });\n ctx.Db.velocities.Insert(new Velocity { EntityId = 2, VX = -2, VY = 3 });\n }\n\n [SpacetimeDB.Reducer]\n public static void Step(ReducerContext ctx)\n {\n foreach (var position in ctx.Db.positions.Iter())\n {\n var velocity = ctx.Db.velocities.EntityId.Find(position.EntityId);\n if (velocity != null)\n {\n var nextPosition = new NextPosition\n {\n EntityId = position.EntityId,\n X = position.X + velocity.VX,\n Y = position.Y + velocity.VY,\n };\n ctx.Db.next_positions.EntityId.Update(nextPosition);\n }\n }\n }\n}",
- "category": "schema",
- "route_api_model": "meta-llama/llama-3.1-405b-instruct",
- "golden_db": "schema-t-020-ecs-golden",
- "llm_db": "schema-t-020-ecs-meta-llama-3-1-405b-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\meta-llama-3-1-405b\\llm",
- "scorer_details": {
- "publish_error": {
- "pass": false,
- "partial": 0.0,
+ },
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
"notes": {
- "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\meta-llama-3-1-405b\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'entities' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\meta-llama-3-1-405b\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\meta-llama-3-1-405b\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(113,24): warning CS8981: The type name 'positions' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\meta-llama-3-1-405b\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\meta-llama-3-1-405b\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(156,24): warning CS8981: The type name 'velocities' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\meta-llama-3-1-405b\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\meta-llama-3-1-405b\\llm\\Lib.cs(64,47): error CS1061: 'Module.Velocity?' does not contain a definition for 'VX' and no accessible extension method 'VX' accepting a first argument of type 'Module.Velocity?' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\meta-llama-3-1-405b\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\meta-llama-3-1-405b\\llm\\Lib.cs(65,47): error CS1061: 'Module.Velocity?' does not contain a definition for 'VY' and no accessible extension method 'VY' accepting a first argument of type 'Module.Velocity?' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\meta-llama-3-1-405b\\llm\\StdbModule.csproj]\r\n",
- "phase": "build_or_publish"
+ "golden_db": "schema-t-014-elementary-columns-golden",
+ "llm_db": "schema-t-014-elementary-columns-deepseek-v3-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
}
}
},
- "vendor": "meta",
- "started_at": "2025-10-22T02:38:07.633585800Z",
- "finished_at": "2025-10-22T02:38:35.899918Z"
+ "vendor": "deepseek",
+ "started_at": "2025-10-21T21:47:25.560295400Z",
+ "finished_at": "2025-10-21T21:49:06.926867300Z"
},
- "t_021_multi_column_index": {
+ "t_015_product_type_columns": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_021_multi_column_index",
+ "task": "t_015_product_type_columns",
"lang": "csharp",
"golden_published": true,
- "model_name": "Meta Llama 3.1 405B",
- "total_tests": 4,
- "passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"logs\", Public = true)]\n [SpacetimeDB.Index.BTree(Name = \"by_user_day\", Columns = new[] { nameof(Log.UserId), nameof(Log.Day) })]\n public partial struct Log\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int Day;\n public string Message;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.logs.Insert(new Log { Id = 0, UserId = 7, Day = 1, Message = \"a\" });\n ctx.Db.logs.Insert(new Log { Id = 0, UserId = 7, Day = 2, Message = \"b\" });\n ctx.Db.logs.Insert(new Log { Id = 0, UserId = 9, Day = 1, Message = \"c\" });\n }\n}",
+ "model_name": "DeepSeek V3",
+ "total_tests": 3,
+ "passed_tests": 3,
+ "llm_output": "using SpacetimeDB;\n\n[SpacetimeDB.Type]\npublic partial struct Address\n{\n public string Street;\n public int Zip;\n}\n\n[SpacetimeDB.Type]\npublic partial struct Position\n{\n public int X;\n public int Y;\n}\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"profiles\")]\n public partial struct Profile\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Address Home;\n public Address Work;\n public Position Pos;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.profiles.Insert(new Profile\n {\n Id = 1,\n Home = new Address { Street = \"1 Main\", Zip = 11111 },\n Work = new Address { Street = \"2 Broad\", Zip = 22222 },\n Pos = new Position { X = 7, Y = 9 }\n });\n }\n}",
"category": "schema",
- "route_api_model": "meta-llama/llama-3.1-405b-instruct",
- "golden_db": "schema-t-021-multi-column-index-golden",
- "llm_db": "schema-t-021-multi-column-index-meta-llama-3-1-405b-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\meta-llama-3-1-405b\\llm",
+ "route_api_model": "deepseek-chat",
+ "golden_db": "schema-t-015-product-type-columns-golden",
+ "llm_db": "schema-t-015-product-type-columns-deepseek-v3-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\deepseek-v3\\llm",
"scorer_details": {
- "mcindex_lookup_u7_d1": {
- "pass": false,
- "partial": 0.0,
+ "product_type_columns_row_count": {
+ "pass": true,
+ "partial": 1.0,
"notes": {
- "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: SpacetimeDB.UniqueConstraintViolationException: Value with given unique identifier already exists\n at SpacetimeDB.Internal.FFI.CheckedStatus.Marshaller.ConvertToManaged(Errno )\n at SpacetimeDB.Internal.FFI.datastore_insert_bsatn(TableId , Span`1 , UInt32& )\n at SpacetimeDB.Internal.ITableView`2[[SpacetimeDB.Internal.TableHandles.logs, StdbModule, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null],[Module.Log, StdbModule, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null]].DoInsert(Log )\n at SpacetimeDB.Internal.TableHandles.logs.Insert(Log )\n at Module.Seed(ReducerContext )\n at ModuleRegistration.Seed.Invoke(BinaryReader , IReducerContext )\n at SpacetimeDB.Internal.Module.__call_reducer__(UInt32 id, UInt64 sender_0, UInt64 sender_1, UInt64 sender_2, UInt64 sender_3, UInt64 conn_id_0, UInt64 conn_id_1, Timestamp timestamp, BytesSource args, BytesSink error)\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:3000/v1/database/c200f0a68817026f4d05dbb5d042800387ea1bfdbfd6dfa868d4da7905d3d18f/call/Seed)\n",
- "phase": "call_reducer"
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM profiles WHERE Id=1"
}
},
- "mcindex_lookup_u7_d2": {
- "pass": false,
- "partial": 0.0,
+ "product_type_columns_row_parity": {
+ "pass": true,
+ "partial": 1.0,
"notes": {
- "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: SpacetimeDB.UniqueConstraintViolationException: Value with given unique identifier already exists\n at SpacetimeDB.Internal.FFI.CheckedStatus.Marshaller.ConvertToManaged(Errno )\n at SpacetimeDB.Internal.FFI.datastore_insert_bsatn(TableId , Span`1 , UInt32& )\n at SpacetimeDB.Internal.ITableView`2[[SpacetimeDB.Internal.TableHandles.logs, StdbModule, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null],[Module.Log, StdbModule, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null]].DoInsert(Log )\n at SpacetimeDB.Internal.TableHandles.logs.Insert(Log )\n at Module.Seed(ReducerContext )\n at ModuleRegistration.Seed.Invoke(BinaryReader , IReducerContext )\n at SpacetimeDB.Internal.Module.__call_reducer__(UInt32 id, UInt64 sender_0, UInt64 sender_1, UInt64 sender_2, UInt64 sender_3, UInt64 conn_id_0, UInt64 conn_id_1, Timestamp timestamp, BytesSource args, BytesSink error)\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:3000/v1/database/c200f0a68817026f4d05dbb5d042800387ea1bfdbfd6dfa868d4da7905d3d18f/call/Seed)\n",
- "phase": "call_reducer"
+ "args": [],
+ "golden_db": "schema-t-015-product-type-columns-golden",
+ "golden_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)",
+ "llm_db": "schema-t-015-product-type-columns-deepseek-v3-llm",
+ "llm_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)",
+ "query": "SELECT Id, Home, Work, Pos FROM profiles WHERE Id=1",
+ "reducer": "Seed",
+ "server": "local"
}
},
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-021-multi-column-index-golden",
- "llm_db": "schema-t-021-multi-column-index-meta-llama-3-1-405b-llm",
+ "golden_db": "schema-t-015-product-type-columns-golden",
+ "llm_db": "schema-t-015-product-type-columns-deepseek-v3-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
"tables_diff": null,
"tables_equal": true
}
- },
- "mcindex_seed_count": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: SpacetimeDB.UniqueConstraintViolationException: Value with given unique identifier already exists\n at SpacetimeDB.Internal.FFI.CheckedStatus.Marshaller.ConvertToManaged(Errno )\n at SpacetimeDB.Internal.FFI.datastore_insert_bsatn(TableId , Span`1 , UInt32& )\n at SpacetimeDB.Internal.ITableView`2[[SpacetimeDB.Internal.TableHandles.logs, StdbModule, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null],[Module.Log, StdbModule, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null]].DoInsert(Log )\n at SpacetimeDB.Internal.TableHandles.logs.Insert(Log )\n at Module.Seed(ReducerContext )\n at ModuleRegistration.Seed.Invoke(BinaryReader , IReducerContext )\n at SpacetimeDB.Internal.Module.__call_reducer__(UInt32 id, UInt64 sender_0, UInt64 sender_1, UInt64 sender_2, UInt64 sender_3, UInt64 conn_id_0, UInt64 conn_id_1, Timestamp timestamp, BytesSource args, BytesSink error)\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:3000/v1/database/c200f0a68817026f4d05dbb5d042800387ea1bfdbfd6dfa868d4da7905d3d18f/call/Seed)\n",
- "phase": "call_reducer"
- }
}
},
- "vendor": "meta",
- "started_at": "2025-10-22T02:38:35.949337100Z",
- "finished_at": "2025-10-22T02:40:19.703765800Z"
+ "vendor": "deepseek",
+ "started_at": "2025-10-21T21:47:26.187633700Z",
+ "finished_at": "2025-10-21T21:49:08.548904400Z"
},
- "t_011_helper_function": {
+ "t_016_sum_type_columns": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_011_helper_function",
+ "task": "t_016_sum_type_columns",
"lang": "csharp",
"golden_published": true,
- "model_name": "Meta Llama 3.1 405B",
+ "model_name": "DeepSeek V3",
"total_tests": 3,
"passed_tests": 3,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"results\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Sum;\n }\n\n public static int Add(int a, int b)\n {\n return a + b;\n }\n\n [SpacetimeDB.Reducer]\n public static void ComputeSum(ReducerContext ctx, int id, int a, int b)\n {\n ctx.Db.results.Insert(new Result { Id = id, Sum = Add(a, b) });\n }\n}",
- "category": "basics",
- "route_api_model": "meta-llama/llama-3.1-405b-instruct",
- "golden_db": "basics-t-011-helper-function-golden",
- "llm_db": "basics-t-011-helper-function-meta-llama-3-1-405b-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\meta-llama-3-1-405b\\llm",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"drawings\")]\n public partial struct Drawing\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape A;\n public Shape B;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.drawings.Insert(new Drawing\n {\n Id = 1,\n A = new Shape.Circle(new Circle { Radius = 10 }),\n B = new Shape.Rectangle(new Rectangle { Width = 4, Height = 6 })\n });\n }\n}",
+ "category": "schema",
+ "route_api_model": "deepseek-chat",
+ "golden_db": "schema-t-016-sum-type-columns-golden",
+ "llm_db": "schema-t-016-sum-type-columns-deepseek-v3-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\deepseek-v3\\llm",
"scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-011-helper-function-golden",
- "llm_db": "basics-t-011-helper-function-meta-llama-3-1-405b-llm",
+ "golden_db": "schema-t-016-sum-type-columns-golden",
+ "llm_db": "schema-t-016-sum-type-columns-deepseek-v3-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -13674,144 +13171,56 @@
"tables_equal": true
}
},
- "helper_func_sum_parity": {
+ "sum_type_columns_row_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "args": [
- 1,
- 2,
- 3
- ],
- "golden_db": "basics-t-011-helper-function-golden",
- "golden_out": "Id | Sum ----+----- 1 | 5",
- "llm_db": "basics-t-011-helper-function-meta-llama-3-1-405b-llm",
- "llm_out": "Id | Sum ----+----- 1 | 5",
- "query": "SELECT Id, Sum FROM results WHERE Id=1",
- "reducer": "ComputeSum",
+ "args": [],
+ "golden_db": "schema-t-016-sum-type-columns-golden",
+ "golden_out": "Id | A | B ----+--------------------------+--------------------------------------- 1 | (Circle = (Radius = 10)) | (Rectangle = (Width = 4, Height = 6))",
+ "llm_db": "schema-t-016-sum-type-columns-deepseek-v3-llm",
+ "llm_out": "Id | A | B ----+--------------------------+--------------------------------------- 1 | (Circle = (Radius = 10)) | (Rectangle = (Width = 4, Height = 6))",
+ "query": "SELECT Id, A, B FROM drawings WHERE Id=1",
+ "reducer": "Seed",
"server": "local"
}
},
- "helper_func_sum_abs": {
+ "sum_type_columns_row_count": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1 AND Sum=5"
+ "sql": "SELECT COUNT(*) AS n FROM drawings WHERE Id=1"
}
}
},
- "vendor": "meta",
- "started_at": "2025-10-22T02:37:48.644870100Z",
- "finished_at": "2025-10-22T02:39:59.058161500Z"
+ "vendor": "deepseek",
+ "started_at": "2025-10-21T21:47:26.781535900Z",
+ "finished_at": "2025-10-21T21:49:07.572308Z"
},
- "t_003_struct_in_table": {
+ "t_017_scheduled_columns": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_003_struct_in_table",
+ "task": "t_017_scheduled_columns",
"lang": "csharp",
"golden_published": true,
- "model_name": "Meta Llama 3.1 405B",
- "total_tests": 1,
- "passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"entities\")]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Position Pos;\n }\n}",
- "category": "basics",
- "route_api_model": "meta-llama/llama-3.1-405b-instruct",
- "golden_db": "basics-t-003-struct-in-table-golden",
- "llm_db": "basics-t-003-struct-in-table-meta-llama-3-1-405b-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_003_struct_in_table\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_003_struct_in_table\\csharp\\server\\meta-llama-3-1-405b\\llm",
- "scorer_details": {
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "golden_db": "basics-t-003-struct-in-table-golden",
- "llm_db": "basics-t-003-struct-in-table-meta-llama-3-1-405b-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
- }
- }
- },
- "vendor": "meta",
- "started_at": "2025-10-22T02:37:44.946431800Z",
- "finished_at": "2025-10-22T02:39:46.784002900Z"
- },
- "t_006_delete": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_006_delete",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "Meta Llama 3.1 405B",
- "total_tests": 3,
- "passed_tests": 3,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void DeleteUser(ReducerContext ctx, int id)\n {\n ctx.Db.users.Id.Delete(id);\n }\n}",
- "category": "basics",
- "route_api_model": "meta-llama/llama-3.1-405b-instruct",
- "golden_db": "basics-t-006-delete-golden",
- "llm_db": "basics-t-006-delete-meta-llama-3-1-405b-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\meta-llama-3-1-405b\\llm",
- "scorer_details": {
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "golden_db": "basics-t-006-delete-golden",
- "llm_db": "basics-t-006-delete-meta-llama-3-1-405b-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
- }
- },
- "seed_users_row": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
- }
- },
- "delete_user_count_zero": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 0,
- "expected": 0,
- "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=1"
- }
- }
- },
- "vendor": "meta",
- "started_at": "2025-10-22T02:37:46.342054500Z",
- "finished_at": "2025-10-22T02:39:51.759231100Z"
- },
- "t_007_crud": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_007_crud",
- "lang": "csharp",
- "golden_published": true,
- "model_name": "Meta Llama 3.1 405B",
- "total_tests": 4,
- "passed_tests": 4,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void Crud(ReducerContext ctx)\n {\n ctx.Db.users.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.users.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n ctx.Db.users.Id.Update(new User { Id = 1, Name = \"Alice2\", Age = 31, Active = false });\n ctx.Db.users.Id.Delete(2);\n }\n}",
- "category": "basics",
- "route_api_model": "meta-llama/llama-3.1-405b-instruct",
- "golden_db": "basics-t-007-crud-golden",
- "llm_db": "basics-t-007-crud-meta-llama-3-1-405b-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\meta-llama-3-1-405b\\llm",
+ "model_name": "DeepSeek V3",
+ "total_tests": 2,
+ "passed_tests": 2,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"tick_timer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public ulong ScheduledId;\n\n public ScheduleAt ScheduledAt;\n }\n\n [SpacetimeDB.Reducer]\n public static void Tick(ReducerContext ctx, TickTimer schedule)\n {\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.tick_timer.Insert(new TickTimer\n {\n ScheduledId = 0,\n ScheduledAt = new ScheduleAt.Interval(new TimeDuration { Microseconds = 50_000 })\n });\n }\n}",
+ "category": "schema",
+ "route_api_model": "deepseek-chat",
+ "golden_db": "schema-t-017-scheduled-columns-golden",
+ "llm_db": "schema-t-017-scheduled-columns-deepseek-v3-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_017_scheduled_columns\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_017_scheduled_columns\\csharp\\server\\deepseek-v3\\llm",
"scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-007-crud-golden",
- "llm_db": "basics-t-007-crud-meta-llama-3-1-405b-llm",
+ "golden_db": "schema-t-017-scheduled-columns-golden",
+ "llm_db": "schema-t-017-scheduled-columns-deepseek-v3-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -13819,136 +13228,111 @@
"tables_equal": true
}
},
- "crud_row_id1_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "args": [],
- "golden_db": "basics-t-007-crud-golden",
- "golden_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
- "llm_db": "basics-t-007-crud-meta-llama-3-1-405b-llm",
- "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
- "query": "SELECT Id, Name, Age, Active FROM users WHERE Id=1",
- "reducer": "Crud",
- "server": "local"
- }
- },
- "crud_total_count_one": {
+ "scheduled_seeded_one_row": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM users"
- }
- },
- "crud_row_id2_deleted": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 0,
- "expected": 0,
- "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=2"
+ "sql": "SELECT COUNT(*) AS n FROM tick_timer WHERE ScheduledId>=0"
}
}
},
- "vendor": "meta",
- "started_at": "2025-10-22T02:37:46.803786900Z",
- "finished_at": "2025-10-22T02:40:17.040963300Z"
+ "vendor": "deepseek",
+ "started_at": "2025-10-21T21:47:27.382264500Z",
+ "finished_at": "2025-10-21T21:49:09.480877Z"
},
- "t_009_init": {
+ "t_018_constraints": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_009_init",
+ "task": "t_018_constraints",
"lang": "csharp",
"golden_published": true,
- "model_name": "Meta Llama 3.1 405B",
- "total_tests": 4,
- "passed_tests": 4,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.users.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.users.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n }\n}",
- "category": "basics",
- "route_api_model": "meta-llama/llama-3.1-405b-instruct",
- "golden_db": "basics-t-009-init-golden",
- "llm_db": "basics-t-009-init-meta-llama-3-1-405b-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_009_init\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_009_init\\csharp\\server\\meta-llama-3-1-405b\\llm",
+ "model_name": "DeepSeek V3",
+ "total_tests": 3,
+ "passed_tests": 2,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"accounts\", Public = true)]\n [SpacetimeDB.Index.BTree(Name = \"by_name\", Columns = [nameof(Name)])]\n public partial struct Account\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n [SpacetimeDB.Unique]\n public string Email;\n public string Name;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.accounts.Insert(new Account { Id = 1, Email = \"a@example.com\", Name = \"Alice\" });\n ctx.Db.accounts.Insert(new Account { Id = 2, Email = \"b@example.com\", Name = \"Bob\" });\n }\n}",
+ "category": "schema",
+ "route_api_model": "deepseek-chat",
+ "golden_db": "schema-t-018-constraints-golden",
+ "llm_db": "schema-t-018-constraints-deepseek-v3-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\deepseek-v3\\llm",
"scorer_details": {
- "init_seed_bob": {
- "pass": true,
- "partial": 1.0,
+ "constraints_row_parity_after_seed": {
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false"
+ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: can't directly call special Init lifecycle reducer\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:3000/v1/database/c20079919ed90a64ecacb120b3bdd78cba0f54c43555b600e7f7c763eb2d1274/call/Seed)\n",
+ "phase": "call_reducer_llm"
}
},
- "init_seed_alice": {
+ "constraints_seed_two_rows": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true"
+ "sql": "SELECT COUNT(*) AS n FROM accounts WHERE Id=2"
}
},
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-009-init-golden",
- "llm_db": "basics-t-009-init-meta-llama-3-1-405b-llm",
+ "golden_db": "schema-t-018-constraints-golden",
+ "llm_db": "schema-t-018-constraints-deepseek-v3-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
"tables_diff": null,
"tables_equal": true
}
- },
- "init_total_two": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 2,
- "expected": 2,
- "sql": "SELECT COUNT(*) AS n FROM users"
- }
}
},
- "vendor": "meta",
- "started_at": "2025-10-22T02:37:47.729631500Z",
- "finished_at": "2025-10-22T02:40:18.905430400Z"
+ "vendor": "deepseek",
+ "started_at": "2025-10-21T21:47:27.983142300Z",
+ "finished_at": "2025-10-21T21:49:08.019543700Z"
},
"t_019_many_to_many": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
"task": "t_019_many_to_many",
"lang": "csharp",
"golden_published": true,
- "model_name": "Meta Llama 3.1 405B",
+ "model_name": "DeepSeek V3",
"total_tests": 5,
- "passed_tests": 5,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int UserId;\n public string Name;\n }\n\n [SpacetimeDB.Table(Name = \"groups\", Public = true)]\n public partial struct Group\n {\n [SpacetimeDB.PrimaryKey]\n public int GroupId;\n public string Title;\n }\n\n [SpacetimeDB.Table(Name = \"memberships\", Public = true)]\n public partial struct Membership\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n [SpacetimeDB.Index.BTree(Name = \"by_user\")]\n public int UserId;\n [SpacetimeDB.Index.BTree(Name = \"by_group\")]\n public int GroupId;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.users.Insert(new User { UserId = 1, Name = \"Alice\" });\n ctx.Db.users.Insert(new User { UserId = 2, Name = \"Bob\" });\n\n ctx.Db.groups.Insert(new Group { GroupId = 10, Title = \"Admin\" });\n ctx.Db.groups.Insert(new Group { GroupId = 20, Title = \"Dev\" });\n\n ctx.Db.memberships.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\n ctx.Db.memberships.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\n ctx.Db.memberships.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\n }\n}",
+ "passed_tests": 4,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int UserId;\n public string Name;\n }\n\n [SpacetimeDB.Table(Name = \"groups\", Public = true)]\n public partial struct Group\n {\n [SpacetimeDB.PrimaryKey]\n public int GroupId;\n public string Title;\n }\n\n [SpacetimeDB.Table(Name = \"memberships\", Public = true)]\n [SpacetimeDB.Index.BTree(Name = \"by_user\", Columns = [nameof(UserId)])]\n [SpacetimeDB.Index.BTree(Name = \"by_group\", Columns = [nameof(GroupId)])]\n public partial struct Membership\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int GroupId;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.users.Insert(new User { UserId = 1, Name = \"Alice\" });\n ctx.Db.users.Insert(new User { UserId = 2, Name = \"Bob\" });\n\n ctx.Db.groups.Insert(new Group { GroupId = 10, Title = \"Admin\" });\n ctx.Db.groups.Insert(new Group { GroupId = 20, Title = \"Dev\" });\n\n ctx.Db.memberships.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\n ctx.Db.memberships.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\n ctx.Db.memberships.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\n }\n}",
"category": "schema",
- "route_api_model": "meta-llama/llama-3.1-405b-instruct",
+ "route_api_model": "deepseek-chat",
"golden_db": "schema-t-019-many-to-many-golden",
- "llm_db": "schema-t-019-many-to-many-meta-llama-3-1-405b-llm",
+ "llm_db": "schema-t-019-many-to-many-deepseek-v3-llm",
"work_dir_golden": "target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\meta-llama-3-1-405b\\llm",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\deepseek-v3\\llm",
"scorer_details": {
- "m2m_has_1_10": {
+ "m2m_has_1_20": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM memberships WHERE UserId=1 AND GroupId=10"
+ "sql": "SELECT COUNT(*) AS n FROM memberships WHERE UserId=1 AND GroupId=20"
}
},
- "m2m_has_1_20": {
+ "m2m_has_2_20": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM memberships WHERE UserId=1 AND GroupId=20"
+ "sql": "SELECT COUNT(*) AS n FROM memberships WHERE UserId=2 AND GroupId=20"
+ }
+ },
+ "m2m_has_1_10": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: can't directly call special Init lifecycle reducer\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:3000/v1/database/c2006709be6e83ff9679cb5896e4dcfd226fc63c542008337dfa50ce18aa3869/call/Seed)\n",
+ "phase": "call_reducer"
}
},
"memberships_three_rows": {
@@ -13965,113 +13349,148 @@
"partial": 1.0,
"notes": {
"golden_db": "schema-t-019-many-to-many-golden",
- "llm_db": "schema-t-019-many-to-many-meta-llama-3-1-405b-llm",
+ "llm_db": "schema-t-019-many-to-many-deepseek-v3-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
"tables_diff": null,
"tables_equal": true
}
- },
- "m2m_has_2_20": {
- "pass": true,
- "partial": 1.0,
+ }
+ },
+ "vendor": "deepseek",
+ "started_at": "2025-10-21T21:47:28.582794500Z",
+ "finished_at": "2025-10-21T21:49:10.680876500Z"
+ },
+ "t_020_ecs": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_020_ecs",
+ "lang": "csharp",
+ "golden_published": true,
+ "model_name": "DeepSeek V3",
+ "total_tests": 5,
+ "passed_tests": 0,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"entities\")]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n }\n\n [SpacetimeDB.Table(Name = \"positions\")]\n public partial struct Position\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"velocities\")]\n public partial struct Velocity\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int VX;\n public int VY;\n }\n\n [SpacetimeDB.Table(Name = \"next_positions\")]\n public partial struct NextPosition\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.entities.Insert(new Entity { Id = 1 });\n ctx.Db.positions.Insert(new Position { EntityId = 1, X = 0, Y = 0 });\n ctx.Db.velocities.Insert(new Velocity { EntityId = 1, VX = 1, VY = 0 });\n\n ctx.Db.entities.Insert(new Entity { Id = 2 });\n ctx.Db.positions.Insert(new Position { EntityId = 2, X = 10, Y = 0 });\n ctx.Db.velocities.Insert(new Velocity { EntityId = 2, VX = -2, VY = 3 });\n }\n\n [SpacetimeDB.Reducer]\n public static void Step(ReducerContext ctx)\n {\n foreach (var pos in ctx.Db.positions.Iter())\n {\n var vel = ctx.Db.velocities.EntityId.Find(pos.EntityId);\n if (vel != null)\n {\n var nextPos = new NextPosition\n {\n EntityId = pos.EntityId,\n X = pos.X + vel.VX,\n Y = pos.Y + vel.VY\n };\n ctx.Db.next_positions.EntityId.Update(nextPos);\n }\n }\n }\n}",
+ "category": "schema",
+ "route_api_model": "deepseek-chat",
+ "golden_db": "schema-t-020-ecs-golden",
+ "llm_db": "schema-t-020-ecs-deepseek-v3-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\deepseek-v3\\llm",
+ "scorer_details": {
+ "publish_error": {
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM memberships WHERE UserId=2 AND GroupId=20"
+ "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\deepseek-v3\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'entities' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\deepseek-v3\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\deepseek-v3\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(113,24): warning CS8981: The type name 'positions' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\deepseek-v3\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\deepseek-v3\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(156,24): warning CS8981: The type name 'velocities' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\deepseek-v3\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\deepseek-v3\\llm\\Lib.cs(63,37): error CS1061: 'Module.Velocity?' does not contain a definition for 'VX' and no accessible extension method 'VX' accepting a first argument of type 'Module.Velocity?' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\deepseek-v3\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\deepseek-v3\\llm\\Lib.cs(64,37): error CS1061: 'Module.Velocity?' does not contain a definition for 'VY' and no accessible extension method 'VY' accepting a first argument of type 'Module.Velocity?' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\deepseek-v3\\llm\\StdbModule.csproj]\r\n",
+ "phase": "build_or_publish"
}
}
},
- "vendor": "meta",
- "started_at": "2025-10-22T02:37:52.288238100Z",
- "finished_at": "2025-10-22T02:40:15.906799100Z"
+ "vendor": "deepseek",
+ "started_at": "2025-10-21T21:47:29.171789600Z",
+ "finished_at": "2025-10-21T21:47:53.751440300Z"
},
- "t_008_index_lookup": {
+ "t_021_multi_column_index": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_008_index_lookup",
+ "task": "t_021_multi_column_index",
"lang": "csharp",
"golden_published": true,
- "model_name": "Meta Llama 3.1 405B",
- "total_tests": 3,
- "passed_tests": 3,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"results\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void LookupUserName(ReducerContext ctx, int id)\n {\n if (ctx.Db.users.Id.Find(id) is User user)\n {\n ctx.Db.results.Insert(new Result { Id = user.Id, Name = user.Name });\n }\n }\n}",
- "category": "basics",
- "route_api_model": "meta-llama/llama-3.1-405b-instruct",
- "golden_db": "basics-t-008-index-lookup-golden",
- "llm_db": "basics-t-008-index-lookup-meta-llama-3-1-405b-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\meta-llama-3-1-405b\\llm",
+ "model_name": "DeepSeek V3",
+ "total_tests": 4,
+ "passed_tests": 4,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"logs\")]\n [SpacetimeDB.Index.BTree(Name = \"by_user_day\", Columns = [nameof(UserId), nameof(Day)])]\n public partial struct Log\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int Day;\n public string Message;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.logs.Insert(new Log { Id = 1, UserId = 7, Day = 1, Message = \"a\" });\n ctx.Db.logs.Insert(new Log { Id = 2, UserId = 7, Day = 2, Message = \"b\" });\n ctx.Db.logs.Insert(new Log { Id = 3, UserId = 9, Day = 1, Message = \"c\" });\n }\n}",
+ "category": "schema",
+ "route_api_model": "deepseek-chat",
+ "golden_db": "schema-t-021-multi-column-index-golden",
+ "llm_db": "schema-t-021-multi-column-index-deepseek-v3-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\deepseek-v3\\llm",
"scorer_details": {
- "index_lookup_projection_parity": {
+ "schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "args": [
- 1
- ],
- "golden_db": "basics-t-008-index-lookup-golden",
- "golden_out": "Id | Name ----+--------- 1 | \"Alice\"",
- "llm_db": "basics-t-008-index-lookup-meta-llama-3-1-405b-llm",
- "llm_out": "Id | Name ----+--------- 1 | \"Alice\"",
- "query": "SELECT Id, Name FROM results WHERE Id=1",
- "reducer": "LookupUserName",
- "server": "local"
+ "golden_db": "schema-t-021-multi-column-index-golden",
+ "llm_db": "schema-t-021-multi-column-index-deepseek-v3-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
}
},
- "seed_user_row": {
+ "mcindex_lookup_u7_d1": {
"pass": true,
"partial": 1.0,
"notes": {
- "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM logs WHERE UserId=7 AND Day=1"
}
},
- "schema_parity": {
+ "mcindex_lookup_u7_d2": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-008-index-lookup-golden",
- "llm_db": "basics-t-008-index-lookup-meta-llama-3-1-405b-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM logs WHERE UserId=7 AND Day=2"
+ }
+ },
+ "mcindex_seed_count": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 3,
+ "expected": 3,
+ "sql": "SELECT COUNT(*) AS n FROM logs"
}
}
},
- "vendor": "meta",
- "started_at": "2025-10-22T02:37:47.257153900Z",
- "finished_at": "2025-10-22T02:40:18.620488400Z"
- },
- "t_005_update": {
+ "vendor": "deepseek",
+ "started_at": "2025-10-21T21:47:29.875295400Z",
+ "finished_at": "2025-10-21T21:49:09.219649600Z"
+ }
+ }
+ },
+ {
+ "name": "Meta Llama 3.1 405B",
+ "route_api_model": "meta-llama/llama-3.1-405b-instruct",
+ "tasks": {
+ "t_000_empty_reducers": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_005_update",
+ "task": "t_000_empty_reducers",
"lang": "csharp",
- "golden_published": false,
+ "golden_published": true,
"model_name": "Meta Llama 3.1 405B",
- "total_tests": 3,
- "passed_tests": 0,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n var user = ctx.Db.users.Id.Find(id);\n if (user != null)\n {\n user.Name = name;\n user.Age = age;\n user.Active = active;\n ctx.Db.users.Id.Update(user);\n }\n }\n}",
+ "total_tests": 1,
+ "passed_tests": 1,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Reducer]\n public static void EmptyReducer_NoArgs(ReducerContext ctx)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithInt(ReducerContext ctx, int count)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithString(ReducerContext ctx, string name)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithTwoArgs(ReducerContext ctx, int count, string name)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithThreeArgs(ReducerContext ctx, bool active, float ratio, string label)\n {\n }\n}",
"category": "basics",
"route_api_model": "meta-llama/llama-3.1-405b-instruct",
- "golden_db": "basics-t-005-update-golden",
- "llm_db": "basics-t-005-update-meta-llama-3-1-405b-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_005_update\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_005_update\\csharp\\server\\meta-llama-3-1-405b\\llm",
+ "golden_db": "basics-t-000-empty-reducers-golden",
+ "llm_db": "basics-t-000-empty-reducers-meta-llama-3-1-405b-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\meta-llama-3-1-405b\\llm",
"scorer_details": {
- "publish_error": {
- "pass": false,
- "partial": 0.0,
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
"notes": {
- "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_005_update\\csharp\\server\\meta-llama-3-1-405b\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'users' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_005_update\\csharp\\server\\meta-llama-3-1-405b\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_005_update\\csharp\\server\\meta-llama-3-1-405b\\llm\\Lib.cs(22,18): error CS1061: 'Module.User?' does not contain a definition for 'Name' and no accessible extension method 'Name' accepting a first argument of type 'Module.User?' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_005_update\\csharp\\server\\meta-llama-3-1-405b\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_005_update\\csharp\\server\\meta-llama-3-1-405b\\llm\\Lib.cs(23,18): error CS1061: 'Module.User?' does not contain a definition for 'Age' and no accessible extension method 'Age' accepting a first argument of type 'Module.User?' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_005_update\\csharp\\server\\meta-llama-3-1-405b\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_005_update\\csharp\\server\\meta-llama-3-1-405b\\llm\\Lib.cs(24,18): error CS1061: 'Module.User?' does not contain a definition for 'Active' and no accessible extension method 'Active' accepting a first argument of type 'Module.User?' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_005_update\\csharp\\server\\meta-llama-3-1-405b\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_005_update\\csharp\\server\\meta-llama-3-1-405b\\llm\\Lib.cs(25,36): error CS1503: Argument 1: cannot convert from 'Module.User?' to 'Module.User' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_005_update\\csharp\\server\\meta-llama-3-1-405b\\llm\\StdbModule.csproj]\r\n",
- "phase": "build_or_publish"
+ "golden_db": "basics-t-000-empty-reducers-golden",
+ "llm_db": "basics-t-000-empty-reducers-meta-llama-3-1-405b-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
}
}
},
"vendor": "meta",
- "started_at": "2025-10-22T02:37:45.883600500Z",
- "finished_at": "2025-10-22T02:38:07.589547600Z"
+ "started_at": "2025-10-22T02:37:43.500904700Z",
+ "finished_at": "2025-10-22T02:40:16.036585600Z"
},
"t_001_basic_tables": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
@@ -14141,88 +13560,62 @@
"started_at": "2025-10-22T02:37:44.463523900Z",
"finished_at": "2025-10-22T02:40:19.067957300Z"
},
- "t_013_spacetime_sum_type": {
+ "t_003_struct_in_table": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_013_spacetime_sum_type",
+ "task": "t_003_struct_in_table",
"lang": "csharp",
"golden_published": true,
"model_name": "Meta Llama 3.1 405B",
- "total_tests": 3,
- "passed_tests": 3,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : SpacetimeDB.TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"results\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetCircle(ReducerContext ctx, int id, int radius)\n {\n ctx.Db.results.Insert(new Result { Id = id, Value = new Shape.Circle(new Circle { Radius = radius }) });\n }\n}",
- "category": "schema",
+ "total_tests": 1,
+ "passed_tests": 1,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"entities\")]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Position Pos;\n }\n}",
+ "category": "basics",
"route_api_model": "meta-llama/llama-3.1-405b-instruct",
- "golden_db": "schema-t-013-spacetime-sum-type-golden",
- "llm_db": "schema-t-013-spacetime-sum-type-meta-llama-3-1-405b-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\meta-llama-3-1-405b\\llm",
+ "golden_db": "basics-t-003-struct-in-table-golden",
+ "llm_db": "basics-t-003-struct-in-table-meta-llama-3-1-405b-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_003_struct_in_table\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_003_struct_in_table\\csharp\\server\\meta-llama-3-1-405b\\llm",
"scorer_details": {
- "sum_type_row_count": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1"
- }
- },
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-013-spacetime-sum-type-golden",
- "llm_db": "schema-t-013-spacetime-sum-type-meta-llama-3-1-405b-llm",
+ "golden_db": "basics-t-003-struct-in-table-golden",
+ "llm_db": "basics-t-003-struct-in-table-meta-llama-3-1-405b-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
"tables_diff": null,
"tables_equal": true
}
- },
- "sum_type_row_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "args": [
- 1,
- 10
- ],
- "golden_db": "schema-t-013-spacetime-sum-type-golden",
- "golden_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))",
- "llm_db": "schema-t-013-spacetime-sum-type-meta-llama-3-1-405b-llm",
- "llm_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))",
- "query": "SELECT Id, Value FROM results WHERE Id=1",
- "reducer": "SetCircle",
- "server": "local"
- }
}
},
"vendor": "meta",
- "started_at": "2025-10-22T02:37:49.557394700Z",
- "finished_at": "2025-10-22T02:40:15.010918600Z"
+ "started_at": "2025-10-22T02:37:44.946431800Z",
+ "finished_at": "2025-10-22T02:39:46.784002900Z"
},
- "t_012_spacetime_product_type": {
+ "t_004_insert": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_012_spacetime_product_type",
+ "task": "t_004_insert",
"lang": "csharp",
"golden_published": true,
"model_name": "Meta Llama 3.1 405B",
- "total_tests": 3,
- "passed_tests": 3,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Score\n {\n public int Left;\n public int Right;\n }\n\n [SpacetimeDB.Table(Name = \"results\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Score Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetScore(ReducerContext ctx, int id, int left, int right)\n {\n ctx.Db.results.Insert(new Result { Id = id, Value = new Score { Left = left, Right = right } });\n }\n}",
- "category": "schema",
+ "total_tests": 2,
+ "passed_tests": 2,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void InsertUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n ctx.Db.users.Insert(new User { Id = id, Name = name, Age = age, Active = active });\n }\n}",
+ "category": "basics",
"route_api_model": "meta-llama/llama-3.1-405b-instruct",
- "golden_db": "schema-t-012-spacetime-product-type-golden",
- "llm_db": "schema-t-012-spacetime-product-type-meta-llama-3-1-405b-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\meta-llama-3-1-405b\\llm",
+ "golden_db": "basics-t-004-insert-golden",
+ "llm_db": "basics-t-004-insert-meta-llama-3-1-405b-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_004_insert\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_004_insert\\csharp\\server\\meta-llama-3-1-405b\\llm",
"scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-012-spacetime-product-type-golden",
- "llm_db": "schema-t-012-spacetime-product-type-meta-llama-3-1-405b-llm",
+ "golden_db": "basics-t-004-insert-golden",
+ "llm_db": "basics-t-004-insert-meta-llama-3-1-405b-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -14230,94 +13623,131 @@
"tables_equal": true
}
},
- "product_type_row_count": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1"
- }
- },
- "product_type_row_parity": {
+ "data_parity_insert_user": {
"pass": true,
"partial": 1.0,
"notes": {
"args": [
1,
- 2,
- 3
+ "Alice",
+ 30,
+ true
],
- "golden_db": "schema-t-012-spacetime-product-type-golden",
- "golden_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)",
- "llm_db": "schema-t-012-spacetime-product-type-meta-llama-3-1-405b-llm",
- "llm_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)",
- "query": "SELECT Id, Value FROM results WHERE Id=1",
- "reducer": "SetScore",
+ "golden_db": "basics-t-004-insert-golden",
+ "golden_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true",
+ "llm_db": "basics-t-004-insert-meta-llama-3-1-405b-llm",
+ "llm_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true",
+ "query": "SELECT Id, Name, Age, Active FROM users WHERE Id=1",
+ "reducer": "InsertUser",
"server": "local"
}
}
},
"vendor": "meta",
- "started_at": "2025-10-22T02:37:49.101881600Z",
- "finished_at": "2025-10-22T02:40:19.424521900Z"
+ "started_at": "2025-10-22T02:37:45.419651600Z",
+ "finished_at": "2025-10-22T02:39:54.827032Z"
},
- "t_000_empty_reducers": {
+ "t_005_update": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_000_empty_reducers",
+ "task": "t_005_update",
"lang": "csharp",
- "golden_published": true,
+ "golden_published": false,
"model_name": "Meta Llama 3.1 405B",
- "total_tests": 1,
- "passed_tests": 1,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Reducer]\n public static void EmptyReducer_NoArgs(ReducerContext ctx)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithInt(ReducerContext ctx, int count)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithString(ReducerContext ctx, string name)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithTwoArgs(ReducerContext ctx, int count, string name)\n {\n }\n\n [Reducer]\n public static void EmptyReducer_WithThreeArgs(ReducerContext ctx, bool active, float ratio, string label)\n {\n }\n}",
- "category": "basics",
- "route_api_model": "meta-llama/llama-3.1-405b-instruct",
- "golden_db": "basics-t-000-empty-reducers-golden",
- "llm_db": "basics-t-000-empty-reducers-meta-llama-3-1-405b-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_000_empty_reducers\\csharp\\server\\meta-llama-3-1-405b\\llm",
+ "total_tests": 3,
+ "passed_tests": 0,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n var user = ctx.Db.users.Id.Find(id);\n if (user != null)\n {\n user.Name = name;\n user.Age = age;\n user.Active = active;\n ctx.Db.users.Id.Update(user);\n }\n }\n}",
+ "category": "basics",
+ "route_api_model": "meta-llama/llama-3.1-405b-instruct",
+ "golden_db": "basics-t-005-update-golden",
+ "llm_db": "basics-t-005-update-meta-llama-3-1-405b-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_005_update\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_005_update\\csharp\\server\\meta-llama-3-1-405b\\llm",
+ "scorer_details": {
+ "publish_error": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_005_update\\csharp\\server\\meta-llama-3-1-405b\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'users' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_005_update\\csharp\\server\\meta-llama-3-1-405b\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_005_update\\csharp\\server\\meta-llama-3-1-405b\\llm\\Lib.cs(22,18): error CS1061: 'Module.User?' does not contain a definition for 'Name' and no accessible extension method 'Name' accepting a first argument of type 'Module.User?' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_005_update\\csharp\\server\\meta-llama-3-1-405b\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_005_update\\csharp\\server\\meta-llama-3-1-405b\\llm\\Lib.cs(23,18): error CS1061: 'Module.User?' does not contain a definition for 'Age' and no accessible extension method 'Age' accepting a first argument of type 'Module.User?' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_005_update\\csharp\\server\\meta-llama-3-1-405b\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_005_update\\csharp\\server\\meta-llama-3-1-405b\\llm\\Lib.cs(24,18): error CS1061: 'Module.User?' does not contain a definition for 'Active' and no accessible extension method 'Active' accepting a first argument of type 'Module.User?' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_005_update\\csharp\\server\\meta-llama-3-1-405b\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_005_update\\csharp\\server\\meta-llama-3-1-405b\\llm\\Lib.cs(25,36): error CS1503: Argument 1: cannot convert from 'Module.User?' to 'Module.User' [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_005_update\\csharp\\server\\meta-llama-3-1-405b\\llm\\StdbModule.csproj]\r\n",
+ "phase": "build_or_publish"
+ }
+ }
+ },
+ "vendor": "meta",
+ "started_at": "2025-10-22T02:37:45.883600500Z",
+ "finished_at": "2025-10-22T02:38:07.589547600Z"
+ },
+ "t_006_delete": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_006_delete",
+ "lang": "csharp",
+ "golden_published": true,
+ "model_name": "Meta Llama 3.1 405B",
+ "total_tests": 3,
+ "passed_tests": 3,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void DeleteUser(ReducerContext ctx, int id)\n {\n ctx.Db.users.Id.Delete(id);\n }\n}",
+ "category": "basics",
+ "route_api_model": "meta-llama/llama-3.1-405b-instruct",
+ "golden_db": "basics-t-006-delete-golden",
+ "llm_db": "basics-t-006-delete-meta-llama-3-1-405b-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_006_delete\\csharp\\server\\meta-llama-3-1-405b\\llm",
"scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-000-empty-reducers-golden",
- "llm_db": "basics-t-000-empty-reducers-meta-llama-3-1-405b-llm",
+ "golden_db": "basics-t-006-delete-golden",
+ "llm_db": "basics-t-006-delete-meta-llama-3-1-405b-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
"tables_diff": null,
"tables_equal": true
}
+ },
+ "seed_users_row": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
+ }
+ },
+ "delete_user_count_zero": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 0,
+ "expected": 0,
+ "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=1"
+ }
}
},
"vendor": "meta",
- "started_at": "2025-10-22T02:37:43.500904700Z",
- "finished_at": "2025-10-22T02:40:16.036585600Z"
+ "started_at": "2025-10-22T02:37:46.342054500Z",
+ "finished_at": "2025-10-22T02:39:51.759231100Z"
},
- "t_004_insert": {
+ "t_007_crud": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_004_insert",
+ "task": "t_007_crud",
"lang": "csharp",
"golden_published": true,
"model_name": "Meta Llama 3.1 405B",
- "total_tests": 2,
- "passed_tests": 2,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void InsertUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n ctx.Db.users.Insert(new User { Id = id, Name = name, Age = age, Active = active });\n }\n}",
+ "total_tests": 4,
+ "passed_tests": 4,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void Crud(ReducerContext ctx)\n {\n ctx.Db.users.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.users.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n ctx.Db.users.Id.Update(new User { Id = 1, Name = \"Alice2\", Age = 31, Active = false });\n ctx.Db.users.Id.Delete(2);\n }\n}",
"category": "basics",
"route_api_model": "meta-llama/llama-3.1-405b-instruct",
- "golden_db": "basics-t-004-insert-golden",
- "llm_db": "basics-t-004-insert-meta-llama-3-1-405b-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_004_insert\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_004_insert\\csharp\\server\\meta-llama-3-1-405b\\llm",
+ "golden_db": "basics-t-007-crud-golden",
+ "llm_db": "basics-t-007-crud-meta-llama-3-1-405b-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_007_crud\\csharp\\server\\meta-llama-3-1-405b\\llm",
"scorer_details": {
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-004-insert-golden",
- "llm_db": "basics-t-004-insert-meta-llama-3-1-405b-llm",
+ "golden_db": "basics-t-007-crud-golden",
+ "llm_db": "basics-t-007-crud-meta-llama-3-1-405b-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -14325,75 +13755,88 @@
"tables_equal": true
}
},
- "data_parity_insert_user": {
+ "crud_row_id1_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "args": [
- 1,
- "Alice",
- 30,
- true
- ],
- "golden_db": "basics-t-004-insert-golden",
- "golden_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true",
- "llm_db": "basics-t-004-insert-meta-llama-3-1-405b-llm",
- "llm_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true",
+ "args": [],
+ "golden_db": "basics-t-007-crud-golden",
+ "golden_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
+ "llm_db": "basics-t-007-crud-meta-llama-3-1-405b-llm",
+ "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
"query": "SELECT Id, Name, Age, Active FROM users WHERE Id=1",
- "reducer": "InsertUser",
+ "reducer": "Crud",
"server": "local"
}
+ },
+ "crud_total_count_one": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM users"
+ }
+ },
+ "crud_row_id2_deleted": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 0,
+ "expected": 0,
+ "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=2"
+ }
}
},
"vendor": "meta",
- "started_at": "2025-10-22T02:37:45.419651600Z",
- "finished_at": "2025-10-22T02:39:54.827032Z"
+ "started_at": "2025-10-22T02:37:46.803786900Z",
+ "finished_at": "2025-10-22T02:40:17.040963300Z"
},
- "t_014_elementary_columns": {
+ "t_008_index_lookup": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_014_elementary_columns",
+ "task": "t_008_index_lookup",
"lang": "csharp",
"golden_published": true,
"model_name": "Meta Llama 3.1 405B",
"total_tests": 3,
"passed_tests": 3,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"primitives\", Public = true)]\n public partial struct Primitive\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Count;\n public long Total;\n public float Price;\n public double Ratio;\n public bool Active;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.primitives.Insert(new Primitive\n {\n Id = 1,\n Count = 2,\n Total = 3000000000,\n Price = 1.5f,\n Ratio = 2.25,\n Active = true,\n Name = \"Alice\"\n });\n }\n}",
- "category": "schema",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"results\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void LookupUserName(ReducerContext ctx, int id)\n {\n if (ctx.Db.users.Id.Find(id) is User user)\n {\n ctx.Db.results.Insert(new Result { Id = user.Id, Name = user.Name });\n }\n }\n}",
+ "category": "basics",
"route_api_model": "meta-llama/llama-3.1-405b-instruct",
- "golden_db": "schema-t-014-elementary-columns-golden",
- "llm_db": "schema-t-014-elementary-columns-meta-llama-3-1-405b-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_014_elementary_columns\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_014_elementary_columns\\csharp\\server\\meta-llama-3-1-405b\\llm",
+ "golden_db": "basics-t-008-index-lookup-golden",
+ "llm_db": "basics-t-008-index-lookup-meta-llama-3-1-405b-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_008_index_lookup\\csharp\\server\\meta-llama-3-1-405b\\llm",
"scorer_details": {
- "elementary_columns_row_parity": {
+ "index_lookup_projection_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "args": [],
- "golden_db": "schema-t-014-elementary-columns-golden",
- "golden_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"",
- "llm_db": "schema-t-014-elementary-columns-meta-llama-3-1-405b-llm",
- "llm_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"",
- "query": "SELECT Id, Count, Total, Price, Ratio, Active, Name FROM primitives WHERE Id=1",
- "reducer": "Seed",
+ "args": [
+ 1
+ ],
+ "golden_db": "basics-t-008-index-lookup-golden",
+ "golden_out": "Id | Name ----+--------- 1 | \"Alice\"",
+ "llm_db": "basics-t-008-index-lookup-meta-llama-3-1-405b-llm",
+ "llm_out": "Id | Name ----+--------- 1 | \"Alice\"",
+ "query": "SELECT Id, Name FROM results WHERE Id=1",
+ "reducer": "LookupUserName",
"server": "local"
}
},
- "elementary_columns_row_count": {
+ "seed_user_row": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM primitives WHERE Id=1"
+ "sql": "INSERT INTO users(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)"
}
},
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-014-elementary-columns-golden",
- "llm_db": "schema-t-014-elementary-columns-meta-llama-3-1-405b-llm",
+ "golden_db": "basics-t-008-index-lookup-golden",
+ "llm_db": "basics-t-008-index-lookup-meta-llama-3-1-405b-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -14403,65 +13846,69 @@
}
},
"vendor": "meta",
- "started_at": "2025-10-22T02:37:50.002760300Z",
- "finished_at": "2025-10-22T02:40:17.419892900Z"
+ "started_at": "2025-10-22T02:37:47.257153900Z",
+ "finished_at": "2025-10-22T02:40:18.620488400Z"
},
- "t_018_constraints": {
+ "t_009_init": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_018_constraints",
+ "task": "t_009_init",
"lang": "csharp",
"golden_published": true,
"model_name": "Meta Llama 3.1 405B",
- "total_tests": 3,
- "passed_tests": 3,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"accounts\", Public = true)]\n public partial struct Account\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n [SpacetimeDB.Unique]\n public string Email;\n [SpacetimeDB.Index.BTree(Name = \"by_name\")]\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.accounts.Insert(new Account { Id = 1, Email = \"a@example.com\", Name = \"Alice\" });\n ctx.Db.accounts.Insert(new Account { Id = 2, Email = \"b@example.com\", Name = \"Bob\" });\n }\n}",
- "category": "schema",
+ "total_tests": 4,
+ "passed_tests": 4,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.users.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.users.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n }\n}",
+ "category": "basics",
"route_api_model": "meta-llama/llama-3.1-405b-instruct",
- "golden_db": "schema-t-018-constraints-golden",
- "llm_db": "schema-t-018-constraints-meta-llama-3-1-405b-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\meta-llama-3-1-405b\\llm",
+ "golden_db": "basics-t-009-init-golden",
+ "llm_db": "basics-t-009-init-meta-llama-3-1-405b-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_009_init\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_009_init\\csharp\\server\\meta-llama-3-1-405b\\llm",
"scorer_details": {
- "schema_parity": {
+ "init_seed_alice": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-018-constraints-golden",
- "llm_db": "schema-t-018-constraints-meta-llama-3-1-405b-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true"
}
},
- "constraints_seed_two_rows": {
+ "init_seed_bob": {
"pass": true,
"partial": 1.0,
"notes": {
"actual": 1,
"expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM accounts WHERE Id=2"
+ "sql": "SELECT COUNT(*) AS n FROM users WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false"
}
},
- "constraints_row_parity_after_seed": {
+ "schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "args": [],
- "golden_db": "schema-t-018-constraints-golden",
- "golden_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"",
- "llm_db": "schema-t-018-constraints-meta-llama-3-1-405b-llm",
- "llm_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"",
- "query": "SELECT Id, Email, Name FROM accounts WHERE Id=1",
- "reducer": "Seed",
- "server": "local"
+ "golden_db": "basics-t-009-init-golden",
+ "llm_db": "basics-t-009-init-meta-llama-3-1-405b-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
+ }
+ },
+ "init_total_two": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 2,
+ "expected": 2,
+ "sql": "SELECT COUNT(*) AS n FROM users"
}
}
},
"vendor": "meta",
- "started_at": "2025-10-22T02:37:51.844622400Z",
- "finished_at": "2025-10-22T02:40:18.151265200Z"
+ "started_at": "2025-10-22T02:37:47.729631500Z",
+ "finished_at": "2025-10-22T02:40:18.905430400Z"
},
"t_010_connect": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
@@ -14497,51 +13944,55 @@
"started_at": "2025-10-22T02:37:48.187327400Z",
"finished_at": "2025-10-22T02:40:15.142467300Z"
},
- "t_016_sum_type_columns": {
+ "t_011_helper_function": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_016_sum_type_columns",
+ "task": "t_011_helper_function",
"lang": "csharp",
"golden_published": true,
"model_name": "Meta Llama 3.1 405B",
"total_tests": 3,
"passed_tests": 3,
- "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : SpacetimeDB.TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"drawings\")]\n public partial struct Drawing\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape A;\n public Shape B;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.drawings.Insert(new Drawing { Id = 1, A = new Shape.Circle(new Circle { Radius = 10 }), B = new Shape.Rectangle(new Rectangle { Width = 4, Height = 6 }) });\n }\n}",
- "category": "schema",
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"results\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Sum;\n }\n\n public static int Add(int a, int b)\n {\n return a + b;\n }\n\n [SpacetimeDB.Reducer]\n public static void ComputeSum(ReducerContext ctx, int id, int a, int b)\n {\n ctx.Db.results.Insert(new Result { Id = id, Sum = Add(a, b) });\n }\n}",
+ "category": "basics",
"route_api_model": "meta-llama/llama-3.1-405b-instruct",
- "golden_db": "schema-t-016-sum-type-columns-golden",
- "llm_db": "schema-t-016-sum-type-columns-meta-llama-3-1-405b-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\meta-llama-3-1-405b\\llm",
+ "golden_db": "basics-t-011-helper-function-golden",
+ "llm_db": "basics-t-011-helper-function-meta-llama-3-1-405b-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_011_helper_function\\csharp\\server\\meta-llama-3-1-405b\\llm",
"scorer_details": {
- "sum_type_columns_row_count": {
+ "helper_func_sum_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM drawings WHERE Id=1"
+ "args": [
+ 1,
+ 2,
+ 3
+ ],
+ "golden_db": "basics-t-011-helper-function-golden",
+ "golden_out": "Id | Sum ----+----- 1 | 5",
+ "llm_db": "basics-t-011-helper-function-meta-llama-3-1-405b-llm",
+ "llm_out": "Id | Sum ----+----- 1 | 5",
+ "query": "SELECT Id, Sum FROM results WHERE Id=1",
+ "reducer": "ComputeSum",
+ "server": "local"
}
},
- "sum_type_columns_row_parity": {
+ "helper_func_sum_abs": {
"pass": true,
"partial": 1.0,
"notes": {
- "args": [],
- "golden_db": "schema-t-016-sum-type-columns-golden",
- "golden_out": "Id | A | B ----+--------------------------+--------------------------------------- 1 | (Circle = (Radius = 10)) | (Rectangle = (Width = 4, Height = 6))",
- "llm_db": "schema-t-016-sum-type-columns-meta-llama-3-1-405b-llm",
- "llm_out": "Id | A | B ----+--------------------------+--------------------------------------- 1 | (Circle = (Radius = 10)) | (Rectangle = (Width = 4, Height = 6))",
- "query": "SELECT Id, A, B FROM drawings WHERE Id=1",
- "reducer": "Seed",
- "server": "local"
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1 AND Sum=5"
}
},
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "schema-t-016-sum-type-columns-golden",
- "llm_db": "schema-t-016-sum-type-columns-meta-llama-3-1-405b-llm",
+ "golden_db": "basics-t-011-helper-function-golden",
+ "llm_db": "basics-t-011-helper-function-meta-llama-3-1-405b-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -14551,137 +14002,570 @@
}
},
"vendor": "meta",
- "started_at": "2025-10-22T02:37:50.921364300Z",
- "finished_at": "2025-10-22T02:40:15.535405700Z"
- }
- }
- }
- ]
- },
- {
- "mode": "llms.md",
- "hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4",
- "models": [
- {
- "name": "GPT-4.1",
- "route_api_model": "gpt-4.1",
- "tasks": {
- "t_011_helper_function": {
- "hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4",
- "task": "t_011_helper_function",
- "lang": "csharp",
- "golden_published": false,
- "model_name": "GPT-4.1",
- "total_tests": 1,
- "passed_tests": 0,
- "llm_output": null,
- "category": "basics",
- "route_api_model": "gpt-4.1",
- "golden_db": null,
- "llm_db": null,
- "work_dir_golden": null,
- "work_dir_llm": null,
- "scorer_details": {
- "publish_error": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "POST https://api.openai.com/v1/responses -> 400 Bad Request: {\n \"error\": {\n \"message\": \"Unsupported parameter: 'reasoning.effort' is not supported with this model.\",\n \"type\": \"invalid_request_error\",\n \"param\": \"reasoning.effort\",\n \"code\": \"unsupported_parameter\"\n }\n}",
- "phase": "build_or_publish"
- }
- }
- },
- "vendor": "openai",
- "started_at": "2025-10-19T21:28:33.433605700Z",
- "finished_at": "2025-10-19T21:28:33.433605700Z"
+ "started_at": "2025-10-22T02:37:48.644870100Z",
+ "finished_at": "2025-10-22T02:39:59.058161500Z"
},
- "t_017_scheduled_columns": {
- "hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4",
- "task": "t_017_scheduled_columns",
+ "t_012_spacetime_product_type": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_012_spacetime_product_type",
"lang": "csharp",
- "golden_published": false,
- "model_name": "GPT-4.1",
- "total_tests": 1,
- "passed_tests": 0,
- "llm_output": null,
+ "golden_published": true,
+ "model_name": "Meta Llama 3.1 405B",
+ "total_tests": 3,
+ "passed_tests": 3,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Score\n {\n public int Left;\n public int Right;\n }\n\n [SpacetimeDB.Table(Name = \"results\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Score Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetScore(ReducerContext ctx, int id, int left, int right)\n {\n ctx.Db.results.Insert(new Result { Id = id, Value = new Score { Left = left, Right = right } });\n }\n}",
"category": "schema",
- "route_api_model": "gpt-4.1",
- "golden_db": null,
- "llm_db": null,
- "work_dir_golden": null,
- "work_dir_llm": null,
+ "route_api_model": "meta-llama/llama-3.1-405b-instruct",
+ "golden_db": "schema-t-012-spacetime-product-type-golden",
+ "llm_db": "schema-t-012-spacetime-product-type-meta-llama-3-1-405b-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_012_spacetime_product_type\\csharp\\server\\meta-llama-3-1-405b\\llm",
"scorer_details": {
- "publish_error": {
- "pass": false,
- "partial": 0.0,
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
"notes": {
- "error": "POST https://api.openai.com/v1/responses -> 400 Bad Request: {\n \"error\": {\n \"message\": \"Unsupported parameter: 'reasoning.effort' is not supported with this model.\",\n \"type\": \"invalid_request_error\",\n \"param\": \"reasoning.effort\",\n \"code\": \"unsupported_parameter\"\n }\n}",
- "phase": "build_or_publish"
+ "golden_db": "schema-t-012-spacetime-product-type-golden",
+ "llm_db": "schema-t-012-spacetime-product-type-meta-llama-3-1-405b-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
+ }
+ },
+ "product_type_row_count": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1"
+ }
+ },
+ "product_type_row_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "args": [
+ 1,
+ 2,
+ 3
+ ],
+ "golden_db": "schema-t-012-spacetime-product-type-golden",
+ "golden_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)",
+ "llm_db": "schema-t-012-spacetime-product-type-meta-llama-3-1-405b-llm",
+ "llm_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)",
+ "query": "SELECT Id, Value FROM results WHERE Id=1",
+ "reducer": "SetScore",
+ "server": "local"
}
}
},
- "vendor": "openai",
- "started_at": "2025-10-19T21:28:33.433811900Z",
- "finished_at": "2025-10-19T21:28:33.433811900Z"
+ "vendor": "meta",
+ "started_at": "2025-10-22T02:37:49.101881600Z",
+ "finished_at": "2025-10-22T02:40:19.424521900Z"
},
- "t_012_spacetime_product_type": {
- "hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4",
- "task": "t_012_spacetime_product_type",
+ "t_013_spacetime_sum_type": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_013_spacetime_sum_type",
"lang": "csharp",
- "golden_published": false,
- "model_name": "GPT-4.1",
- "total_tests": 1,
- "passed_tests": 0,
- "llm_output": null,
+ "golden_published": true,
+ "model_name": "Meta Llama 3.1 405B",
+ "total_tests": 3,
+ "passed_tests": 3,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : SpacetimeDB.TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"results\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetCircle(ReducerContext ctx, int id, int radius)\n {\n ctx.Db.results.Insert(new Result { Id = id, Value = new Shape.Circle(new Circle { Radius = radius }) });\n }\n}",
"category": "schema",
- "route_api_model": "gpt-4.1",
- "golden_db": null,
- "llm_db": null,
- "work_dir_golden": null,
- "work_dir_llm": null,
+ "route_api_model": "meta-llama/llama-3.1-405b-instruct",
+ "golden_db": "schema-t-013-spacetime-sum-type-golden",
+ "llm_db": "schema-t-013-spacetime-sum-type-meta-llama-3-1-405b-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\meta-llama-3-1-405b\\llm",
"scorer_details": {
- "publish_error": {
- "pass": false,
- "partial": 0.0,
+ "sum_type_row_count": {
+ "pass": true,
+ "partial": 1.0,
"notes": {
- "error": "POST https://api.openai.com/v1/responses -> 400 Bad Request: {\n \"error\": {\n \"message\": \"Unsupported parameter: 'reasoning.effort' is not supported with this model.\",\n \"type\": \"invalid_request_error\",\n \"param\": \"reasoning.effort\",\n \"code\": \"unsupported_parameter\"\n }\n}",
- "phase": "build_or_publish"
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM results WHERE Id=1"
+ }
+ },
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "golden_db": "schema-t-013-spacetime-sum-type-golden",
+ "llm_db": "schema-t-013-spacetime-sum-type-meta-llama-3-1-405b-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
+ }
+ },
+ "sum_type_row_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "args": [
+ 1,
+ 10
+ ],
+ "golden_db": "schema-t-013-spacetime-sum-type-golden",
+ "golden_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))",
+ "llm_db": "schema-t-013-spacetime-sum-type-meta-llama-3-1-405b-llm",
+ "llm_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))",
+ "query": "SELECT Id, Value FROM results WHERE Id=1",
+ "reducer": "SetCircle",
+ "server": "local"
}
}
},
- "vendor": "openai",
- "started_at": "2025-10-19T21:28:33.435143Z",
- "finished_at": "2025-10-19T21:28:33.435143Z"
+ "vendor": "meta",
+ "started_at": "2025-10-22T02:37:49.557394700Z",
+ "finished_at": "2025-10-22T02:40:15.010918600Z"
},
- "t_015_product_type_columns": {
- "hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4",
- "task": "t_015_product_type_columns",
+ "t_014_elementary_columns": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_014_elementary_columns",
"lang": "csharp",
- "golden_published": false,
- "model_name": "GPT-4.1",
- "total_tests": 1,
- "passed_tests": 0,
- "llm_output": null,
+ "golden_published": true,
+ "model_name": "Meta Llama 3.1 405B",
+ "total_tests": 3,
+ "passed_tests": 3,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"primitives\", Public = true)]\n public partial struct Primitive\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Count;\n public long Total;\n public float Price;\n public double Ratio;\n public bool Active;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.primitives.Insert(new Primitive\n {\n Id = 1,\n Count = 2,\n Total = 3000000000,\n Price = 1.5f,\n Ratio = 2.25,\n Active = true,\n Name = \"Alice\"\n });\n }\n}",
"category": "schema",
- "route_api_model": "gpt-4.1",
- "golden_db": null,
- "llm_db": null,
- "work_dir_golden": null,
- "work_dir_llm": null,
+ "route_api_model": "meta-llama/llama-3.1-405b-instruct",
+ "golden_db": "schema-t-014-elementary-columns-golden",
+ "llm_db": "schema-t-014-elementary-columns-meta-llama-3-1-405b-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_014_elementary_columns\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_014_elementary_columns\\csharp\\server\\meta-llama-3-1-405b\\llm",
"scorer_details": {
- "publish_error": {
+ "elementary_columns_row_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "args": [],
+ "golden_db": "schema-t-014-elementary-columns-golden",
+ "golden_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"",
+ "llm_db": "schema-t-014-elementary-columns-meta-llama-3-1-405b-llm",
+ "llm_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"",
+ "query": "SELECT Id, Count, Total, Price, Ratio, Active, Name FROM primitives WHERE Id=1",
+ "reducer": "Seed",
+ "server": "local"
+ }
+ },
+ "elementary_columns_row_count": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM primitives WHERE Id=1"
+ }
+ },
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "golden_db": "schema-t-014-elementary-columns-golden",
+ "llm_db": "schema-t-014-elementary-columns-meta-llama-3-1-405b-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
+ }
+ }
+ },
+ "vendor": "meta",
+ "started_at": "2025-10-22T02:37:50.002760300Z",
+ "finished_at": "2025-10-22T02:40:17.419892900Z"
+ },
+ "t_015_product_type_columns": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_015_product_type_columns",
+ "lang": "csharp",
+ "golden_published": true,
+ "model_name": "Meta Llama 3.1 405B",
+ "total_tests": 3,
+ "passed_tests": 3,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Address\n {\n public string Street;\n public int Zip;\n }\n\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"profiles\")]\n public partial struct Profile\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Address Home;\n public Address Work;\n public Position Pos;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.profiles.Insert(new Profile\n {\n Id = 1,\n Home = new Address { Street = \"1 Main\", Zip = 11111 },\n Work = new Address { Street = \"2 Broad\", Zip = 22222 },\n Pos = new Position { X = 7, Y = 9 }\n });\n }\n}",
+ "category": "schema",
+ "route_api_model": "meta-llama/llama-3.1-405b-instruct",
+ "golden_db": "schema-t-015-product-type-columns-golden",
+ "llm_db": "schema-t-015-product-type-columns-meta-llama-3-1-405b-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_015_product_type_columns\\csharp\\server\\meta-llama-3-1-405b\\llm",
+ "scorer_details": {
+ "product_type_columns_row_count": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM profiles WHERE Id=1"
+ }
+ },
+ "product_type_columns_row_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "args": [],
+ "golden_db": "schema-t-015-product-type-columns-golden",
+ "golden_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)",
+ "llm_db": "schema-t-015-product-type-columns-meta-llama-3-1-405b-llm",
+ "llm_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)",
+ "query": "SELECT Id, Home, Work, Pos FROM profiles WHERE Id=1",
+ "reducer": "Seed",
+ "server": "local"
+ }
+ },
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "golden_db": "schema-t-015-product-type-columns-golden",
+ "llm_db": "schema-t-015-product-type-columns-meta-llama-3-1-405b-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
+ }
+ }
+ },
+ "vendor": "meta",
+ "started_at": "2025-10-22T02:37:50.454758900Z",
+ "finished_at": "2025-10-22T02:40:17.776456700Z"
+ },
+ "t_016_sum_type_columns": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_016_sum_type_columns",
+ "lang": "csharp",
+ "golden_published": true,
+ "model_name": "Meta Llama 3.1 405B",
+ "total_tests": 3,
+ "passed_tests": 3,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : SpacetimeDB.TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"drawings\")]\n public partial struct Drawing\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape A;\n public Shape B;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.drawings.Insert(new Drawing { Id = 1, A = new Shape.Circle(new Circle { Radius = 10 }), B = new Shape.Rectangle(new Rectangle { Width = 4, Height = 6 }) });\n }\n}",
+ "category": "schema",
+ "route_api_model": "meta-llama/llama-3.1-405b-instruct",
+ "golden_db": "schema-t-016-sum-type-columns-golden",
+ "llm_db": "schema-t-016-sum-type-columns-meta-llama-3-1-405b-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\meta-llama-3-1-405b\\llm",
+ "scorer_details": {
+ "sum_type_columns_row_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "args": [],
+ "golden_db": "schema-t-016-sum-type-columns-golden",
+ "golden_out": "Id | A | B ----+--------------------------+--------------------------------------- 1 | (Circle = (Radius = 10)) | (Rectangle = (Width = 4, Height = 6))",
+ "llm_db": "schema-t-016-sum-type-columns-meta-llama-3-1-405b-llm",
+ "llm_out": "Id | A | B ----+--------------------------+--------------------------------------- 1 | (Circle = (Radius = 10)) | (Rectangle = (Width = 4, Height = 6))",
+ "query": "SELECT Id, A, B FROM drawings WHERE Id=1",
+ "reducer": "Seed",
+ "server": "local"
+ }
+ },
+ "sum_type_columns_row_count": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM drawings WHERE Id=1"
+ }
+ },
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "golden_db": "schema-t-016-sum-type-columns-golden",
+ "llm_db": "schema-t-016-sum-type-columns-meta-llama-3-1-405b-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
+ }
+ }
+ },
+ "vendor": "meta",
+ "started_at": "2025-10-22T02:37:50.921364300Z",
+ "finished_at": "2025-10-22T02:40:15.535405700Z"
+ },
+ "t_017_scheduled_columns": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_017_scheduled_columns",
+ "lang": "csharp",
+ "golden_published": true,
+ "model_name": "Meta Llama 3.1 405B",
+ "total_tests": 2,
+ "passed_tests": 2,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"tick_timer\", Scheduled = nameof(Tick), ScheduledAt = nameof(TickTimer.ScheduledAt))]\n public partial struct TickTimer\n {\n [PrimaryKey]\n [AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [Reducer]\n public static void Tick(ReducerContext ctx, TickTimer schedule)\n {\n Log.Info(\"Tick reducer called\");\n }\n\n [Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n var currentTime = ctx.Timestamp;\n var interval = new TimeDuration { Microseconds = +50_000 };\n var futureTimestamp = currentTime + interval;\n\n ctx.Db.tick_timer.Insert(new()\n {\n ScheduledId = 0, // Have [AutoInc] assign an Id.\n ScheduledAt = new ScheduleAt.Interval(interval),\n });\n }\n}",
+ "category": "schema",
+ "route_api_model": "meta-llama/llama-3.1-405b-instruct",
+ "golden_db": "schema-t-017-scheduled-columns-golden",
+ "llm_db": "schema-t-017-scheduled-columns-meta-llama-3-1-405b-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_017_scheduled_columns\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_017_scheduled_columns\\csharp\\server\\meta-llama-3-1-405b\\llm",
+ "scorer_details": {
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "golden_db": "schema-t-017-scheduled-columns-golden",
+ "llm_db": "schema-t-017-scheduled-columns-meta-llama-3-1-405b-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
+ }
+ },
+ "scheduled_seeded_one_row": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM tick_timer WHERE ScheduledId>=0"
+ }
+ }
+ },
+ "vendor": "meta",
+ "started_at": "2025-10-22T02:37:51.377249200Z",
+ "finished_at": "2025-10-22T02:40:16.461659400Z"
+ },
+ "t_018_constraints": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_018_constraints",
+ "lang": "csharp",
+ "golden_published": true,
+ "model_name": "Meta Llama 3.1 405B",
+ "total_tests": 3,
+ "passed_tests": 3,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"accounts\", Public = true)]\n public partial struct Account\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n [SpacetimeDB.Unique]\n public string Email;\n [SpacetimeDB.Index.BTree(Name = \"by_name\")]\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.accounts.Insert(new Account { Id = 1, Email = \"a@example.com\", Name = \"Alice\" });\n ctx.Db.accounts.Insert(new Account { Id = 2, Email = \"b@example.com\", Name = \"Bob\" });\n }\n}",
+ "category": "schema",
+ "route_api_model": "meta-llama/llama-3.1-405b-instruct",
+ "golden_db": "schema-t-018-constraints-golden",
+ "llm_db": "schema-t-018-constraints-meta-llama-3-1-405b-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\meta-llama-3-1-405b\\llm",
+ "scorer_details": {
+ "constraints_row_parity_after_seed": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "args": [],
+ "golden_db": "schema-t-018-constraints-golden",
+ "golden_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"",
+ "llm_db": "schema-t-018-constraints-meta-llama-3-1-405b-llm",
+ "llm_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"",
+ "query": "SELECT Id, Email, Name FROM accounts WHERE Id=1",
+ "reducer": "Seed",
+ "server": "local"
+ }
+ },
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "golden_db": "schema-t-018-constraints-golden",
+ "llm_db": "schema-t-018-constraints-meta-llama-3-1-405b-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
+ }
+ },
+ "constraints_seed_two_rows": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM accounts WHERE Id=2"
+ }
+ }
+ },
+ "vendor": "meta",
+ "started_at": "2025-10-22T02:37:51.844622400Z",
+ "finished_at": "2025-10-22T02:40:18.151265200Z"
+ },
+ "t_019_many_to_many": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_019_many_to_many",
+ "lang": "csharp",
+ "golden_published": true,
+ "model_name": "Meta Llama 3.1 405B",
+ "total_tests": 5,
+ "passed_tests": 5,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"users\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int UserId;\n public string Name;\n }\n\n [SpacetimeDB.Table(Name = \"groups\", Public = true)]\n public partial struct Group\n {\n [SpacetimeDB.PrimaryKey]\n public int GroupId;\n public string Title;\n }\n\n [SpacetimeDB.Table(Name = \"memberships\", Public = true)]\n public partial struct Membership\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n [SpacetimeDB.Index.BTree(Name = \"by_user\")]\n public int UserId;\n [SpacetimeDB.Index.BTree(Name = \"by_group\")]\n public int GroupId;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.users.Insert(new User { UserId = 1, Name = \"Alice\" });\n ctx.Db.users.Insert(new User { UserId = 2, Name = \"Bob\" });\n\n ctx.Db.groups.Insert(new Group { GroupId = 10, Title = \"Admin\" });\n ctx.Db.groups.Insert(new Group { GroupId = 20, Title = \"Dev\" });\n\n ctx.Db.memberships.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\n ctx.Db.memberships.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\n ctx.Db.memberships.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\n }\n}",
+ "category": "schema",
+ "route_api_model": "meta-llama/llama-3.1-405b-instruct",
+ "golden_db": "schema-t-019-many-to-many-golden",
+ "llm_db": "schema-t-019-many-to-many-meta-llama-3-1-405b-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\meta-llama-3-1-405b\\llm",
+ "scorer_details": {
+ "memberships_three_rows": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 3,
+ "expected": 3,
+ "sql": "SELECT COUNT(*) AS n FROM memberships"
+ }
+ },
+ "m2m_has_1_20": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM memberships WHERE UserId=1 AND GroupId=20"
+ }
+ },
+ "m2m_has_1_10": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM memberships WHERE UserId=1 AND GroupId=10"
+ }
+ },
+ "m2m_has_2_20": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM memberships WHERE UserId=2 AND GroupId=20"
+ }
+ },
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "golden_db": "schema-t-019-many-to-many-golden",
+ "llm_db": "schema-t-019-many-to-many-meta-llama-3-1-405b-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
+ }
+ }
+ },
+ "vendor": "meta",
+ "started_at": "2025-10-22T02:37:52.288238100Z",
+ "finished_at": "2025-10-22T02:40:15.906799100Z"
+ },
+ "t_020_ecs": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_020_ecs",
+ "lang": "csharp",
+ "golden_published": false,
+ "model_name": "Meta Llama 3.1 405B",
+ "total_tests": 5,
+ "passed_tests": 0,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"entities\", Public = true)]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n }\n\n [SpacetimeDB.Table(Name = \"positions\", Public = true)]\n public partial struct Position\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"velocities\", Public = true)]\n public partial struct Velocity\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int VX;\n public int VY;\n }\n\n [SpacetimeDB.Table(Name = \"next_positions\", Public = true)]\n public partial struct NextPosition\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.entities.Insert(new Entity { Id = 1 });\n ctx.Db.entities.Insert(new Entity { Id = 2 });\n\n ctx.Db.positions.Insert(new Position { EntityId = 1, X = 0, Y = 0 });\n ctx.Db.positions.Insert(new Position { EntityId = 2, X = 10, Y = 0 });\n\n ctx.Db.velocities.Insert(new Velocity { EntityId = 1, VX = 1, VY = 0 });\n ctx.Db.velocities.Insert(new Velocity { EntityId = 2, VX = -2, VY = 3 });\n }\n\n [SpacetimeDB.Reducer]\n public static void Step(ReducerContext ctx)\n {\n foreach (var position in ctx.Db.positions.Iter())\n {\n var velocity = ctx.Db.velocities.EntityId.Find(position.EntityId);\n if (velocity != null)\n {\n var nextPosition = new NextPosition\n {\n EntityId = position.EntityId,\n X = position.X + velocity.VX,\n Y = position.Y + velocity.VY,\n };\n ctx.Db.next_positions.EntityId.Update(nextPosition);\n }\n }\n }\n}",
+ "category": "schema",
+ "route_api_model": "meta-llama/llama-3.1-405b-instruct",
+ "golden_db": "schema-t-020-ecs-golden",
+ "llm_db": "schema-t-020-ecs-meta-llama-3-1-405b-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\meta-llama-3-1-405b\\llm",
+ "scorer_details": {
+ "publish_error": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "POST https://api.openai.com/v1/responses -> 400 Bad Request: {\n \"error\": {\n \"message\": \"Unsupported parameter: 'reasoning.effort' is not supported with this model.\",\n \"type\": \"invalid_request_error\",\n \"param\": \"reasoning.effort\",\n \"code\": \"unsupported_parameter\"\n }\n}",
+ "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\meta-llama-3-1-405b\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(27,32): warning CS8981: The type name 'entities' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\meta-llama-3-1-405b\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\meta-llama-3-1-405b\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(113,24): warning CS8981: The type name 'positions' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\meta-llama-3-1-405b\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\meta-llama-3-1-405b\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\FFI.cs(156,24): warning CS8981: The type name 'velocities' only contains lower-cased ascii characters. Such names may become reserved for the language. [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\meta-llama-3-1-405b\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\meta-llama-3-1-405b\\llm\\Lib.cs(64,47): error CS1061: 'Module.Velocity?' does not contain a definition for 'VX' and no accessible extension method 'VX' accepting a first argument of type 'Module.Velocity?' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\meta-llama-3-1-405b\\llm\\StdbModule.csproj]\r\nE:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\meta-llama-3-1-405b\\llm\\Lib.cs(65,47): error CS1061: 'Module.Velocity?' does not contain a definition for 'VY' and no accessible extension method 'VY' accepting a first argument of type 'Module.Velocity?' could be found (are you missing a using directive or an assembly reference?) [E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\meta-llama-3-1-405b\\llm\\StdbModule.csproj]\r\n",
"phase": "build_or_publish"
}
}
},
- "vendor": "openai",
- "started_at": "2025-10-19T21:28:33.434461200Z",
- "finished_at": "2025-10-19T21:28:33.434461200Z"
+ "vendor": "meta",
+ "started_at": "2025-10-22T02:38:07.633585800Z",
+ "finished_at": "2025-10-22T02:38:35.899918Z"
},
+ "t_021_multi_column_index": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_021_multi_column_index",
+ "lang": "csharp",
+ "golden_published": true,
+ "model_name": "Meta Llama 3.1 405B",
+ "total_tests": 4,
+ "passed_tests": 1,
+ "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"logs\", Public = true)]\n [SpacetimeDB.Index.BTree(Name = \"by_user_day\", Columns = new[] { nameof(Log.UserId), nameof(Log.Day) })]\n public partial struct Log\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int Day;\n public string Message;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.logs.Insert(new Log { Id = 0, UserId = 7, Day = 1, Message = \"a\" });\n ctx.Db.logs.Insert(new Log { Id = 0, UserId = 7, Day = 2, Message = \"b\" });\n ctx.Db.logs.Insert(new Log { Id = 0, UserId = 9, Day = 1, Message = \"c\" });\n }\n}",
+ "category": "schema",
+ "route_api_model": "meta-llama/llama-3.1-405b-instruct",
+ "golden_db": "schema-t-021-multi-column-index-golden",
+ "llm_db": "schema-t-021-multi-column-index-meta-llama-3-1-405b-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\meta-llama-3-1-405b\\llm",
+ "scorer_details": {
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "golden_db": "schema-t-021-multi-column-index-golden",
+ "llm_db": "schema-t-021-multi-column-index-meta-llama-3-1-405b-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
+ }
+ },
+ "mcindex_seed_count": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: SpacetimeDB.UniqueConstraintViolationException: Value with given unique identifier already exists\n at SpacetimeDB.Internal.FFI.CheckedStatus.Marshaller.ConvertToManaged(Errno )\n at SpacetimeDB.Internal.FFI.datastore_insert_bsatn(TableId , Span`1 , UInt32& )\n at SpacetimeDB.Internal.ITableView`2[[SpacetimeDB.Internal.TableHandles.logs, StdbModule, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null],[Module.Log, StdbModule, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null]].DoInsert(Log )\n at SpacetimeDB.Internal.TableHandles.logs.Insert(Log )\n at Module.Seed(ReducerContext )\n at ModuleRegistration.Seed.Invoke(BinaryReader , IReducerContext )\n at SpacetimeDB.Internal.Module.__call_reducer__(UInt32 id, UInt64 sender_0, UInt64 sender_1, UInt64 sender_2, UInt64 sender_3, UInt64 conn_id_0, UInt64 conn_id_1, Timestamp timestamp, BytesSource args, BytesSink error)\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:3000/v1/database/c200f0a68817026f4d05dbb5d042800387ea1bfdbfd6dfa868d4da7905d3d18f/call/Seed)\n",
+ "phase": "call_reducer"
+ }
+ },
+ "mcindex_lookup_u7_d1": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: SpacetimeDB.UniqueConstraintViolationException: Value with given unique identifier already exists\n at SpacetimeDB.Internal.FFI.CheckedStatus.Marshaller.ConvertToManaged(Errno )\n at SpacetimeDB.Internal.FFI.datastore_insert_bsatn(TableId , Span`1 , UInt32& )\n at SpacetimeDB.Internal.ITableView`2[[SpacetimeDB.Internal.TableHandles.logs, StdbModule, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null],[Module.Log, StdbModule, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null]].DoInsert(Log )\n at SpacetimeDB.Internal.TableHandles.logs.Insert(Log )\n at Module.Seed(ReducerContext )\n at ModuleRegistration.Seed.Invoke(BinaryReader , IReducerContext )\n at SpacetimeDB.Internal.Module.__call_reducer__(UInt32 id, UInt64 sender_0, UInt64 sender_1, UInt64 sender_2, UInt64 sender_3, UInt64 conn_id_0, UInt64 conn_id_1, Timestamp timestamp, BytesSource args, BytesSink error)\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:3000/v1/database/c200f0a68817026f4d05dbb5d042800387ea1bfdbfd6dfa868d4da7905d3d18f/call/Seed)\n",
+ "phase": "call_reducer"
+ }
+ },
+ "mcindex_lookup_u7_d2": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: SpacetimeDB.UniqueConstraintViolationException: Value with given unique identifier already exists\n at SpacetimeDB.Internal.FFI.CheckedStatus.Marshaller.ConvertToManaged(Errno )\n at SpacetimeDB.Internal.FFI.datastore_insert_bsatn(TableId , Span`1 , UInt32& )\n at SpacetimeDB.Internal.ITableView`2[[SpacetimeDB.Internal.TableHandles.logs, StdbModule, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null],[Module.Log, StdbModule, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null]].DoInsert(Log )\n at SpacetimeDB.Internal.TableHandles.logs.Insert(Log )\n at Module.Seed(ReducerContext )\n at ModuleRegistration.Seed.Invoke(BinaryReader , IReducerContext )\n at SpacetimeDB.Internal.Module.__call_reducer__(UInt32 id, UInt64 sender_0, UInt64 sender_1, UInt64 sender_2, UInt64 sender_3, UInt64 conn_id_0, UInt64 conn_id_1, Timestamp timestamp, BytesSource args, BytesSink error)\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:3000/v1/database/c200f0a68817026f4d05dbb5d042800387ea1bfdbfd6dfa868d4da7905d3d18f/call/Seed)\n",
+ "phase": "call_reducer"
+ }
+ }
+ },
+ "vendor": "meta",
+ "started_at": "2025-10-22T02:38:35.949337100Z",
+ "finished_at": "2025-10-22T02:40:19.703765800Z"
+ }
+ }
+ }
+ ]
+ },
+ {
+ "mode": "llms.md",
+ "hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4",
+ "models": [
+ {
+ "name": "GPT-4.1",
+ "route_api_model": "gpt-4.1",
+ "tasks": {
"t_000_empty_reducers": {
"hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4",
"task": "t_000_empty_reducers",
@@ -14711,9 +14595,9 @@
"started_at": "2025-10-19T21:28:33.432967400Z",
"finished_at": "2025-10-19T21:28:33.432967400Z"
},
- "t_009_init": {
+ "t_001_basic_tables": {
"hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4",
- "task": "t_009_init",
+ "task": "t_001_basic_tables",
"lang": "csharp",
"golden_published": false,
"model_name": "GPT-4.1",
@@ -14737,19 +14621,19 @@
}
},
"vendor": "openai",
- "started_at": "2025-10-19T21:28:33.431525800Z",
- "finished_at": "2025-10-19T21:28:33.431525800Z"
+ "started_at": "2025-10-19T21:28:33.434691200Z",
+ "finished_at": "2025-10-19T21:28:33.434691200Z"
},
- "t_013_spacetime_sum_type": {
+ "t_002_scheduled_table": {
"hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4",
- "task": "t_013_spacetime_sum_type",
+ "task": "t_002_scheduled_table",
"lang": "csharp",
"golden_published": false,
"model_name": "GPT-4.1",
"total_tests": 1,
"passed_tests": 0,
"llm_output": null,
- "category": "schema",
+ "category": "basics",
"route_api_model": "gpt-4.1",
"golden_db": null,
"llm_db": null,
@@ -14766,12 +14650,12 @@
}
},
"vendor": "openai",
- "started_at": "2025-10-19T21:28:33.431179800Z",
- "finished_at": "2025-10-19T21:28:33.431179800Z"
+ "started_at": "2025-10-19T21:28:33.430120700Z",
+ "finished_at": "2025-10-19T21:28:33.430120700Z"
},
- "t_001_basic_tables": {
+ "t_003_struct_in_table": {
"hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4",
- "task": "t_001_basic_tables",
+ "task": "t_003_struct_in_table",
"lang": "csharp",
"golden_published": false,
"model_name": "GPT-4.1",
@@ -14795,19 +14679,19 @@
}
},
"vendor": "openai",
- "started_at": "2025-10-19T21:28:33.434691200Z",
- "finished_at": "2025-10-19T21:28:33.434691200Z"
+ "started_at": "2025-10-19T21:28:33.434047400Z",
+ "finished_at": "2025-10-19T21:28:33.434047400Z"
},
- "t_021_multi_column_index": {
+ "t_004_insert": {
"hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4",
- "task": "t_021_multi_column_index",
+ "task": "t_004_insert",
"lang": "csharp",
"golden_published": false,
"model_name": "GPT-4.1",
"total_tests": 1,
"passed_tests": 0,
"llm_output": null,
- "category": "schema",
+ "category": "basics",
"route_api_model": "gpt-4.1",
"golden_db": null,
"llm_db": null,
@@ -14824,8 +14708,8 @@
}
},
"vendor": "openai",
- "started_at": "2025-10-19T21:28:33.433206700Z",
- "finished_at": "2025-10-19T21:28:33.433206700Z"
+ "started_at": "2025-10-19T21:28:33.430505400Z",
+ "finished_at": "2025-10-19T21:28:33.430505400Z"
},
"t_005_update": {
"hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4",
@@ -14856,9 +14740,9 @@
"started_at": "2025-10-19T21:28:33.431839700Z",
"finished_at": "2025-10-19T21:28:33.431839700Z"
},
- "t_002_scheduled_table": {
+ "t_006_delete": {
"hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4",
- "task": "t_002_scheduled_table",
+ "task": "t_006_delete",
"lang": "csharp",
"golden_published": false,
"model_name": "GPT-4.1",
@@ -14882,19 +14766,19 @@
}
},
"vendor": "openai",
- "started_at": "2025-10-19T21:28:33.430120700Z",
- "finished_at": "2025-10-19T21:28:33.430120700Z"
+ "started_at": "2025-10-19T21:28:33.435349900Z",
+ "finished_at": "2025-10-19T21:28:33.435349900Z"
},
- "t_018_constraints": {
+ "t_007_crud": {
"hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4",
- "task": "t_018_constraints",
+ "task": "t_007_crud",
"lang": "csharp",
"golden_published": false,
"model_name": "GPT-4.1",
"total_tests": 1,
"passed_tests": 0,
"llm_output": null,
- "category": "schema",
+ "category": "basics",
"route_api_model": "gpt-4.1",
"golden_db": null,
"llm_db": null,
@@ -14911,19 +14795,19 @@
}
},
"vendor": "openai",
- "started_at": "2025-10-19T21:28:33.434921500Z",
- "finished_at": "2025-10-19T21:28:33.434921500Z"
+ "started_at": "2025-10-19T21:28:33.435572400Z",
+ "finished_at": "2025-10-19T21:28:33.435572400Z"
},
- "t_020_ecs": {
+ "t_008_index_lookup": {
"hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4",
- "task": "t_020_ecs",
+ "task": "t_008_index_lookup",
"lang": "csharp",
"golden_published": false,
"model_name": "GPT-4.1",
"total_tests": 1,
"passed_tests": 0,
"llm_output": null,
- "category": "schema",
+ "category": "basics",
"route_api_model": "gpt-4.1",
"golden_db": null,
"llm_db": null,
@@ -14940,12 +14824,12 @@
}
},
"vendor": "openai",
- "started_at": "2025-10-19T21:28:33.435872Z",
- "finished_at": "2025-10-19T21:28:33.435872Z"
+ "started_at": "2025-10-19T21:28:33.432390Z",
+ "finished_at": "2025-10-19T21:28:33.432390Z"
},
- "t_008_index_lookup": {
+ "t_009_init": {
"hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4",
- "task": "t_008_index_lookup",
+ "task": "t_009_init",
"lang": "csharp",
"golden_published": false,
"model_name": "GPT-4.1",
@@ -14969,12 +14853,12 @@
}
},
"vendor": "openai",
- "started_at": "2025-10-19T21:28:33.432390Z",
- "finished_at": "2025-10-19T21:28:33.432390Z"
+ "started_at": "2025-10-19T21:28:33.431525800Z",
+ "finished_at": "2025-10-19T21:28:33.431525800Z"
},
- "t_003_struct_in_table": {
+ "t_010_connect": {
"hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4",
- "task": "t_003_struct_in_table",
+ "task": "t_010_connect",
"lang": "csharp",
"golden_published": false,
"model_name": "GPT-4.1",
@@ -14998,19 +14882,19 @@
}
},
"vendor": "openai",
- "started_at": "2025-10-19T21:28:33.434047400Z",
- "finished_at": "2025-10-19T21:28:33.434047400Z"
+ "started_at": "2025-10-19T21:28:33.432767Z",
+ "finished_at": "2025-10-19T21:28:33.432767Z"
},
- "t_014_elementary_columns": {
+ "t_011_helper_function": {
"hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4",
- "task": "t_014_elementary_columns",
+ "task": "t_011_helper_function",
"lang": "csharp",
"golden_published": false,
"model_name": "GPT-4.1",
"total_tests": 1,
"passed_tests": 0,
"llm_output": null,
- "category": "schema",
+ "category": "basics",
"route_api_model": "gpt-4.1",
"golden_db": null,
"llm_db": null,
@@ -15027,48 +14911,19 @@
}
},
"vendor": "openai",
- "started_at": "2025-10-19T21:28:33.433396900Z",
- "finished_at": "2025-10-19T21:28:33.433396900Z"
+ "started_at": "2025-10-19T21:28:33.433605700Z",
+ "finished_at": "2025-10-19T21:28:33.433605700Z"
},
- "t_004_insert": {
+ "t_012_spacetime_product_type": {
"hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4",
- "task": "t_004_insert",
- "lang": "csharp",
- "golden_published": false,
- "model_name": "GPT-4.1",
- "total_tests": 1,
- "passed_tests": 0,
- "llm_output": null,
- "category": "basics",
- "route_api_model": "gpt-4.1",
- "golden_db": null,
- "llm_db": null,
- "work_dir_golden": null,
- "work_dir_llm": null,
- "scorer_details": {
- "publish_error": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "POST https://api.openai.com/v1/responses -> 400 Bad Request: {\n \"error\": {\n \"message\": \"Unsupported parameter: 'reasoning.effort' is not supported with this model.\",\n \"type\": \"invalid_request_error\",\n \"param\": \"reasoning.effort\",\n \"code\": \"unsupported_parameter\"\n }\n}",
- "phase": "build_or_publish"
- }
- }
- },
- "vendor": "openai",
- "started_at": "2025-10-19T21:28:33.430505400Z",
- "finished_at": "2025-10-19T21:28:33.430505400Z"
- },
- "t_010_connect": {
- "hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4",
- "task": "t_010_connect",
+ "task": "t_012_spacetime_product_type",
"lang": "csharp",
"golden_published": false,
"model_name": "GPT-4.1",
"total_tests": 1,
"passed_tests": 0,
"llm_output": null,
- "category": "basics",
+ "category": "schema",
"route_api_model": "gpt-4.1",
"golden_db": null,
"llm_db": null,
@@ -15085,19 +14940,19 @@
}
},
"vendor": "openai",
- "started_at": "2025-10-19T21:28:33.432767Z",
- "finished_at": "2025-10-19T21:28:33.432767Z"
+ "started_at": "2025-10-19T21:28:33.435143Z",
+ "finished_at": "2025-10-19T21:28:33.435143Z"
},
- "t_007_crud": {
+ "t_013_spacetime_sum_type": {
"hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4",
- "task": "t_007_crud",
+ "task": "t_013_spacetime_sum_type",
"lang": "csharp",
"golden_published": false,
"model_name": "GPT-4.1",
"total_tests": 1,
"passed_tests": 0,
"llm_output": null,
- "category": "basics",
+ "category": "schema",
"route_api_model": "gpt-4.1",
"golden_db": null,
"llm_db": null,
@@ -15114,12 +14969,12 @@
}
},
"vendor": "openai",
- "started_at": "2025-10-19T21:28:33.435572400Z",
- "finished_at": "2025-10-19T21:28:33.435572400Z"
+ "started_at": "2025-10-19T21:28:33.431179800Z",
+ "finished_at": "2025-10-19T21:28:33.431179800Z"
},
- "t_019_many_to_many": {
+ "t_014_elementary_columns": {
"hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4",
- "task": "t_019_many_to_many",
+ "task": "t_014_elementary_columns",
"lang": "csharp",
"golden_published": false,
"model_name": "GPT-4.1",
@@ -15143,19 +14998,19 @@
}
},
"vendor": "openai",
- "started_at": "2025-10-19T21:28:33.434246200Z",
- "finished_at": "2025-10-19T21:28:33.434246200Z"
+ "started_at": "2025-10-19T21:28:33.433396900Z",
+ "finished_at": "2025-10-19T21:28:33.433396900Z"
},
- "t_006_delete": {
+ "t_015_product_type_columns": {
"hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4",
- "task": "t_006_delete",
+ "task": "t_015_product_type_columns",
"lang": "csharp",
"golden_published": false,
"model_name": "GPT-4.1",
"total_tests": 1,
"passed_tests": 0,
"llm_output": null,
- "category": "basics",
+ "category": "schema",
"route_api_model": "gpt-4.1",
"golden_db": null,
"llm_db": null,
@@ -15172,8 +15027,8 @@
}
},
"vendor": "openai",
- "started_at": "2025-10-19T21:28:33.435349900Z",
- "finished_at": "2025-10-19T21:28:33.435349900Z"
+ "started_at": "2025-10-19T21:28:33.434461200Z",
+ "finished_at": "2025-10-19T21:28:33.434461200Z"
},
"t_016_sum_type_columns": {
"hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4",
@@ -15203,53 +15058,18 @@
"vendor": "openai",
"started_at": "2025-10-19T21:28:33.432177300Z",
"finished_at": "2025-10-19T21:28:33.432177300Z"
- }
- }
- },
- {
- "name": "GPT-4o",
- "route_api_model": "gpt-4o",
- "tasks": {
- "t_002_scheduled_table": {
- "hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4",
- "task": "t_002_scheduled_table",
- "lang": "csharp",
- "golden_published": false,
- "model_name": "GPT-4o",
- "total_tests": 1,
- "passed_tests": 0,
- "llm_output": null,
- "category": "basics",
- "route_api_model": "gpt-4o",
- "golden_db": null,
- "llm_db": null,
- "work_dir_golden": null,
- "work_dir_llm": null,
- "scorer_details": {
- "publish_error": {
- "pass": false,
- "partial": 0.0,
- "notes": {
- "error": "POST https://api.openai.com/v1/responses -> 400 Bad Request: {\n \"error\": {\n \"message\": \"Unsupported parameter: 'reasoning.effort' is not supported with this model.\",\n \"type\": \"invalid_request_error\",\n \"param\": \"reasoning.effort\",\n \"code\": \"unsupported_parameter\"\n }\n}",
- "phase": "build_or_publish"
- }
- }
- },
- "vendor": "openai",
- "started_at": "2025-10-19T21:28:34.268046600Z",
- "finished_at": "2025-10-19T21:28:34.268046600Z"
},
- "t_014_elementary_columns": {
+ "t_017_scheduled_columns": {
"hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4",
- "task": "t_014_elementary_columns",
+ "task": "t_017_scheduled_columns",
"lang": "csharp",
"golden_published": false,
- "model_name": "GPT-4o",
+ "model_name": "GPT-4.1",
"total_tests": 1,
"passed_tests": 0,
"llm_output": null,
"category": "schema",
- "route_api_model": "gpt-4o",
+ "route_api_model": "gpt-4.1",
"golden_db": null,
"llm_db": null,
"work_dir_golden": null,
@@ -15265,20 +15085,20 @@
}
},
"vendor": "openai",
- "started_at": "2025-10-19T21:28:34.266545400Z",
- "finished_at": "2025-10-19T21:28:34.266545400Z"
+ "started_at": "2025-10-19T21:28:33.433811900Z",
+ "finished_at": "2025-10-19T21:28:33.433811900Z"
},
- "t_020_ecs": {
+ "t_018_constraints": {
"hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4",
- "task": "t_020_ecs",
+ "task": "t_018_constraints",
"lang": "csharp",
"golden_published": false,
- "model_name": "GPT-4o",
+ "model_name": "GPT-4.1",
"total_tests": 1,
"passed_tests": 0,
"llm_output": null,
"category": "schema",
- "route_api_model": "gpt-4o",
+ "route_api_model": "gpt-4.1",
"golden_db": null,
"llm_db": null,
"work_dir_golden": null,
@@ -15294,20 +15114,20 @@
}
},
"vendor": "openai",
- "started_at": "2025-10-19T21:28:34.266767800Z",
- "finished_at": "2025-10-19T21:28:34.266767800Z"
+ "started_at": "2025-10-19T21:28:33.434921500Z",
+ "finished_at": "2025-10-19T21:28:33.434921500Z"
},
"t_019_many_to_many": {
"hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4",
"task": "t_019_many_to_many",
"lang": "csharp",
"golden_published": false,
- "model_name": "GPT-4o",
+ "model_name": "GPT-4.1",
"total_tests": 1,
"passed_tests": 0,
"llm_output": null,
"category": "schema",
- "route_api_model": "gpt-4o",
+ "route_api_model": "gpt-4.1",
"golden_db": null,
"llm_db": null,
"work_dir_golden": null,
@@ -15323,20 +15143,20 @@
}
},
"vendor": "openai",
- "started_at": "2025-10-19T21:28:34.266156200Z",
- "finished_at": "2025-10-19T21:28:34.266156200Z"
+ "started_at": "2025-10-19T21:28:33.434246200Z",
+ "finished_at": "2025-10-19T21:28:33.434246200Z"
},
- "t_017_scheduled_columns": {
+ "t_020_ecs": {
"hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4",
- "task": "t_017_scheduled_columns",
+ "task": "t_020_ecs",
"lang": "csharp",
"golden_published": false,
- "model_name": "GPT-4o",
+ "model_name": "GPT-4.1",
"total_tests": 1,
"passed_tests": 0,
"llm_output": null,
"category": "schema",
- "route_api_model": "gpt-4o",
+ "route_api_model": "gpt-4.1",
"golden_db": null,
"llm_db": null,
"work_dir_golden": null,
@@ -15352,20 +15172,20 @@
}
},
"vendor": "openai",
- "started_at": "2025-10-19T21:28:34.267850900Z",
- "finished_at": "2025-10-19T21:28:34.267850900Z"
+ "started_at": "2025-10-19T21:28:33.435872Z",
+ "finished_at": "2025-10-19T21:28:33.435872Z"
},
"t_021_multi_column_index": {
"hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4",
"task": "t_021_multi_column_index",
"lang": "csharp",
"golden_published": false,
- "model_name": "GPT-4o",
+ "model_name": "GPT-4.1",
"total_tests": 1,
"passed_tests": 0,
"llm_output": null,
"category": "schema",
- "route_api_model": "gpt-4o",
+ "route_api_model": "gpt-4.1",
"golden_db": null,
"llm_db": null,
"work_dir_golden": null,
@@ -15381,12 +15201,18 @@
}
},
"vendor": "openai",
- "started_at": "2025-10-19T21:28:34.267654800Z",
- "finished_at": "2025-10-19T21:28:34.267654800Z"
- },
- "t_007_crud": {
+ "started_at": "2025-10-19T21:28:33.433206700Z",
+ "finished_at": "2025-10-19T21:28:33.433206700Z"
+ }
+ }
+ },
+ {
+ "name": "GPT-4o",
+ "route_api_model": "gpt-4o",
+ "tasks": {
+ "t_000_empty_reducers": {
"hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4",
- "task": "t_007_crud",
+ "task": "t_000_empty_reducers",
"lang": "csharp",
"golden_published": false,
"model_name": "GPT-4o",
@@ -15410,19 +15236,19 @@
}
},
"vendor": "openai",
- "started_at": "2025-10-19T21:28:34.269670600Z",
- "finished_at": "2025-10-19T21:28:34.269670600Z"
+ "started_at": "2025-10-19T21:28:34.269329Z",
+ "finished_at": "2025-10-19T21:28:34.269329Z"
},
- "t_012_spacetime_product_type": {
+ "t_001_basic_tables": {
"hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4",
- "task": "t_012_spacetime_product_type",
+ "task": "t_001_basic_tables",
"lang": "csharp",
"golden_published": false,
"model_name": "GPT-4o",
"total_tests": 1,
"passed_tests": 0,
"llm_output": null,
- "category": "schema",
+ "category": "basics",
"route_api_model": "gpt-4o",
"golden_db": null,
"llm_db": null,
@@ -15439,19 +15265,19 @@
}
},
"vendor": "openai",
- "started_at": "2025-10-19T21:28:34.265747400Z",
- "finished_at": "2025-10-19T21:28:34.265747400Z"
+ "started_at": "2025-10-19T21:28:34.267104500Z",
+ "finished_at": "2025-10-19T21:28:34.267104500Z"
},
- "t_016_sum_type_columns": {
+ "t_002_scheduled_table": {
"hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4",
- "task": "t_016_sum_type_columns",
+ "task": "t_002_scheduled_table",
"lang": "csharp",
"golden_published": false,
"model_name": "GPT-4o",
"total_tests": 1,
"passed_tests": 0,
"llm_output": null,
- "category": "schema",
+ "category": "basics",
"route_api_model": "gpt-4o",
"golden_db": null,
"llm_db": null,
@@ -15468,19 +15294,19 @@
}
},
"vendor": "openai",
- "started_at": "2025-10-19T21:28:34.268242400Z",
- "finished_at": "2025-10-19T21:28:34.268242400Z"
+ "started_at": "2025-10-19T21:28:34.268046600Z",
+ "finished_at": "2025-10-19T21:28:34.268046600Z"
},
- "t_013_spacetime_sum_type": {
+ "t_003_struct_in_table": {
"hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4",
- "task": "t_013_spacetime_sum_type",
+ "task": "t_003_struct_in_table",
"lang": "csharp",
"golden_published": false,
"model_name": "GPT-4o",
"total_tests": 1,
"passed_tests": 0,
"llm_output": null,
- "category": "schema",
+ "category": "basics",
"route_api_model": "gpt-4o",
"golden_db": null,
"llm_db": null,
@@ -15497,12 +15323,12 @@
}
},
"vendor": "openai",
- "started_at": "2025-10-19T21:28:34.264860500Z",
- "finished_at": "2025-10-19T21:28:34.264860500Z"
+ "started_at": "2025-10-19T21:28:34.269948300Z",
+ "finished_at": "2025-10-19T21:28:34.269948300Z"
},
- "t_000_empty_reducers": {
+ "t_004_insert": {
"hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4",
- "task": "t_000_empty_reducers",
+ "task": "t_004_insert",
"lang": "csharp",
"golden_published": false,
"model_name": "GPT-4o",
@@ -15526,12 +15352,12 @@
}
},
"vendor": "openai",
- "started_at": "2025-10-19T21:28:34.269329Z",
- "finished_at": "2025-10-19T21:28:34.269329Z"
+ "started_at": "2025-10-19T21:28:34.268769400Z",
+ "finished_at": "2025-10-19T21:28:34.268769400Z"
},
- "t_011_helper_function": {
+ "t_005_update": {
"hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4",
- "task": "t_011_helper_function",
+ "task": "t_005_update",
"lang": "csharp",
"golden_published": false,
"model_name": "GPT-4o",
@@ -15555,12 +15381,12 @@
}
},
"vendor": "openai",
- "started_at": "2025-10-19T21:28:34.268445400Z",
- "finished_at": "2025-10-19T21:28:34.268445400Z"
+ "started_at": "2025-10-19T21:28:34.265366Z",
+ "finished_at": "2025-10-19T21:28:34.265366Z"
},
- "t_005_update": {
+ "t_006_delete": {
"hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4",
- "task": "t_005_update",
+ "task": "t_006_delete",
"lang": "csharp",
"golden_published": false,
"model_name": "GPT-4o",
@@ -15584,12 +15410,12 @@
}
},
"vendor": "openai",
- "started_at": "2025-10-19T21:28:34.265366Z",
- "finished_at": "2025-10-19T21:28:34.265366Z"
+ "started_at": "2025-10-19T21:28:34.265956300Z",
+ "finished_at": "2025-10-19T21:28:34.265956300Z"
},
- "t_006_delete": {
+ "t_007_crud": {
"hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4",
- "task": "t_006_delete",
+ "task": "t_007_crud",
"lang": "csharp",
"golden_published": false,
"model_name": "GPT-4o",
@@ -15613,12 +15439,12 @@
}
},
"vendor": "openai",
- "started_at": "2025-10-19T21:28:34.265956300Z",
- "finished_at": "2025-10-19T21:28:34.265956300Z"
+ "started_at": "2025-10-19T21:28:34.269670600Z",
+ "finished_at": "2025-10-19T21:28:34.269670600Z"
},
- "t_003_struct_in_table": {
+ "t_008_index_lookup": {
"hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4",
- "task": "t_003_struct_in_table",
+ "task": "t_008_index_lookup",
"lang": "csharp",
"golden_published": false,
"model_name": "GPT-4o",
@@ -15642,12 +15468,12 @@
}
},
"vendor": "openai",
- "started_at": "2025-10-19T21:28:34.269948300Z",
- "finished_at": "2025-10-19T21:28:34.269948300Z"
+ "started_at": "2025-10-19T21:28:34.267457800Z",
+ "finished_at": "2025-10-19T21:28:34.267457800Z"
},
- "t_001_basic_tables": {
+ "t_009_init": {
"hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4",
- "task": "t_001_basic_tables",
+ "task": "t_009_init",
"lang": "csharp",
"golden_published": false,
"model_name": "GPT-4o",
@@ -15671,12 +15497,12 @@
}
},
"vendor": "openai",
- "started_at": "2025-10-19T21:28:34.267104500Z",
- "finished_at": "2025-10-19T21:28:34.267104500Z"
+ "started_at": "2025-10-19T21:28:34.270319500Z",
+ "finished_at": "2025-10-19T21:28:34.270319500Z"
},
- "t_009_init": {
+ "t_010_connect": {
"hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4",
- "task": "t_009_init",
+ "task": "t_010_connect",
"lang": "csharp",
"golden_published": false,
"model_name": "GPT-4o",
@@ -15700,12 +15526,12 @@
}
},
"vendor": "openai",
- "started_at": "2025-10-19T21:28:34.270319500Z",
- "finished_at": "2025-10-19T21:28:34.270319500Z"
+ "started_at": "2025-10-19T21:28:34.270559900Z",
+ "finished_at": "2025-10-19T21:28:34.270559900Z"
},
- "t_008_index_lookup": {
+ "t_011_helper_function": {
"hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4",
- "task": "t_008_index_lookup",
+ "task": "t_011_helper_function",
"lang": "csharp",
"golden_published": false,
"model_name": "GPT-4o",
@@ -15729,19 +15555,19 @@
}
},
"vendor": "openai",
- "started_at": "2025-10-19T21:28:34.267457800Z",
- "finished_at": "2025-10-19T21:28:34.267457800Z"
+ "started_at": "2025-10-19T21:28:34.268445400Z",
+ "finished_at": "2025-10-19T21:28:34.268445400Z"
},
- "t_010_connect": {
+ "t_012_spacetime_product_type": {
"hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4",
- "task": "t_010_connect",
+ "task": "t_012_spacetime_product_type",
"lang": "csharp",
"golden_published": false,
"model_name": "GPT-4o",
"total_tests": 1,
"passed_tests": 0,
"llm_output": null,
- "category": "basics",
+ "category": "schema",
"route_api_model": "gpt-4o",
"golden_db": null,
"llm_db": null,
@@ -15758,8 +15584,66 @@
}
},
"vendor": "openai",
- "started_at": "2025-10-19T21:28:34.270559900Z",
- "finished_at": "2025-10-19T21:28:34.270559900Z"
+ "started_at": "2025-10-19T21:28:34.265747400Z",
+ "finished_at": "2025-10-19T21:28:34.265747400Z"
+ },
+ "t_013_spacetime_sum_type": {
+ "hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4",
+ "task": "t_013_spacetime_sum_type",
+ "lang": "csharp",
+ "golden_published": false,
+ "model_name": "GPT-4o",
+ "total_tests": 1,
+ "passed_tests": 0,
+ "llm_output": null,
+ "category": "schema",
+ "route_api_model": "gpt-4o",
+ "golden_db": null,
+ "llm_db": null,
+ "work_dir_golden": null,
+ "work_dir_llm": null,
+ "scorer_details": {
+ "publish_error": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "POST https://api.openai.com/v1/responses -> 400 Bad Request: {\n \"error\": {\n \"message\": \"Unsupported parameter: 'reasoning.effort' is not supported with this model.\",\n \"type\": \"invalid_request_error\",\n \"param\": \"reasoning.effort\",\n \"code\": \"unsupported_parameter\"\n }\n}",
+ "phase": "build_or_publish"
+ }
+ }
+ },
+ "vendor": "openai",
+ "started_at": "2025-10-19T21:28:34.264860500Z",
+ "finished_at": "2025-10-19T21:28:34.264860500Z"
+ },
+ "t_014_elementary_columns": {
+ "hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4",
+ "task": "t_014_elementary_columns",
+ "lang": "csharp",
+ "golden_published": false,
+ "model_name": "GPT-4o",
+ "total_tests": 1,
+ "passed_tests": 0,
+ "llm_output": null,
+ "category": "schema",
+ "route_api_model": "gpt-4o",
+ "golden_db": null,
+ "llm_db": null,
+ "work_dir_golden": null,
+ "work_dir_llm": null,
+ "scorer_details": {
+ "publish_error": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "POST https://api.openai.com/v1/responses -> 400 Bad Request: {\n \"error\": {\n \"message\": \"Unsupported parameter: 'reasoning.effort' is not supported with this model.\",\n \"type\": \"invalid_request_error\",\n \"param\": \"reasoning.effort\",\n \"code\": \"unsupported_parameter\"\n }\n}",
+ "phase": "build_or_publish"
+ }
+ }
+ },
+ "vendor": "openai",
+ "started_at": "2025-10-19T21:28:34.266545400Z",
+ "finished_at": "2025-10-19T21:28:34.266545400Z"
},
"t_015_product_type_columns": {
"hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4",
@@ -15790,6 +15674,64 @@
"started_at": "2025-10-19T21:28:34.271054500Z",
"finished_at": "2025-10-19T21:28:34.271054500Z"
},
+ "t_016_sum_type_columns": {
+ "hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4",
+ "task": "t_016_sum_type_columns",
+ "lang": "csharp",
+ "golden_published": false,
+ "model_name": "GPT-4o",
+ "total_tests": 1,
+ "passed_tests": 0,
+ "llm_output": null,
+ "category": "schema",
+ "route_api_model": "gpt-4o",
+ "golden_db": null,
+ "llm_db": null,
+ "work_dir_golden": null,
+ "work_dir_llm": null,
+ "scorer_details": {
+ "publish_error": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "POST https://api.openai.com/v1/responses -> 400 Bad Request: {\n \"error\": {\n \"message\": \"Unsupported parameter: 'reasoning.effort' is not supported with this model.\",\n \"type\": \"invalid_request_error\",\n \"param\": \"reasoning.effort\",\n \"code\": \"unsupported_parameter\"\n }\n}",
+ "phase": "build_or_publish"
+ }
+ }
+ },
+ "vendor": "openai",
+ "started_at": "2025-10-19T21:28:34.268242400Z",
+ "finished_at": "2025-10-19T21:28:34.268242400Z"
+ },
+ "t_017_scheduled_columns": {
+ "hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4",
+ "task": "t_017_scheduled_columns",
+ "lang": "csharp",
+ "golden_published": false,
+ "model_name": "GPT-4o",
+ "total_tests": 1,
+ "passed_tests": 0,
+ "llm_output": null,
+ "category": "schema",
+ "route_api_model": "gpt-4o",
+ "golden_db": null,
+ "llm_db": null,
+ "work_dir_golden": null,
+ "work_dir_llm": null,
+ "scorer_details": {
+ "publish_error": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "POST https://api.openai.com/v1/responses -> 400 Bad Request: {\n \"error\": {\n \"message\": \"Unsupported parameter: 'reasoning.effort' is not supported with this model.\",\n \"type\": \"invalid_request_error\",\n \"param\": \"reasoning.effort\",\n \"code\": \"unsupported_parameter\"\n }\n}",
+ "phase": "build_or_publish"
+ }
+ }
+ },
+ "vendor": "openai",
+ "started_at": "2025-10-19T21:28:34.267850900Z",
+ "finished_at": "2025-10-19T21:28:34.267850900Z"
+ },
"t_018_constraints": {
"hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4",
"task": "t_018_constraints",
@@ -15819,16 +15761,16 @@
"started_at": "2025-10-19T21:28:34.265067700Z",
"finished_at": "2025-10-19T21:28:34.265067700Z"
},
- "t_004_insert": {
+ "t_019_many_to_many": {
"hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4",
- "task": "t_004_insert",
+ "task": "t_019_many_to_many",
"lang": "csharp",
"golden_published": false,
"model_name": "GPT-4o",
"total_tests": 1,
"passed_tests": 0,
"llm_output": null,
- "category": "basics",
+ "category": "schema",
"route_api_model": "gpt-4o",
"golden_db": null,
"llm_db": null,
@@ -15845,8 +15787,66 @@
}
},
"vendor": "openai",
- "started_at": "2025-10-19T21:28:34.268769400Z",
- "finished_at": "2025-10-19T21:28:34.268769400Z"
+ "started_at": "2025-10-19T21:28:34.266156200Z",
+ "finished_at": "2025-10-19T21:28:34.266156200Z"
+ },
+ "t_020_ecs": {
+ "hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4",
+ "task": "t_020_ecs",
+ "lang": "csharp",
+ "golden_published": false,
+ "model_name": "GPT-4o",
+ "total_tests": 1,
+ "passed_tests": 0,
+ "llm_output": null,
+ "category": "schema",
+ "route_api_model": "gpt-4o",
+ "golden_db": null,
+ "llm_db": null,
+ "work_dir_golden": null,
+ "work_dir_llm": null,
+ "scorer_details": {
+ "publish_error": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "POST https://api.openai.com/v1/responses -> 400 Bad Request: {\n \"error\": {\n \"message\": \"Unsupported parameter: 'reasoning.effort' is not supported with this model.\",\n \"type\": \"invalid_request_error\",\n \"param\": \"reasoning.effort\",\n \"code\": \"unsupported_parameter\"\n }\n}",
+ "phase": "build_or_publish"
+ }
+ }
+ },
+ "vendor": "openai",
+ "started_at": "2025-10-19T21:28:34.266767800Z",
+ "finished_at": "2025-10-19T21:28:34.266767800Z"
+ },
+ "t_021_multi_column_index": {
+ "hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4",
+ "task": "t_021_multi_column_index",
+ "lang": "csharp",
+ "golden_published": false,
+ "model_name": "GPT-4o",
+ "total_tests": 1,
+ "passed_tests": 0,
+ "llm_output": null,
+ "category": "schema",
+ "route_api_model": "gpt-4o",
+ "golden_db": null,
+ "llm_db": null,
+ "work_dir_golden": null,
+ "work_dir_llm": null,
+ "scorer_details": {
+ "publish_error": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "POST https://api.openai.com/v1/responses -> 400 Bad Request: {\n \"error\": {\n \"message\": \"Unsupported parameter: 'reasoning.effort' is not supported with this model.\",\n \"type\": \"invalid_request_error\",\n \"param\": \"reasoning.effort\",\n \"code\": \"unsupported_parameter\"\n }\n}",
+ "phase": "build_or_publish"
+ }
+ }
+ },
+ "vendor": "openai",
+ "started_at": "2025-10-19T21:28:34.267654800Z",
+ "finished_at": "2025-10-19T21:28:34.267654800Z"
}
}
}
@@ -15854,180 +15854,180 @@
}
],
"golden_answers": {
- "basics/t_003_struct_in_table": {
- "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Type]\r\n public partial struct Position\r\n {\r\n public int X;\r\n public int Y;\r\n }\r\n\r\n [Table(Name = \"entities\")]\r\n public partial struct Entity\r\n {\r\n [PrimaryKey] public int Id;\r\n public Position Pos;\r\n }\r\n}\r\n",
- "syntax": "csharp"
- },
- "t_001_basic_tables": {
- "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"users\")]\n public partial struct Users\n {\n [PrimaryKey] public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [Table(Name = \"products\")]\n public partial struct Products\n {\n [PrimaryKey] public int Id;\n public string Title;\n public float Price;\n public bool InStock;\n }\n\n [Table(Name = \"notes\")]\n public partial struct Notes\n {\n [PrimaryKey] public int Id;\n public string Body;\n public long Rating;\n public bool Pinned;\n }\n}\n",
- "syntax": "csharp"
- },
- "t_015_product_type_columns": {
- "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Type]\n public partial struct Address\n {\n public string Street;\n public int Zip;\n }\n\n [Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [Table(Name = \"profiles\")]\n public partial struct Profile\n {\n [PrimaryKey] public int Id;\n public Address Home;\n public Address Work;\n public Position Pos;\n }\n\n [Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.profiles.Insert(new Profile {\n Id = 1,\n Home = new Address { Street = \"1 Main\", Zip = 11111 },\n Work = new Address { Street = \"2 Broad\", Zip = 22222 },\n Pos = new Position { X = 7, Y = 9 }\n });\n }\n}\n",
- "syntax": "csharp"
- },
- "schema/t_021_multi_column_index": {
- "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Name = \"logs\")]\r\n [SpacetimeDB.Index.BTree(Name = \"by_user_day\", Columns = new[] { nameof(UserId), nameof(Day) })]\r\n public partial struct Log\r\n {\r\n [PrimaryKey] public int Id;\r\n public int UserId;\r\n public int Day;\r\n public string Message;\r\n }\r\n\r\n [Reducer]\r\n public static void Seed(ReducerContext ctx)\r\n {\r\n ctx.Db.logs.Insert(new Log { Id = 1, UserId = 7, Day = 1, Message = \"a\" });\r\n ctx.Db.logs.Insert(new Log { Id = 2, UserId = 7, Day = 2, Message = \"b\" });\r\n ctx.Db.logs.Insert(new Log { Id = 3, UserId = 9, Day = 1, Message = \"c\" });\r\n }\r\n}\r\n",
+ "basics/t_000_empty_reducers": {
+ "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Reducer]\r\n public static void EmptyReducer_NoArgs(ReducerContext ctx) { }\r\n\r\n [Reducer]\r\n public static void EmptyReducer_WithInt(ReducerContext ctx, int count) { }\r\n\r\n [Reducer]\r\n public static void EmptyReducer_WithString(ReducerContext ctx, string name) { }\r\n\r\n [Reducer]\r\n public static void EmptyReducer_WithTwoArgs(ReducerContext ctx, int count, string name) { }\r\n\r\n [Reducer]\r\n public static void EmptyReducer_WithThreeArgs(ReducerContext ctx, bool active, float ratio, string label) { }\r\n}",
"syntax": "csharp"
},
- "schema/t_016_sum_type_columns": {
- "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Type]\r\n public partial struct Circle { public int Radius; }\r\n\r\n [Type]\r\n public partial struct Rectangle { public int Width; public int Height; }\r\n\r\n [Type]\r\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> {}\r\n\r\n [Table(Name = \"drawings\")]\r\n public partial struct Drawing\r\n {\r\n [PrimaryKey] public int Id;\r\n public Shape A;\r\n public Shape B;\r\n }\r\n\r\n [Reducer]\r\n public static void Seed(ReducerContext ctx)\r\n {\r\n ctx.Db.drawings.Insert(new Drawing {\r\n Id = 1,\r\n A = new Shape.Circle(new Circle { Radius = 10 }),\r\n B = new Shape.Rectangle(new Rectangle { Width = 4, Height = 6 })\r\n });\r\n }\r\n}\r\n",
+ "basics/t_001_basic_tables": {
+ "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Name = \"users\")]\r\n public partial struct Users\r\n {\r\n [PrimaryKey] public int Id;\r\n public string Name;\r\n public int Age;\r\n public bool Active;\r\n }\r\n\r\n [Table(Name = \"products\")]\r\n public partial struct Products\r\n {\r\n [PrimaryKey] public int Id;\r\n public string Title;\r\n public float Price;\r\n public bool InStock;\r\n }\r\n\r\n [Table(Name = \"notes\")]\r\n public partial struct Notes\r\n {\r\n [PrimaryKey] public int Id;\r\n public string Body;\r\n public long Rating;\r\n public bool Pinned;\r\n }\r\n}\r\n",
"syntax": "csharp"
},
- "t_017_scheduled_columns": {
- "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"tick_timer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [PrimaryKey, AutoInc] public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [Reducer]\n public static void Tick(ReducerContext ctx, TickTimer schedule) { }\n\n [Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n var interval = new TimeDuration { Microseconds = 50_000 };\n ctx.Db.tick_timer.Insert(new TickTimer\n {\n ScheduledId = 0,\n ScheduledAt = new ScheduleAt.Interval(interval)\n });\n }\n}\n",
+ "basics/t_002_scheduled_table": {
+ "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Name = \"tick_timer\", Scheduled = nameof(Tick), ScheduledAt = nameof(TickTimer.ScheduledAt))]\r\n public partial struct TickTimer\r\n {\r\n [PrimaryKey, AutoInc] public ulong ScheduledId;\r\n public ScheduleAt ScheduledAt;\r\n }\r\n\r\n [Reducer]\r\n public static void Tick(ReducerContext ctx, TickTimer timer) { }\r\n\r\n [Reducer(ReducerKind.Init)]\r\n public static void Init(ReducerContext ctx)\r\n {\r\n var interval = new TimeDuration { Microseconds = 50_000 };\r\n ctx.Db.tick_timer.Insert(new TickTimer\r\n {\r\n ScheduledAt = new ScheduleAt.Interval(interval)\r\n });\r\n }\r\n}",
"syntax": "csharp"
},
- "t_008_index_lookup": {
- "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"users\")]\n public partial struct User\n {\n [PrimaryKey] public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [Table(Name = \"results\")]\n public partial struct Result\n {\n [PrimaryKey] public int Id;\n public string Name;\n }\n\n [Reducer]\n public static void LookupUserName(ReducerContext ctx, int id)\n {\n var u = ctx.Db.users.Id.Find(id);\n if (u.HasValue)\n {\n var row = u.Value;\n ctx.Db.results.Insert(new Result { Id = row.Id, Name = row.Name });\n }\n }\n}\n",
+ "basics/t_003_struct_in_table": {
+ "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Type]\r\n public partial struct Position\r\n {\r\n public int X;\r\n public int Y;\r\n }\r\n\r\n [Table(Name = \"entities\")]\r\n public partial struct Entity\r\n {\r\n [PrimaryKey] public int Id;\r\n public Position Pos;\r\n }\r\n}\r\n",
"syntax": "csharp"
},
- "schema/t_019_many_to_many": {
- "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Name = \"users\")]\r\n public partial struct User\r\n {\r\n [PrimaryKey] public int UserId;\r\n public string Name;\r\n }\r\n\r\n [Table(Name = \"groups\")]\r\n public partial struct Group\r\n {\r\n [PrimaryKey] public int GroupId;\r\n public string Title;\r\n }\r\n\r\n [Table(Name = \"memberships\")]\r\n [SpacetimeDB.Index.BTree(Name = \"by_user\", Columns = new[] { nameof(UserId) })]\r\n [SpacetimeDB.Index.BTree(Name = \"by_group\", Columns = new[] { nameof(GroupId) })]\r\n public partial struct Membership\r\n {\r\n [PrimaryKey] public int Id;\r\n public int UserId;\r\n public int GroupId;\r\n }\r\n\r\n [Reducer]\r\n public static void Seed(ReducerContext ctx)\r\n {\r\n ctx.Db.users.Insert(new User { UserId = 1, Name = \"Alice\" });\r\n ctx.Db.users.Insert(new User { UserId = 2, Name = \"Bob\" });\r\n\r\n ctx.Db.groups.Insert(new Group { GroupId = 10, Title = \"Admin\" });\r\n ctx.Db.groups.Insert(new Group { GroupId = 20, Title = \"Dev\" });\r\n\r\n ctx.Db.memberships.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\r\n ctx.Db.memberships.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\r\n ctx.Db.memberships.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\r\n }\r\n}\r\n",
+ "basics/t_004_insert": {
+ "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Name = \"users\")]\r\n public partial struct User\r\n {\r\n [PrimaryKey] public int Id;\r\n public string Name;\r\n public int Age;\r\n public bool Active;\r\n }\r\n\r\n [Reducer]\r\n public static void InsertUser(ReducerContext ctx, int id, string name, int age, bool active)\r\n {\r\n ctx.Db.users.Insert(new User { Id = id, Name = name, Age = age, Active = active });\r\n }\r\n}\r\n",
"syntax": "csharp"
},
- "schema/t_014_elementary_columns": {
- "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Name = \"primitives\")]\r\n public partial struct Primitive\r\n {\r\n [PrimaryKey] public int Id;\r\n public int Count;\r\n public long Total;\r\n public float Price;\r\n public double Ratio;\r\n public bool Active;\r\n public string Name;\r\n }\r\n\r\n [Reducer]\r\n public static void Seed(ReducerContext ctx)\r\n {\r\n ctx.Db.primitives.Insert(new Primitive {\r\n Id = 1,\r\n Count = 2,\r\n Total = 3000000000,\r\n Price = 1.5f,\r\n Ratio = 2.25,\r\n Active = true,\r\n Name = \"Alice\"\r\n });\r\n }\r\n}\r\n",
+ "basics/t_005_update": {
+ "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Name = \"users\")]\r\n public partial struct User\r\n {\r\n [PrimaryKey] public int Id;\r\n public string Name;\r\n public int Age;\r\n public bool Active;\r\n }\r\n\r\n [Reducer]\r\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\r\n {\r\n ctx.Db.users.Id.Update(new User { Id = id, Name = name, Age = age, Active = active });\r\n }\r\n}\r\n",
"syntax": "csharp"
},
- "t_010_connect": {
- "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"events\")]\n public partial struct Event\n {\n [PrimaryKey, AutoInc] public int Id;\n public string Kind;\n }\n\n [Reducer(ReducerKind.ClientConnected)]\n public static void ClientConnected(ReducerContext ctx)\n {\n ctx.Db.events.Insert(new Event { Kind = \"connected\" });\n }\n\n [Reducer(ReducerKind.ClientDisconnected)]\n public static void ClientDisconnected(ReducerContext ctx)\n {\n ctx.Db.events.Insert(new Event { Kind = \"disconnected\" });\n }\n}\n",
+ "basics/t_006_delete": {
+ "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Name = \"users\")]\r\n public partial struct User\r\n {\r\n [PrimaryKey] public int Id;\r\n public string Name;\r\n public int Age;\r\n public bool Active;\r\n }\r\n\r\n [Reducer]\r\n public static void DeleteUser(ReducerContext ctx, int id)\r\n {\r\n ctx.Db.users.Id.Delete(id);\r\n }\r\n}\r\n",
"syntax": "csharp"
},
- "schema/t_012_spacetime_product_type": {
- "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Type]\r\n public partial struct Score\r\n {\r\n public int Left;\r\n public int Right;\r\n }\r\n\r\n [Table(Name = \"results\")]\r\n public partial struct Result\r\n {\r\n [PrimaryKey] public int Id;\r\n public Score Value;\r\n }\r\n\r\n [Reducer]\r\n public static void SetScore(ReducerContext ctx, int id, int left, int right)\r\n {\r\n ctx.Db.results.Insert(new Result { Id = id, Value = new Score { Left = left, Right = right } });\r\n }\r\n}\r\n",
+ "basics/t_007_crud": {
+ "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Name = \"users\")]\r\n public partial struct User\r\n {\r\n [PrimaryKey] public int Id;\r\n public string Name;\r\n public int Age;\r\n public bool Active;\r\n }\r\n\r\n [Reducer]\r\n public static void Crud(ReducerContext ctx)\r\n {\r\n ctx.Db.users.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\r\n ctx.Db.users.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\r\n ctx.Db.users.Id.Update(new User { Id = 1, Name = \"Alice2\", Age = 31, Active = false });\r\n ctx.Db.users.Id.Delete(2);\r\n }\r\n}\r\n",
"syntax": "csharp"
},
- "t_011_helper_function": {
- "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"results\")]\n public partial struct Result\n {\n [PrimaryKey] public int Id;\n public int Sum;\n }\n\n static int Add(int a, int b) => a + b;\n\n [Reducer]\n public static void ComputeSum(ReducerContext ctx, int id, int a, int b)\n {\n ctx.Db.results.Insert(new Result { Id = id, Sum = Add(a, b) });\n }\n}\n",
+ "basics/t_008_index_lookup": {
+ "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Name = \"users\")]\r\n public partial struct User\r\n {\r\n [PrimaryKey] public int Id;\r\n public string Name;\r\n public int Age;\r\n public bool Active;\r\n }\r\n\r\n [Table(Name = \"results\")]\r\n public partial struct Result\r\n {\r\n [PrimaryKey] public int Id;\r\n public string Name;\r\n }\r\n\r\n [Reducer]\r\n public static void LookupUserName(ReducerContext ctx, int id)\r\n {\r\n var u = ctx.Db.users.Id.Find(id);\r\n if (u.HasValue)\r\n {\r\n var row = u.Value;\r\n ctx.Db.results.Insert(new Result { Id = row.Id, Name = row.Name });\r\n }\r\n }\r\n}\r\n",
"syntax": "csharp"
},
- "t_020_ecs": {
- "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"entities\")]\n public partial struct Entity { [PrimaryKey] public int Id; }\n\n [Table(Name = \"positions\")]\n public partial struct Position\n {\n [PrimaryKey] public int EntityId;\n public int X;\n public int Y;\n }\n\n [Table(Name = \"velocities\")]\n public partial struct Velocity\n {\n [PrimaryKey] public int EntityId;\n public int VX;\n public int VY;\n }\n\n [Table(Name = \"next_positions\")]\n public partial struct NextPosition\n {\n [PrimaryKey] public int EntityId;\n public int X;\n public int Y;\n }\n\n [Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.entities.Insert(new Entity { Id = 1 });\n ctx.Db.entities.Insert(new Entity { Id = 2 });\n\n ctx.Db.positions.Insert(new Position { EntityId = 1, X = 0, Y = 0 });\n ctx.Db.positions.Insert(new Position { EntityId = 2, X = 10, Y = 0 });\n\n ctx.Db.velocities.Insert(new Velocity { EntityId = 1, VX = 1, VY = 0 });\n ctx.Db.velocities.Insert(new Velocity { EntityId = 2, VX = -2, VY = 3 });\n }\n\n [Reducer]\n public static void Step(ReducerContext ctx)\n {\n foreach (var p in ctx.Db.positions.Iter())\n {\n var velOpt = ctx.Db.velocities.EntityId.Find(p.EntityId);\n if (!velOpt.HasValue) continue;\n\n var np = new NextPosition {\n EntityId = p.EntityId,\n X = p.X + velOpt.Value.VX,\n Y = p.Y + velOpt.Value.VY\n };\n\n if (ctx.Db.next_positions.EntityId.Find(p.EntityId).HasValue)\n ctx.Db.next_positions.EntityId.Update(np);\n else\n ctx.Db.next_positions.Insert(np);\n }\n }\n}\n",
+ "basics/t_009_init": {
+ "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Name = \"users\")]\r\n public partial struct User\r\n {\r\n [PrimaryKey] public int Id;\r\n public string Name;\r\n public int Age;\r\n public bool Active;\r\n }\r\n\r\n [Reducer(ReducerKind.Init)]\r\n public static void Init(ReducerContext ctx)\r\n {\r\n ctx.Db.users.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\r\n ctx.Db.users.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\r\n }\r\n}\r\n",
"syntax": "csharp"
},
- "basics/t_001_basic_tables": {
- "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Name = \"users\")]\r\n public partial struct Users\r\n {\r\n [PrimaryKey] public int Id;\r\n public string Name;\r\n public int Age;\r\n public bool Active;\r\n }\r\n\r\n [Table(Name = \"products\")]\r\n public partial struct Products\r\n {\r\n [PrimaryKey] public int Id;\r\n public string Title;\r\n public float Price;\r\n public bool InStock;\r\n }\r\n\r\n [Table(Name = \"notes\")]\r\n public partial struct Notes\r\n {\r\n [PrimaryKey] public int Id;\r\n public string Body;\r\n public long Rating;\r\n public bool Pinned;\r\n }\r\n}\r\n",
+ "basics/t_010_connect": {
+ "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Name = \"events\")]\r\n public partial struct Event\r\n {\r\n [PrimaryKey, AutoInc] public int Id;\r\n public string Kind;\r\n }\r\n\r\n [Reducer(ReducerKind.ClientConnected)]\r\n public static void ClientConnected(ReducerContext ctx)\r\n {\r\n ctx.Db.events.Insert(new Event { Kind = \"connected\" });\r\n }\r\n\r\n [Reducer(ReducerKind.ClientDisconnected)]\r\n public static void ClientDisconnected(ReducerContext ctx)\r\n {\r\n ctx.Db.events.Insert(new Event { Kind = \"disconnected\" });\r\n }\r\n}\r\n",
"syntax": "csharp"
},
- "basics/t_000_empty_reducers": {
- "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Reducer]\r\n public static void EmptyReducer_NoArgs(ReducerContext ctx) { }\r\n\r\n [Reducer]\r\n public static void EmptyReducer_WithInt(ReducerContext ctx, int count) { }\r\n\r\n [Reducer]\r\n public static void EmptyReducer_WithString(ReducerContext ctx, string name) { }\r\n\r\n [Reducer]\r\n public static void EmptyReducer_WithTwoArgs(ReducerContext ctx, int count, string name) { }\r\n\r\n [Reducer]\r\n public static void EmptyReducer_WithThreeArgs(ReducerContext ctx, bool active, float ratio, string label) { }\r\n}",
+ "basics/t_011_helper_function": {
+ "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Name = \"results\")]\r\n public partial struct Result\r\n {\r\n [PrimaryKey] public int Id;\r\n public int Sum;\r\n }\r\n\r\n static int Add(int a, int b) => a + b;\r\n\r\n [Reducer]\r\n public static void ComputeSum(ReducerContext ctx, int id, int a, int b)\r\n {\r\n ctx.Db.results.Insert(new Result { Id = id, Sum = Add(a, b) });\r\n }\r\n}\r\n",
"syntax": "csharp"
},
- "basics/t_004_insert": {
- "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Name = \"users\")]\r\n public partial struct User\r\n {\r\n [PrimaryKey] public int Id;\r\n public string Name;\r\n public int Age;\r\n public bool Active;\r\n }\r\n\r\n [Reducer]\r\n public static void InsertUser(ReducerContext ctx, int id, string name, int age, bool active)\r\n {\r\n ctx.Db.users.Insert(new User { Id = id, Name = name, Age = age, Active = active });\r\n }\r\n}\r\n",
+ "schema/t_012_spacetime_product_type": {
+ "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Type]\r\n public partial struct Score\r\n {\r\n public int Left;\r\n public int Right;\r\n }\r\n\r\n [Table(Name = \"results\")]\r\n public partial struct Result\r\n {\r\n [PrimaryKey] public int Id;\r\n public Score Value;\r\n }\r\n\r\n [Reducer]\r\n public static void SetScore(ReducerContext ctx, int id, int left, int right)\r\n {\r\n ctx.Db.results.Insert(new Result { Id = id, Value = new Score { Left = left, Right = right } });\r\n }\r\n}\r\n",
"syntax": "csharp"
},
- "basics/t_006_delete": {
- "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Name = \"users\")]\r\n public partial struct User\r\n {\r\n [PrimaryKey] public int Id;\r\n public string Name;\r\n public int Age;\r\n public bool Active;\r\n }\r\n\r\n [Reducer]\r\n public static void DeleteUser(ReducerContext ctx, int id)\r\n {\r\n ctx.Db.users.Id.Delete(id);\r\n }\r\n}\r\n",
+ "schema/t_013_spacetime_sum_type": {
+ "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Type]\r\n public partial struct Circle { public int Radius; }\r\n\r\n [Type]\r\n public partial struct Rectangle { public int Width; public int Height; }\r\n\r\n [Type]\r\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> {}\r\n\r\n [Table(Name = \"results\")]\r\n public partial struct Result\r\n {\r\n [PrimaryKey] public int Id;\r\n public Shape Value;\r\n }\r\n\r\n [Reducer]\r\n public static void SetCircle(ReducerContext ctx, int id, int radius)\r\n {\r\n ctx.Db.results.Insert(new Result { Id = id, Value = new Shape.Circle(new Circle { Radius = radius }) });\r\n }\r\n}\r\n",
"syntax": "csharp"
},
- "t_012_spacetime_product_type": {
- "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Type]\n public partial struct Score\n {\n public int Left;\n public int Right;\n }\n\n [Table(Name = \"results\")]\n public partial struct Result\n {\n [PrimaryKey] public int Id;\n public Score Value;\n }\n\n [Reducer]\n public static void SetScore(ReducerContext ctx, int id, int left, int right)\n {\n ctx.Db.results.Insert(new Result { Id = id, Value = new Score { Left = left, Right = right } });\n }\n}\n",
+ "schema/t_014_elementary_columns": {
+ "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Name = \"primitives\")]\r\n public partial struct Primitive\r\n {\r\n [PrimaryKey] public int Id;\r\n public int Count;\r\n public long Total;\r\n public float Price;\r\n public double Ratio;\r\n public bool Active;\r\n public string Name;\r\n }\r\n\r\n [Reducer]\r\n public static void Seed(ReducerContext ctx)\r\n {\r\n ctx.Db.primitives.Insert(new Primitive {\r\n Id = 1,\r\n Count = 2,\r\n Total = 3000000000,\r\n Price = 1.5f,\r\n Ratio = 2.25,\r\n Active = true,\r\n Name = \"Alice\"\r\n });\r\n }\r\n}\r\n",
"syntax": "csharp"
},
- "t_002_scheduled_table": {
- "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"tick_timer\", Scheduled = nameof(Tick), ScheduledAt = nameof(TickTimer.ScheduledAt))]\n public partial struct TickTimer\n {\n [PrimaryKey, AutoInc] public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [Reducer]\n public static void Tick(ReducerContext ctx, TickTimer timer) { }\n\n [Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n var interval = new TimeDuration { Microseconds = 50_000 };\n ctx.Db.tick_timer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(interval)\n });\n }\n}",
+ "schema/t_015_product_type_columns": {
+ "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Type]\r\n public partial struct Address\r\n {\r\n public string Street;\r\n public int Zip;\r\n }\r\n\r\n [Type]\r\n public partial struct Position\r\n {\r\n public int X;\r\n public int Y;\r\n }\r\n\r\n [Table(Name = \"profiles\")]\r\n public partial struct Profile\r\n {\r\n [PrimaryKey] public int Id;\r\n public Address Home;\r\n public Address Work;\r\n public Position Pos;\r\n }\r\n\r\n [Reducer]\r\n public static void Seed(ReducerContext ctx)\r\n {\r\n ctx.Db.profiles.Insert(new Profile {\r\n Id = 1,\r\n Home = new Address { Street = \"1 Main\", Zip = 11111 },\r\n Work = new Address { Street = \"2 Broad\", Zip = 22222 },\r\n Pos = new Position { X = 7, Y = 9 }\r\n });\r\n }\r\n}\r\n",
"syntax": "csharp"
},
- "basics/t_002_scheduled_table": {
- "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Name = \"tick_timer\", Scheduled = nameof(Tick), ScheduledAt = nameof(TickTimer.ScheduledAt))]\r\n public partial struct TickTimer\r\n {\r\n [PrimaryKey, AutoInc] public ulong ScheduledId;\r\n public ScheduleAt ScheduledAt;\r\n }\r\n\r\n [Reducer]\r\n public static void Tick(ReducerContext ctx, TickTimer timer) { }\r\n\r\n [Reducer(ReducerKind.Init)]\r\n public static void Init(ReducerContext ctx)\r\n {\r\n var interval = new TimeDuration { Microseconds = 50_000 };\r\n ctx.Db.tick_timer.Insert(new TickTimer\r\n {\r\n ScheduledAt = new ScheduleAt.Interval(interval)\r\n });\r\n }\r\n}",
+ "schema/t_016_sum_type_columns": {
+ "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Type]\r\n public partial struct Circle { public int Radius; }\r\n\r\n [Type]\r\n public partial struct Rectangle { public int Width; public int Height; }\r\n\r\n [Type]\r\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> {}\r\n\r\n [Table(Name = \"drawings\")]\r\n public partial struct Drawing\r\n {\r\n [PrimaryKey] public int Id;\r\n public Shape A;\r\n public Shape B;\r\n }\r\n\r\n [Reducer]\r\n public static void Seed(ReducerContext ctx)\r\n {\r\n ctx.Db.drawings.Insert(new Drawing {\r\n Id = 1,\r\n A = new Shape.Circle(new Circle { Radius = 10 }),\r\n B = new Shape.Rectangle(new Rectangle { Width = 4, Height = 6 })\r\n });\r\n }\r\n}\r\n",
"syntax": "csharp"
},
- "schema/t_013_spacetime_sum_type": {
- "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Type]\r\n public partial struct Circle { public int Radius; }\r\n\r\n [Type]\r\n public partial struct Rectangle { public int Width; public int Height; }\r\n\r\n [Type]\r\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> {}\r\n\r\n [Table(Name = \"results\")]\r\n public partial struct Result\r\n {\r\n [PrimaryKey] public int Id;\r\n public Shape Value;\r\n }\r\n\r\n [Reducer]\r\n public static void SetCircle(ReducerContext ctx, int id, int radius)\r\n {\r\n ctx.Db.results.Insert(new Result { Id = id, Value = new Shape.Circle(new Circle { Radius = radius }) });\r\n }\r\n}\r\n",
+ "schema/t_017_scheduled_columns": {
+ "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Name = \"tick_timer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\r\n public partial struct TickTimer\r\n {\r\n [PrimaryKey, AutoInc] public ulong ScheduledId;\r\n public ScheduleAt ScheduledAt;\r\n }\r\n\r\n [Reducer]\r\n public static void Tick(ReducerContext ctx, TickTimer schedule) { }\r\n\r\n [Reducer(ReducerKind.Init)]\r\n public static void Init(ReducerContext ctx)\r\n {\r\n var interval = new TimeDuration { Microseconds = 50_000 };\r\n ctx.Db.tick_timer.Insert(new TickTimer\r\n {\r\n ScheduledId = 0,\r\n ScheduledAt = new ScheduleAt.Interval(interval)\r\n });\r\n }\r\n}\r\n",
"syntax": "csharp"
},
- "basics/t_010_connect": {
- "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Name = \"events\")]\r\n public partial struct Event\r\n {\r\n [PrimaryKey, AutoInc] public int Id;\r\n public string Kind;\r\n }\r\n\r\n [Reducer(ReducerKind.ClientConnected)]\r\n public static void ClientConnected(ReducerContext ctx)\r\n {\r\n ctx.Db.events.Insert(new Event { Kind = \"connected\" });\r\n }\r\n\r\n [Reducer(ReducerKind.ClientDisconnected)]\r\n public static void ClientDisconnected(ReducerContext ctx)\r\n {\r\n ctx.Db.events.Insert(new Event { Kind = \"disconnected\" });\r\n }\r\n}\r\n",
+ "schema/t_018_constraints": {
+ "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [SpacetimeDB.Table(Name = \"accounts\", Public = true)]\r\n [SpacetimeDB.Index.BTree(Name = \"by_name\", Columns = [nameof(Name)])]\r\n public partial struct Account\r\n {\r\n [SpacetimeDB.PrimaryKey] public int Id;\r\n [SpacetimeDB.Unique] public string Email;\r\n public string Name;\r\n }\r\n\r\n [SpacetimeDB.Reducer]\r\n public static void Seed(ReducerContext ctx)\r\n {\r\n ctx.Db.accounts.Insert(new Account { Id = 1, Email = \"a@example.com\", Name = \"Alice\" });\r\n ctx.Db.accounts.Insert(new Account { Id = 2, Email = \"b@example.com\", Name = \"Bob\" });\r\n }\r\n}",
"syntax": "csharp"
},
- "schema/t_017_scheduled_columns": {
- "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Name = \"tick_timer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\r\n public partial struct TickTimer\r\n {\r\n [PrimaryKey, AutoInc] public ulong ScheduledId;\r\n public ScheduleAt ScheduledAt;\r\n }\r\n\r\n [Reducer]\r\n public static void Tick(ReducerContext ctx, TickTimer schedule) { }\r\n\r\n [Reducer(ReducerKind.Init)]\r\n public static void Init(ReducerContext ctx)\r\n {\r\n var interval = new TimeDuration { Microseconds = 50_000 };\r\n ctx.Db.tick_timer.Insert(new TickTimer\r\n {\r\n ScheduledId = 0,\r\n ScheduledAt = new ScheduleAt.Interval(interval)\r\n });\r\n }\r\n}\r\n",
+ "schema/t_019_many_to_many": {
+ "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Name = \"users\")]\r\n public partial struct User\r\n {\r\n [PrimaryKey] public int UserId;\r\n public string Name;\r\n }\r\n\r\n [Table(Name = \"groups\")]\r\n public partial struct Group\r\n {\r\n [PrimaryKey] public int GroupId;\r\n public string Title;\r\n }\r\n\r\n [Table(Name = \"memberships\")]\r\n [SpacetimeDB.Index.BTree(Name = \"by_user\", Columns = new[] { nameof(UserId) })]\r\n [SpacetimeDB.Index.BTree(Name = \"by_group\", Columns = new[] { nameof(GroupId) })]\r\n public partial struct Membership\r\n {\r\n [PrimaryKey] public int Id;\r\n public int UserId;\r\n public int GroupId;\r\n }\r\n\r\n [Reducer]\r\n public static void Seed(ReducerContext ctx)\r\n {\r\n ctx.Db.users.Insert(new User { UserId = 1, Name = \"Alice\" });\r\n ctx.Db.users.Insert(new User { UserId = 2, Name = \"Bob\" });\r\n\r\n ctx.Db.groups.Insert(new Group { GroupId = 10, Title = \"Admin\" });\r\n ctx.Db.groups.Insert(new Group { GroupId = 20, Title = \"Dev\" });\r\n\r\n ctx.Db.memberships.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\r\n ctx.Db.memberships.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\r\n ctx.Db.memberships.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\r\n }\r\n}\r\n",
"syntax": "csharp"
},
- "t_004_insert": {
- "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"users\")]\n public partial struct User\n {\n [PrimaryKey] public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [Reducer]\n public static void InsertUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n ctx.Db.users.Insert(new User { Id = id, Name = name, Age = age, Active = active });\n }\n}\n",
+ "schema/t_020_ecs": {
+ "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Name = \"entities\")]\r\n public partial struct Entity { [PrimaryKey] public int Id; }\r\n\r\n [Table(Name = \"positions\")]\r\n public partial struct Position\r\n {\r\n [PrimaryKey] public int EntityId;\r\n public int X;\r\n public int Y;\r\n }\r\n\r\n [Table(Name = \"velocities\")]\r\n public partial struct Velocity\r\n {\r\n [PrimaryKey] public int EntityId;\r\n public int VX;\r\n public int VY;\r\n }\r\n\r\n [Table(Name = \"next_positions\")]\r\n public partial struct NextPosition\r\n {\r\n [PrimaryKey] public int EntityId;\r\n public int X;\r\n public int Y;\r\n }\r\n\r\n [Reducer]\r\n public static void Seed(ReducerContext ctx)\r\n {\r\n ctx.Db.entities.Insert(new Entity { Id = 1 });\r\n ctx.Db.entities.Insert(new Entity { Id = 2 });\r\n\r\n ctx.Db.positions.Insert(new Position { EntityId = 1, X = 0, Y = 0 });\r\n ctx.Db.positions.Insert(new Position { EntityId = 2, X = 10, Y = 0 });\r\n\r\n ctx.Db.velocities.Insert(new Velocity { EntityId = 1, VX = 1, VY = 0 });\r\n ctx.Db.velocities.Insert(new Velocity { EntityId = 2, VX = -2, VY = 3 });\r\n }\r\n\r\n [Reducer]\r\n public static void Step(ReducerContext ctx)\r\n {\r\n foreach (var p in ctx.Db.positions.Iter())\r\n {\r\n var velOpt = ctx.Db.velocities.EntityId.Find(p.EntityId);\r\n if (!velOpt.HasValue) continue;\r\n\r\n var np = new NextPosition {\r\n EntityId = p.EntityId,\r\n X = p.X + velOpt.Value.VX,\r\n Y = p.Y + velOpt.Value.VY\r\n };\r\n\r\n if (ctx.Db.next_positions.EntityId.Find(p.EntityId).HasValue)\r\n ctx.Db.next_positions.EntityId.Update(np);\r\n else\r\n ctx.Db.next_positions.Insert(np);\r\n }\r\n }\r\n}\r\n",
"syntax": "csharp"
},
- "t_009_init": {
- "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"users\")]\n public partial struct User\n {\n [PrimaryKey] public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.users.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.users.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n }\n}\n",
+ "schema/t_021_multi_column_index": {
+ "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Name = \"logs\")]\r\n [SpacetimeDB.Index.BTree(Name = \"by_user_day\", Columns = new[] { nameof(UserId), nameof(Day) })]\r\n public partial struct Log\r\n {\r\n [PrimaryKey] public int Id;\r\n public int UserId;\r\n public int Day;\r\n public string Message;\r\n }\r\n\r\n [Reducer]\r\n public static void Seed(ReducerContext ctx)\r\n {\r\n ctx.Db.logs.Insert(new Log { Id = 1, UserId = 7, Day = 1, Message = \"a\" });\r\n ctx.Db.logs.Insert(new Log { Id = 2, UserId = 7, Day = 2, Message = \"b\" });\r\n ctx.Db.logs.Insert(new Log { Id = 3, UserId = 9, Day = 1, Message = \"c\" });\r\n }\r\n}\r\n",
"syntax": "csharp"
},
- "t_007_crud": {
- "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"users\")]\n public partial struct User\n {\n [PrimaryKey] public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [Reducer]\n public static void Crud(ReducerContext ctx)\n {\n ctx.Db.users.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.users.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n ctx.Db.users.Id.Update(new User { Id = 1, Name = \"Alice2\", Age = 31, Active = false });\n ctx.Db.users.Id.Delete(2);\n }\n}\n",
+ "t_000_empty_reducers": {
+ "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Reducer]\n public static void EmptyReducer_NoArgs(ReducerContext ctx) { }\n\n [Reducer]\n public static void EmptyReducer_WithInt(ReducerContext ctx, int count) { }\n\n [Reducer]\n public static void EmptyReducer_WithString(ReducerContext ctx, string name) { }\n\n [Reducer]\n public static void EmptyReducer_WithTwoArgs(ReducerContext ctx, int count, string name) { }\n\n [Reducer]\n public static void EmptyReducer_WithThreeArgs(ReducerContext ctx, bool active, float ratio, string label) { }\n}",
"syntax": "csharp"
},
- "basics/t_007_crud": {
- "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Name = \"users\")]\r\n public partial struct User\r\n {\r\n [PrimaryKey] public int Id;\r\n public string Name;\r\n public int Age;\r\n public bool Active;\r\n }\r\n\r\n [Reducer]\r\n public static void Crud(ReducerContext ctx)\r\n {\r\n ctx.Db.users.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\r\n ctx.Db.users.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\r\n ctx.Db.users.Id.Update(new User { Id = 1, Name = \"Alice2\", Age = 31, Active = false });\r\n ctx.Db.users.Id.Delete(2);\r\n }\r\n}\r\n",
+ "t_001_basic_tables": {
+ "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"users\")]\n public partial struct Users\n {\n [PrimaryKey] public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [Table(Name = \"products\")]\n public partial struct Products\n {\n [PrimaryKey] public int Id;\n public string Title;\n public float Price;\n public bool InStock;\n }\n\n [Table(Name = \"notes\")]\n public partial struct Notes\n {\n [PrimaryKey] public int Id;\n public string Body;\n public long Rating;\n public bool Pinned;\n }\n}\n",
"syntax": "csharp"
},
- "basics/t_009_init": {
- "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Name = \"users\")]\r\n public partial struct User\r\n {\r\n [PrimaryKey] public int Id;\r\n public string Name;\r\n public int Age;\r\n public bool Active;\r\n }\r\n\r\n [Reducer(ReducerKind.Init)]\r\n public static void Init(ReducerContext ctx)\r\n {\r\n ctx.Db.users.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\r\n ctx.Db.users.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\r\n }\r\n}\r\n",
+ "t_002_scheduled_table": {
+ "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"tick_timer\", Scheduled = nameof(Tick), ScheduledAt = nameof(TickTimer.ScheduledAt))]\n public partial struct TickTimer\n {\n [PrimaryKey, AutoInc] public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [Reducer]\n public static void Tick(ReducerContext ctx, TickTimer timer) { }\n\n [Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n var interval = new TimeDuration { Microseconds = 50_000 };\n ctx.Db.tick_timer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(interval)\n });\n }\n}",
"syntax": "csharp"
},
"t_003_struct_in_table": {
"answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [Table(Name = \"entities\")]\n public partial struct Entity\n {\n [PrimaryKey] public int Id;\n public Position Pos;\n }\n}\n",
"syntax": "csharp"
},
- "t_013_spacetime_sum_type": {
- "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Type]\n public partial struct Circle { public int Radius; }\n\n [Type]\n public partial struct Rectangle { public int Width; public int Height; }\n\n [Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> {}\n\n [Table(Name = \"results\")]\n public partial struct Result\n {\n [PrimaryKey] public int Id;\n public Shape Value;\n }\n\n [Reducer]\n public static void SetCircle(ReducerContext ctx, int id, int radius)\n {\n ctx.Db.results.Insert(new Result { Id = id, Value = new Shape.Circle(new Circle { Radius = radius }) });\n }\n}\n",
+ "t_004_insert": {
+ "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"users\")]\n public partial struct User\n {\n [PrimaryKey] public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [Reducer]\n public static void InsertUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n ctx.Db.users.Insert(new User { Id = id, Name = name, Age = age, Active = active });\n }\n}\n",
"syntax": "csharp"
},
"t_005_update": {
"answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"users\")]\n public partial struct User\n {\n [PrimaryKey] public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [Reducer]\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n ctx.Db.users.Id.Update(new User { Id = id, Name = name, Age = age, Active = active });\n }\n}\n",
"syntax": "csharp"
},
- "basics/t_005_update": {
- "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Name = \"users\")]\r\n public partial struct User\r\n {\r\n [PrimaryKey] public int Id;\r\n public string Name;\r\n public int Age;\r\n public bool Active;\r\n }\r\n\r\n [Reducer]\r\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\r\n {\r\n ctx.Db.users.Id.Update(new User { Id = id, Name = name, Age = age, Active = active });\r\n }\r\n}\r\n",
+ "t_006_delete": {
+ "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"users\")]\n public partial struct User\n {\n [PrimaryKey] public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [Reducer]\n public static void DeleteUser(ReducerContext ctx, int id)\n {\n ctx.Db.users.Id.Delete(id);\n }\n}\n",
"syntax": "csharp"
},
- "t_018_constraints": {
- "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"accounts\", Public = true)]\n [SpacetimeDB.Index.BTree(Name = \"by_name\", Columns = [nameof(Name)])]\n public partial struct Account\n {\n [SpacetimeDB.PrimaryKey] public int Id;\n [SpacetimeDB.Unique] public string Email;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.accounts.Insert(new Account { Id = 1, Email = \"a@example.com\", Name = \"Alice\" });\n ctx.Db.accounts.Insert(new Account { Id = 2, Email = \"b@example.com\", Name = \"Bob\" });\n }\n}",
+ "t_007_crud": {
+ "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"users\")]\n public partial struct User\n {\n [PrimaryKey] public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [Reducer]\n public static void Crud(ReducerContext ctx)\n {\n ctx.Db.users.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.users.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n ctx.Db.users.Id.Update(new User { Id = 1, Name = \"Alice2\", Age = 31, Active = false });\n ctx.Db.users.Id.Delete(2);\n }\n}\n",
"syntax": "csharp"
},
- "t_000_empty_reducers": {
- "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Reducer]\n public static void EmptyReducer_NoArgs(ReducerContext ctx) { }\n\n [Reducer]\n public static void EmptyReducer_WithInt(ReducerContext ctx, int count) { }\n\n [Reducer]\n public static void EmptyReducer_WithString(ReducerContext ctx, string name) { }\n\n [Reducer]\n public static void EmptyReducer_WithTwoArgs(ReducerContext ctx, int count, string name) { }\n\n [Reducer]\n public static void EmptyReducer_WithThreeArgs(ReducerContext ctx, bool active, float ratio, string label) { }\n}",
+ "t_008_index_lookup": {
+ "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"users\")]\n public partial struct User\n {\n [PrimaryKey] public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [Table(Name = \"results\")]\n public partial struct Result\n {\n [PrimaryKey] public int Id;\n public string Name;\n }\n\n [Reducer]\n public static void LookupUserName(ReducerContext ctx, int id)\n {\n var u = ctx.Db.users.Id.Find(id);\n if (u.HasValue)\n {\n var row = u.Value;\n ctx.Db.results.Insert(new Result { Id = row.Id, Name = row.Name });\n }\n }\n}\n",
"syntax": "csharp"
},
- "t_021_multi_column_index": {
- "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"logs\")]\n [SpacetimeDB.Index.BTree(Name = \"by_user_day\", Columns = new[] { nameof(UserId), nameof(Day) })]\n public partial struct Log\n {\n [PrimaryKey] public int Id;\n public int UserId;\n public int Day;\n public string Message;\n }\n\n [Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.logs.Insert(new Log { Id = 1, UserId = 7, Day = 1, Message = \"a\" });\n ctx.Db.logs.Insert(new Log { Id = 2, UserId = 7, Day = 2, Message = \"b\" });\n ctx.Db.logs.Insert(new Log { Id = 3, UserId = 9, Day = 1, Message = \"c\" });\n }\n}\n",
+ "t_009_init": {
+ "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"users\")]\n public partial struct User\n {\n [PrimaryKey] public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.users.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.users.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n }\n}\n",
"syntax": "csharp"
},
- "basics/t_011_helper_function": {
- "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Name = \"results\")]\r\n public partial struct Result\r\n {\r\n [PrimaryKey] public int Id;\r\n public int Sum;\r\n }\r\n\r\n static int Add(int a, int b) => a + b;\r\n\r\n [Reducer]\r\n public static void ComputeSum(ReducerContext ctx, int id, int a, int b)\r\n {\r\n ctx.Db.results.Insert(new Result { Id = id, Sum = Add(a, b) });\r\n }\r\n}\r\n",
+ "t_010_connect": {
+ "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"events\")]\n public partial struct Event\n {\n [PrimaryKey, AutoInc] public int Id;\n public string Kind;\n }\n\n [Reducer(ReducerKind.ClientConnected)]\n public static void ClientConnected(ReducerContext ctx)\n {\n ctx.Db.events.Insert(new Event { Kind = \"connected\" });\n }\n\n [Reducer(ReducerKind.ClientDisconnected)]\n public static void ClientDisconnected(ReducerContext ctx)\n {\n ctx.Db.events.Insert(new Event { Kind = \"disconnected\" });\n }\n}\n",
"syntax": "csharp"
},
- "schema/t_015_product_type_columns": {
- "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Type]\r\n public partial struct Address\r\n {\r\n public string Street;\r\n public int Zip;\r\n }\r\n\r\n [Type]\r\n public partial struct Position\r\n {\r\n public int X;\r\n public int Y;\r\n }\r\n\r\n [Table(Name = \"profiles\")]\r\n public partial struct Profile\r\n {\r\n [PrimaryKey] public int Id;\r\n public Address Home;\r\n public Address Work;\r\n public Position Pos;\r\n }\r\n\r\n [Reducer]\r\n public static void Seed(ReducerContext ctx)\r\n {\r\n ctx.Db.profiles.Insert(new Profile {\r\n Id = 1,\r\n Home = new Address { Street = \"1 Main\", Zip = 11111 },\r\n Work = new Address { Street = \"2 Broad\", Zip = 22222 },\r\n Pos = new Position { X = 7, Y = 9 }\r\n });\r\n }\r\n}\r\n",
+ "t_011_helper_function": {
+ "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"results\")]\n public partial struct Result\n {\n [PrimaryKey] public int Id;\n public int Sum;\n }\n\n static int Add(int a, int b) => a + b;\n\n [Reducer]\n public static void ComputeSum(ReducerContext ctx, int id, int a, int b)\n {\n ctx.Db.results.Insert(new Result { Id = id, Sum = Add(a, b) });\n }\n}\n",
"syntax": "csharp"
},
- "schema/t_018_constraints": {
- "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [SpacetimeDB.Table(Name = \"accounts\", Public = true)]\r\n [SpacetimeDB.Index.BTree(Name = \"by_name\", Columns = [nameof(Name)])]\r\n public partial struct Account\r\n {\r\n [SpacetimeDB.PrimaryKey] public int Id;\r\n [SpacetimeDB.Unique] public string Email;\r\n public string Name;\r\n }\r\n\r\n [SpacetimeDB.Reducer]\r\n public static void Seed(ReducerContext ctx)\r\n {\r\n ctx.Db.accounts.Insert(new Account { Id = 1, Email = \"a@example.com\", Name = \"Alice\" });\r\n ctx.Db.accounts.Insert(new Account { Id = 2, Email = \"b@example.com\", Name = \"Bob\" });\r\n }\r\n}",
+ "t_012_spacetime_product_type": {
+ "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Type]\n public partial struct Score\n {\n public int Left;\n public int Right;\n }\n\n [Table(Name = \"results\")]\n public partial struct Result\n {\n [PrimaryKey] public int Id;\n public Score Value;\n }\n\n [Reducer]\n public static void SetScore(ReducerContext ctx, int id, int left, int right)\n {\n ctx.Db.results.Insert(new Result { Id = id, Value = new Score { Left = left, Right = right } });\n }\n}\n",
+ "syntax": "csharp"
+ },
+ "t_013_spacetime_sum_type": {
+ "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Type]\n public partial struct Circle { public int Radius; }\n\n [Type]\n public partial struct Rectangle { public int Width; public int Height; }\n\n [Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> {}\n\n [Table(Name = \"results\")]\n public partial struct Result\n {\n [PrimaryKey] public int Id;\n public Shape Value;\n }\n\n [Reducer]\n public static void SetCircle(ReducerContext ctx, int id, int radius)\n {\n ctx.Db.results.Insert(new Result { Id = id, Value = new Shape.Circle(new Circle { Radius = radius }) });\n }\n}\n",
"syntax": "csharp"
},
"t_014_elementary_columns": {
"answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"primitives\")]\n public partial struct Primitive\n {\n [PrimaryKey] public int Id;\n public int Count;\n public long Total;\n public float Price;\n public double Ratio;\n public bool Active;\n public string Name;\n }\n\n [Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.primitives.Insert(new Primitive {\n Id = 1,\n Count = 2,\n Total = 3000000000,\n Price = 1.5f,\n Ratio = 2.25,\n Active = true,\n Name = \"Alice\"\n });\n }\n}\n",
"syntax": "csharp"
},
- "basics/t_008_index_lookup": {
- "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Name = \"users\")]\r\n public partial struct User\r\n {\r\n [PrimaryKey] public int Id;\r\n public string Name;\r\n public int Age;\r\n public bool Active;\r\n }\r\n\r\n [Table(Name = \"results\")]\r\n public partial struct Result\r\n {\r\n [PrimaryKey] public int Id;\r\n public string Name;\r\n }\r\n\r\n [Reducer]\r\n public static void LookupUserName(ReducerContext ctx, int id)\r\n {\r\n var u = ctx.Db.users.Id.Find(id);\r\n if (u.HasValue)\r\n {\r\n var row = u.Value;\r\n ctx.Db.results.Insert(new Result { Id = row.Id, Name = row.Name });\r\n }\r\n }\r\n}\r\n",
+ "t_015_product_type_columns": {
+ "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Type]\n public partial struct Address\n {\n public string Street;\n public int Zip;\n }\n\n [Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [Table(Name = \"profiles\")]\n public partial struct Profile\n {\n [PrimaryKey] public int Id;\n public Address Home;\n public Address Work;\n public Position Pos;\n }\n\n [Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.profiles.Insert(new Profile {\n Id = 1,\n Home = new Address { Street = \"1 Main\", Zip = 11111 },\n Work = new Address { Street = \"2 Broad\", Zip = 22222 },\n Pos = new Position { X = 7, Y = 9 }\n });\n }\n}\n",
"syntax": "csharp"
},
- "t_006_delete": {
- "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"users\")]\n public partial struct User\n {\n [PrimaryKey] public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [Reducer]\n public static void DeleteUser(ReducerContext ctx, int id)\n {\n ctx.Db.users.Id.Delete(id);\n }\n}\n",
+ "t_016_sum_type_columns": {
+ "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Type]\n public partial struct Circle { public int Radius; }\n\n [Type]\n public partial struct Rectangle { public int Width; public int Height; }\n\n [Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> {}\n\n [Table(Name = \"drawings\")]\n public partial struct Drawing\n {\n [PrimaryKey] public int Id;\n public Shape A;\n public Shape B;\n }\n\n [Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.drawings.Insert(new Drawing {\n Id = 1,\n A = new Shape.Circle(new Circle { Radius = 10 }),\n B = new Shape.Rectangle(new Rectangle { Width = 4, Height = 6 })\n });\n }\n}\n",
+ "syntax": "csharp"
+ },
+ "t_017_scheduled_columns": {
+ "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"tick_timer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [PrimaryKey, AutoInc] public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [Reducer]\n public static void Tick(ReducerContext ctx, TickTimer schedule) { }\n\n [Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n var interval = new TimeDuration { Microseconds = 50_000 };\n ctx.Db.tick_timer.Insert(new TickTimer\n {\n ScheduledId = 0,\n ScheduledAt = new ScheduleAt.Interval(interval)\n });\n }\n}\n",
+ "syntax": "csharp"
+ },
+ "t_018_constraints": {
+ "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"accounts\", Public = true)]\n [SpacetimeDB.Index.BTree(Name = \"by_name\", Columns = [nameof(Name)])]\n public partial struct Account\n {\n [SpacetimeDB.PrimaryKey] public int Id;\n [SpacetimeDB.Unique] public string Email;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.accounts.Insert(new Account { Id = 1, Email = \"a@example.com\", Name = \"Alice\" });\n ctx.Db.accounts.Insert(new Account { Id = 2, Email = \"b@example.com\", Name = \"Bob\" });\n }\n}",
"syntax": "csharp"
},
"t_019_many_to_many": {
"answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"users\")]\n public partial struct User\n {\n [PrimaryKey] public int UserId;\n public string Name;\n }\n\n [Table(Name = \"groups\")]\n public partial struct Group\n {\n [PrimaryKey] public int GroupId;\n public string Title;\n }\n\n [Table(Name = \"memberships\")]\n [SpacetimeDB.Index.BTree(Name = \"by_user\", Columns = new[] { nameof(UserId) })]\n [SpacetimeDB.Index.BTree(Name = \"by_group\", Columns = new[] { nameof(GroupId) })]\n public partial struct Membership\n {\n [PrimaryKey] public int Id;\n public int UserId;\n public int GroupId;\n }\n\n [Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.users.Insert(new User { UserId = 1, Name = \"Alice\" });\n ctx.Db.users.Insert(new User { UserId = 2, Name = \"Bob\" });\n\n ctx.Db.groups.Insert(new Group { GroupId = 10, Title = \"Admin\" });\n ctx.Db.groups.Insert(new Group { GroupId = 20, Title = \"Dev\" });\n\n ctx.Db.memberships.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\n ctx.Db.memberships.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\n ctx.Db.memberships.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\n }\n}\n",
"syntax": "csharp"
},
- "t_016_sum_type_columns": {
- "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Type]\n public partial struct Circle { public int Radius; }\n\n [Type]\n public partial struct Rectangle { public int Width; public int Height; }\n\n [Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> {}\n\n [Table(Name = \"drawings\")]\n public partial struct Drawing\n {\n [PrimaryKey] public int Id;\n public Shape A;\n public Shape B;\n }\n\n [Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.drawings.Insert(new Drawing {\n Id = 1,\n A = new Shape.Circle(new Circle { Radius = 10 }),\n B = new Shape.Rectangle(new Rectangle { Width = 4, Height = 6 })\n });\n }\n}\n",
+ "t_020_ecs": {
+ "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"entities\")]\n public partial struct Entity { [PrimaryKey] public int Id; }\n\n [Table(Name = \"positions\")]\n public partial struct Position\n {\n [PrimaryKey] public int EntityId;\n public int X;\n public int Y;\n }\n\n [Table(Name = \"velocities\")]\n public partial struct Velocity\n {\n [PrimaryKey] public int EntityId;\n public int VX;\n public int VY;\n }\n\n [Table(Name = \"next_positions\")]\n public partial struct NextPosition\n {\n [PrimaryKey] public int EntityId;\n public int X;\n public int Y;\n }\n\n [Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.entities.Insert(new Entity { Id = 1 });\n ctx.Db.entities.Insert(new Entity { Id = 2 });\n\n ctx.Db.positions.Insert(new Position { EntityId = 1, X = 0, Y = 0 });\n ctx.Db.positions.Insert(new Position { EntityId = 2, X = 10, Y = 0 });\n\n ctx.Db.velocities.Insert(new Velocity { EntityId = 1, VX = 1, VY = 0 });\n ctx.Db.velocities.Insert(new Velocity { EntityId = 2, VX = -2, VY = 3 });\n }\n\n [Reducer]\n public static void Step(ReducerContext ctx)\n {\n foreach (var p in ctx.Db.positions.Iter())\n {\n var velOpt = ctx.Db.velocities.EntityId.Find(p.EntityId);\n if (!velOpt.HasValue) continue;\n\n var np = new NextPosition {\n EntityId = p.EntityId,\n X = p.X + velOpt.Value.VX,\n Y = p.Y + velOpt.Value.VY\n };\n\n if (ctx.Db.next_positions.EntityId.Find(p.EntityId).HasValue)\n ctx.Db.next_positions.EntityId.Update(np);\n else\n ctx.Db.next_positions.Insert(np);\n }\n }\n}\n",
"syntax": "csharp"
},
- "schema/t_020_ecs": {
- "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Name = \"entities\")]\r\n public partial struct Entity { [PrimaryKey] public int Id; }\r\n\r\n [Table(Name = \"positions\")]\r\n public partial struct Position\r\n {\r\n [PrimaryKey] public int EntityId;\r\n public int X;\r\n public int Y;\r\n }\r\n\r\n [Table(Name = \"velocities\")]\r\n public partial struct Velocity\r\n {\r\n [PrimaryKey] public int EntityId;\r\n public int VX;\r\n public int VY;\r\n }\r\n\r\n [Table(Name = \"next_positions\")]\r\n public partial struct NextPosition\r\n {\r\n [PrimaryKey] public int EntityId;\r\n public int X;\r\n public int Y;\r\n }\r\n\r\n [Reducer]\r\n public static void Seed(ReducerContext ctx)\r\n {\r\n ctx.Db.entities.Insert(new Entity { Id = 1 });\r\n ctx.Db.entities.Insert(new Entity { Id = 2 });\r\n\r\n ctx.Db.positions.Insert(new Position { EntityId = 1, X = 0, Y = 0 });\r\n ctx.Db.positions.Insert(new Position { EntityId = 2, X = 10, Y = 0 });\r\n\r\n ctx.Db.velocities.Insert(new Velocity { EntityId = 1, VX = 1, VY = 0 });\r\n ctx.Db.velocities.Insert(new Velocity { EntityId = 2, VX = -2, VY = 3 });\r\n }\r\n\r\n [Reducer]\r\n public static void Step(ReducerContext ctx)\r\n {\r\n foreach (var p in ctx.Db.positions.Iter())\r\n {\r\n var velOpt = ctx.Db.velocities.EntityId.Find(p.EntityId);\r\n if (!velOpt.HasValue) continue;\r\n\r\n var np = new NextPosition {\r\n EntityId = p.EntityId,\r\n X = p.X + velOpt.Value.VX,\r\n Y = p.Y + velOpt.Value.VY\r\n };\r\n\r\n if (ctx.Db.next_positions.EntityId.Find(p.EntityId).HasValue)\r\n ctx.Db.next_positions.EntityId.Update(np);\r\n else\r\n ctx.Db.next_positions.Insert(np);\r\n }\r\n }\r\n}\r\n",
+ "t_021_multi_column_index": {
+ "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"logs\")]\n [SpacetimeDB.Index.BTree(Name = \"by_user_day\", Columns = new[] { nameof(UserId), nameof(Day) })]\n public partial struct Log\n {\n [PrimaryKey] public int Id;\n public int UserId;\n public int Day;\n public string Message;\n }\n\n [Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.logs.Insert(new Log { Id = 1, UserId = 7, Day = 1, Message = \"a\" });\n ctx.Db.logs.Insert(new Log { Id = 2, UserId = 7, Day = 2, Message = \"b\" });\n ctx.Db.logs.Insert(new Log { Id = 3, UserId = 9, Day = 1, Message = \"c\" });\n }\n}\n",
"syntax": "csharp"
}
}
@@ -16043,133 +16043,188 @@
"name": "GPT-5",
"route_api_model": "gpt-5",
"tasks": {
- "t_014_elementary_columns": {
- "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_014_elementary_columns",
+ "t_000_empty_reducers": {
+ "hash": "00917b28ec8f75406cd8aac609eeafb254b65945d2bb0c709404ac09a9762fca",
+ "task": "t_000_empty_reducers",
"lang": "rust",
"golden_published": true,
"model_name": "GPT-5",
- "total_tests": 3,
- "passed_tests": 0,
- "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::table(name = primitives)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) -> Result<(), String> {\n ctx.db\n .primitives()\n .try_insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000_i64,\n price: 1.5_f32,\n ratio: 2.25_f64,\n active: true,\n name: \"Alice\".to_string(),\n })\n .map_err(|e| format!(\"{:?}\", e))?;\n Ok(())\n}",
- "category": "schema",
+ "total_tests": 1,
+ "passed_tests": 1,
+ "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::reducer]\npub fn empty_reducer_no_args(_ctx: &ReducerContext) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_int(_ctx: &ReducerContext, _count: i32) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_string(_ctx: &ReducerContext, _name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_two_args(_ctx: &ReducerContext, _count: i32, _name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_three_args(_ctx: &ReducerContext, _active: bool, _ratio: f32, _label: String) {}",
+ "category": "basics",
"route_api_model": "gpt-5",
- "golden_db": "schema-t-014-elementary-columns-golden",
- "llm_db": "schema-t-014-elementary-columns-gpt-5-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_014_elementary_columns\\rust\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_014_elementary_columns\\rust\\server\\gpt-5\\llm",
+ "golden_db": "basics-t-000-empty-reducers-golden",
+ "llm_db": "basics-t-000-empty-reducers-gpt-5-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_000_empty_reducers\\rust\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_000_empty_reducers\\rust\\server\\gpt-5\\llm",
"scorer_details": {
- "publish_error": {
- "pass": false,
- "partial": 0.0,
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
"notes": {
- "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Blocking waiting for file lock on package cache\n Updating crates.io index\n Blocking waiting for file lock on package cache\n Locking 67 packages to latest compatible versions\n Blocking waiting for file lock on package cache\n Blocking waiting for file lock on package cache\n Blocking waiting for file lock on package cache\n Compiling proc-macro2 v1.0.101\n Compiling quote v1.0.41\n Compiling unicode-ident v1.0.20\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling cfg-if v1.0.4\n Compiling shlex v1.3.0\n Compiling serde v1.0.228\n Compiling either v1.15.0\n Compiling zerocopy v0.8.27\n Compiling find-msvc-tools v0.1.4\n Compiling nohash-hasher v0.2.0\n Compiling bitflags v2.10.0\n Compiling thiserror v1.0.69\n Compiling anyhow v1.0.100\n Compiling arrayvec v0.7.6\n Compiling heck v0.5.0\n Compiling keccak v0.1.5\n Compiling heck v0.4.1\n Compiling humantime v2.3.0\n Compiling convert_case v0.4.0\n Compiling arrayref v0.3.9\n Compiling bytes v1.10.1\n Compiling bytemuck v1.24.0\n Compiling hex v0.4.3\n Compiling second-stack v0.3.5\n Compiling spacetimedb-lib v1.6.0\n Compiling getrandom v0.2.16\n Compiling constant_time_eq v0.3.1\n Compiling cc v1.2.41\n Compiling itertools v0.12.1\n Compiling rand_core v0.6.4\n Compiling smallvec v1.15.1\n Compiling scoped-tls v1.0.1\n Compiling log v0.4.28\n Compiling generic-array v0.14.9\n Compiling num-traits v0.2.19\n Compiling spacetimedb-primitives v1.6.0\n Compiling blake3 v1.8.2\n Compiling spacetimedb-bindings-sys v1.6.0\n Compiling ppv-lite86 v0.2.21\n Compiling ethnum v1.5.2\n Compiling rand_chacha v0.3.1\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.6\n Compiling syn v2.0.107\n Compiling rand v0.8.5\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling approx v0.3.2\n Compiling chrono v0.4.42\n Compiling decorum v0.3.1\n Compiling thiserror-impl v1.0.69\n Compiling enum-as-inner v0.6.1\n Compiling derive_more v0.99.20\n Compiling spacetimedb-bindings-macro v1.6.0\n Compiling spacetimedb-sats v1.6.0\n Compiling spacetimedb v1.6.0\n Compiling spacetime-module v0.1.0 (E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_014_elementary_columns\\rust\\server\\gpt-5\\llm)\nerror[E0599]: no method named `try_insert` found for reference `&primitives__TableHandle` in the current scope\n --> src\\lib.rs:20:10\n |\n18 | / ctx.db\n19 | | .primitives()\n20 | | .try_insert(Primitive {\n | |_________-^^^^^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `try_insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `insert` with a similar name\n |\n20 - .try_insert(Primitive {\n20 + .insert(Primitive {\n |\n\nFor more information about this error, try `rustc --explain E0599`.\nerror: could not compile `spacetime-module` (lib) due to 1 previous error\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n",
- "phase": "build_or_publish"
+ "golden_db": "basics-t-000-empty-reducers-golden",
+ "llm_db": "basics-t-000-empty-reducers-gpt-5-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
}
}
},
"vendor": "openai",
- "started_at": "2025-10-21T22:01:05.376366300Z",
- "finished_at": "2025-10-21T22:03:53.896212600Z"
+ "started_at": "2025-10-24T21:13:14.021183600Z",
+ "finished_at": "2025-10-24T21:14:01.165125200Z"
},
- "t_020_ecs": {
+ "t_001_basic_tables": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_020_ecs",
+ "task": "t_001_basic_tables",
"lang": "rust",
"golden_published": true,
"model_name": "GPT-5",
- "total_tests": 5,
- "passed_tests": 0,
- "llm_output": "use spacetimedb::{reducer, table, ReducerContext};\n\n#[spacetimedb::table(name = entities)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[spacetimedb::table(name = positions)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = velocities)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[spacetimedb::table(name = next_positions)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\nfn upsert_position(ctx: &ReducerContext, entity_id: i32, x: i32, y: i32) {\n if let Some(mut p) = ctx.db.positions().entity_id().find(entity_id) {\n p.x = x;\n p.y = y;\n ctx.db.positions().entity_id().update(p);\n } else {\n ctx.db.positions().insert(Position { entity_id, x, y });\n }\n}\n\nfn upsert_velocity(ctx: &ReducerContext, entity_id: i32, vx: i32, vy: i32) {\n if let Some(mut v) = ctx.db.velocities().entity_id().find(entity_id) {\n v.vx = vx;\n v.vy = vy;\n ctx.db.velocities().entity_id().update(v);\n } else {\n ctx.db.velocities().insert(Velocity { entity_id, vx, vy });\n }\n}\n\nfn upsert_next_position(ctx: &ReducerContext, entity_id: i32, x: i32, y: i32) {\n if let Some(mut np) = ctx.db.next_positions().entity_id().find(entity_id) {\n np.x = x;\n np.y = y;\n ctx.db.next_positions().entity_id().update(np);\n } else {\n ctx.db.next_positions().insert(NextPosition { entity_id, x, y });\n }\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n // Ensure entities exist\n if ctx.db.entities().id().find(1).is_none() {\n ctx.db.entities().insert(Entity { id: 1 });\n }\n if ctx.db.entities().id().find(2).is_none() {\n ctx.db.entities().insert(Entity { id: 2 });\n }\n\n // Upsert positions\n upsert_position(ctx, 1, 0, 0);\n upsert_position(ctx, 2, 10, 0);\n\n // Upsert velocities\n upsert_velocity(ctx, 1, 1, 0);\n upsert_velocity(ctx, 2, -2, 3);\n}\n\n#[spacetimedb::reducer]\npub fn step(ctx: &ReducerContext) {\n for pos in ctx.db.positions().iter() {\n if let Some(vel) = ctx.db.velocities().entity_id().find(pos.entity_id) {\n let nx = pos.x + vel.vx;\n let ny = pos.y + vel.vy;\n upsert_next_position(ctx, pos.entity_id, nx, ny);\n }\n }\n}",
- "category": "schema",
+ "total_tests": 1,
+ "passed_tests": 1,
+ "llm_output": "use spacetimedb as _;\n\n#[spacetimedb::table(name = users, public)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = products, public)]\npub struct Product {\n #[primary_key]\n id: i32,\n title: String,\n price: f32,\n in_stock: bool,\n}\n\n#[spacetimedb::table(name = notes, public)]\npub struct Note {\n #[primary_key]\n id: i32,\n body: String,\n rating: i64,\n pinned: bool,\n}",
+ "category": "basics",
"route_api_model": "gpt-5",
- "golden_db": "schema-t-020-ecs-golden",
- "llm_db": "schema-t-020-ecs-gpt-5-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_020_ecs\\rust\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_020_ecs\\rust\\server\\gpt-5\\llm",
+ "golden_db": "basics-t-001-basic-tables-golden",
+ "llm_db": "basics-t-001-basic-tables-gpt-5-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_001_basic_tables\\rust\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_001_basic_tables\\rust\\server\\gpt-5\\llm",
"scorer_details": {
- "publish_error": {
- "pass": false,
- "partial": 0.0,
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
"notes": {
- "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Blocking waiting for file lock on package cache\n Updating crates.io index\n Blocking waiting for file lock on package cache\n Locking 67 packages to latest compatible versions\n Blocking waiting for file lock on package cache\n Compiling proc-macro2 v1.0.101\n Compiling quote v1.0.41\n Compiling unicode-ident v1.0.20\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling cfg-if v1.0.4\n Compiling zerocopy v0.8.27\n Compiling find-msvc-tools v0.1.4\n Compiling serde v1.0.228\n Compiling either v1.15.0\n Compiling shlex v1.3.0\n Compiling bitflags v2.10.0\n Compiling nohash-hasher v0.2.0\n Compiling anyhow v1.0.100\n Compiling thiserror v1.0.69\n Compiling heck v0.4.1\n Compiling convert_case v0.4.0\n Compiling humantime v2.3.0\n Compiling keccak v0.1.5\n Compiling arrayvec v0.7.6\n Compiling heck v0.5.0\n Compiling second-stack v0.3.5\n Compiling hex v0.4.3\n Compiling spacetimedb-lib v1.6.0\n Compiling bytes v1.10.1\n Compiling smallvec v1.15.1\n Compiling arrayref v0.3.9\n Compiling itertools v0.12.1\n Compiling cc v1.2.41\n Compiling getrandom v0.2.16\n Compiling bytemuck v1.24.0\n Compiling constant_time_eq v0.3.1\n Compiling scoped-tls v1.0.1\n Compiling log v0.4.28\n Compiling rand_core v0.6.4\n Compiling generic-array v0.14.9\n Compiling spacetimedb-primitives v1.6.0\n Compiling num-traits v0.2.19\n Compiling spacetimedb-bindings-sys v1.6.0\n Compiling blake3 v1.8.2\n Compiling syn v2.0.107\n Compiling ethnum v1.5.2\n Compiling crypto-common v0.1.6\n Compiling block-buffer v0.10.4\n Compiling ppv-lite86 v0.2.21\n Compiling digest v0.10.7\n Compiling rand_chacha v0.3.1\n Compiling sha3 v0.10.8\n Compiling rand v0.8.5\n Compiling approx v0.3.2\n Compiling chrono v0.4.42\n Compiling decorum v0.3.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling enum-as-inner v0.6.1\n Compiling spacetimedb-bindings-macro v1.6.0\n Compiling spacetimedb-sats v1.6.0\n Compiling spacetimedb v1.6.0\n Compiling spacetime-module v0.1.0 (E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\rust\\server\\gpt-5\\llm)\nwarning: unused imports: `reducer` and `table`\n --> src\\lib.rs:2:19\n |\n2 | use spacetimedb::{reducer, table, ReducerContext};\n | ^^^^^^^ ^^^^^\n |\n = note: `#[warn(unused_imports)]` on by default\n\nerror[E0599]: no method named `insert` found for reference `&positions__TableHandle` in the current scope\n --> src\\lib.rs:40:28\n |\n40 | ctx.db.positions().insert(Position { entity_id, x, y });\n | ^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `try_insert` with a similar name\n |\n40 | ctx.db.positions().try_insert(Position { entity_id, x, y });\n | ++++\n\nerror[E0599]: no method named `insert` found for reference `&velocities__TableHandle` in the current scope\n --> src\\lib.rs:50:29\n |\n50 | ctx.db.velocities().insert(Velocity { entity_id, vx, vy });\n | ^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `try_insert` with a similar name\n |\n50 | ctx.db.velocities().try_insert(Velocity { entity_id, vx, vy });\n | ++++\n\nerror[E0599]: no method named `insert` found for reference `&next_positions__TableHandle` in the current scope\n --> src\\lib.rs:60:33\n |\n60 | ctx.db.next_positions().insert(NextPosition { entity_id, x, y });\n | ^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `try_insert` with a similar name\n |\n60 | ctx.db.next_positions().try_insert(NextPosition { entity_id, x, y });\n | ++++\n\nerror[E0599]: no method named `insert` found for reference `&entities__TableHandle` in the current scope\n --> src\\lib.rs:68:27\n |\n68 | ctx.db.entities().insert(Entity { id: 1 });\n | ^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `try_insert` with a similar name\n |\n68 | ctx.db.entities().try_insert(Entity { id: 1 });\n | ++++\n\nerror[E0599]: no method named `insert` found for reference `&entities__TableHandle` in the current scope\n --> src\\lib.rs:71:27\n |\n71 | ctx.db.entities().insert(Entity { id: 2 });\n | ^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `try_insert` with a similar name\n |\n71 | ctx.db.entities().try_insert(Entity { id: 2 });\n | ++++\n\nerror[E0599]: no method named `iter` found for reference `&positions__TableHandle` in the current scope\n --> src\\lib.rs:85:35\n |\n85 | for pos in ctx.db.positions().iter() {\n | ^^^^ method not found in `&positions__TableHandle`\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `iter` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\n\nFor more information about this error, try `rustc --explain E0599`.\nwarning: `spacetime-module` (lib) generated 1 warning\nerror: could not compile `spacetime-module` (lib) due to 6 previous errors; 1 warning emitted\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n",
- "phase": "build_or_publish"
+ "golden_db": "basics-t-001-basic-tables-golden",
+ "llm_db": "basics-t-001-basic-tables-gpt-5-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
}
}
},
"vendor": "openai",
- "started_at": "2025-10-21T22:02:14.816574700Z",
- "finished_at": "2025-10-21T22:03:58.179900Z"
+ "started_at": "2025-10-21T22:01:04.558378900Z",
+ "finished_at": "2025-10-21T22:02:18.806241100Z"
},
- "t_007_crud": {
+ "t_002_scheduled_table": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_007_crud",
+ "task": "t_002_scheduled_table",
"lang": "rust",
"golden_published": true,
"model_name": "GPT-5",
- "total_tests": 4,
+ "total_tests": 1,
"passed_tests": 0,
- "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::table(name = users)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn crud(ctx: &ReducerContext) {\n // insert (id=1, name=\"Alice\", age=30, active=true)\n ctx.db.users().insert(User {\n id: 1,\n name: \"Alice\".to_string(),\n age: 30,\n active: true,\n });\n\n // insert (id=2, name=\"Bob\", age=22, active=false)\n ctx.db.users().insert(User {\n id: 2,\n name: \"Bob\".to_string(),\n age: 22,\n active: false,\n });\n\n // update (id=1, name=\"Alice2\", age=31, active=false)\n ctx.db.users().id().update(User {\n id: 1,\n name: \"Alice2\".to_string(),\n age: 31,\n active: false,\n });\n\n // delete id=2\n ctx.db.users().id().delete(2);\n}",
+ "llm_output": "use std::time::Duration;\nuse spacetimedb::{ReducerContext, ScheduleAt};\n\n#[spacetimedb::table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n pub scheduled_id: u64,\n pub scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n let interval = Duration::from_micros(50_000);\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::Interval(interval.into()),\n });\n}",
"category": "basics",
"route_api_model": "gpt-5",
- "golden_db": "basics-t-007-crud-golden",
- "llm_db": "basics-t-007-crud-gpt-5-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_007_crud\\rust\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_007_crud\\rust\\server\\gpt-5\\llm",
+ "golden_db": "basics-t-002-scheduled-table-golden",
+ "llm_db": "basics-t-002-scheduled-table-gpt-5-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_002_scheduled_table\\rust\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_002_scheduled_table\\rust\\server\\gpt-5\\llm",
"scorer_details": {
"publish_error": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Blocking waiting for file lock on package cache\n Updating crates.io index\n Blocking waiting for file lock on package cache\n Locking 67 packages to latest compatible versions\n Blocking waiting for file lock on package cache\n Blocking waiting for file lock on package cache\n Blocking waiting for file lock on package cache\n Compiling proc-macro2 v1.0.101\n Compiling quote v1.0.41\n Compiling unicode-ident v1.0.20\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling cfg-if v1.0.4\n Compiling serde_core v1.0.228\n Compiling zerocopy v0.8.27\n Compiling find-msvc-tools v0.1.4\n Compiling serde v1.0.228\n Compiling shlex v1.3.0\n Compiling either v1.15.0\n Compiling nohash-hasher v0.2.0\n Compiling bitflags v2.10.0\n Compiling anyhow v1.0.100\n Compiling thiserror v1.0.69\n Compiling humantime v2.3.0\n Compiling arrayvec v0.7.6\n Compiling heck v0.4.1\n Compiling keccak v0.1.5\n Compiling convert_case v0.4.0\n Compiling heck v0.5.0\n Compiling hex v0.4.3\n Compiling bytemuck v1.24.0\n Compiling second-stack v0.3.5\n Compiling spacetimedb-lib v1.6.0\n Compiling constant_time_eq v0.3.1\n Compiling bytes v1.10.1\n Compiling getrandom v0.2.16\n Compiling arrayref v0.3.9\n Compiling cc v1.2.41\n Compiling itertools v0.12.1\n Compiling rand_core v0.6.4\n Compiling smallvec v1.15.1\n Compiling log v0.4.28\n Compiling scoped-tls v1.0.1\n Compiling generic-array v0.14.9\n Compiling num-traits v0.2.19\n Compiling spacetimedb-primitives v1.6.0\n Compiling blake3 v1.8.2\n Compiling spacetimedb-bindings-sys v1.6.0\n Compiling ppv-lite86 v0.2.21\n Compiling ethnum v1.5.2\n Compiling syn v2.0.107\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling crypto-common v0.1.6\n Compiling block-buffer v0.10.4\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling approx v0.3.2\n Compiling chrono v0.4.42\n Compiling decorum v0.3.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling enum-as-inner v0.6.1\n Compiling spacetimedb-bindings-macro v1.6.0\n Compiling spacetimedb-sats v1.6.0\n Compiling spacetimedb v1.6.0\n Compiling spacetime-module v0.1.0 (E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_007_crud\\rust\\server\\gpt-5\\llm)\nerror[E0599]: no method named `insert` found for reference `&users__TableHandle` in the current scope\n --> src\\lib.rs:16:20\n |\n16 | ctx.db.users().insert(User {\n | ---------------^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `try_insert` with a similar name\n |\n16 | ctx.db.users().try_insert(User {\n | ++++\n\nerror[E0599]: no method named `insert` found for reference `&users__TableHandle` in the current scope\n --> src\\lib.rs:24:20\n |\n24 | ctx.db.users().insert(User {\n | ---------------^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `try_insert` with a similar name\n |\n24 | ctx.db.users().try_insert(User {\n | ++++\n\nFor more information about this error, try `rustc --explain E0599`.\nerror: could not compile `spacetime-module` (lib) due to 2 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n",
+ "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Blocking waiting for file lock on package cache\n Updating crates.io index\n Blocking waiting for file lock on package cache\n Locking 67 packages to latest compatible versions\n Blocking waiting for file lock on package cache\n Blocking waiting for file lock on package cache\n Compiling proc-macro2 v1.0.101\n Compiling unicode-ident v1.0.20\n Compiling quote v1.0.41\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling cfg-if v1.0.4\n Compiling either v1.15.0\n Compiling serde v1.0.228\n Compiling zerocopy v0.8.27\n Compiling shlex v1.3.0\n Compiling find-msvc-tools v0.1.4\n Compiling nohash-hasher v0.2.0\n Compiling thiserror v1.0.69\n Compiling bitflags v2.10.0\n Compiling anyhow v1.0.100\n Compiling convert_case v0.4.0\n Compiling arrayvec v0.7.6\n Compiling heck v0.4.1\n Compiling keccak v0.1.5\n Compiling humantime v2.3.0\n Compiling heck v0.5.0\n Compiling second-stack v0.3.5\n Compiling constant_time_eq v0.3.1\n Compiling bytemuck v1.24.0\n Compiling arrayref v0.3.9\n Compiling spacetimedb-lib v1.6.0\n Compiling bytes v1.10.1\n Compiling smallvec v1.15.1\n Compiling itertools v0.12.1\n Compiling generic-array v0.14.9\n Compiling cc v1.2.41\n Compiling spacetimedb-primitives v1.6.0\n Compiling getrandom v0.2.16\n Compiling hex v0.4.3\n Compiling log v0.4.28\n Compiling scoped-tls v1.0.1\n Compiling num-traits v0.2.19\n Compiling rand_core v0.6.4\n Compiling blake3 v1.8.2\n Compiling spacetimedb-bindings-sys v1.6.0\n Compiling ppv-lite86 v0.2.21\n Compiling ethnum v1.5.2\n Compiling rand_chacha v0.3.1\n Compiling crypto-common v0.1.6\n Compiling block-buffer v0.10.4\n Compiling syn v2.0.107\n Compiling digest v0.10.7\n Compiling rand v0.8.5\n Compiling sha3 v0.10.8\n Compiling approx v0.3.2\n Compiling chrono v0.4.42\n Compiling decorum v0.3.1\n Compiling thiserror-impl v1.0.69\n Compiling spacetimedb-bindings-macro v1.6.0\n Compiling enum-as-inner v0.6.1\n Compiling derive_more v0.99.20\n Compiling spacetimedb-sats v1.6.0\n Compiling spacetimedb v1.6.0\n Compiling spacetime-module v0.1.0 (E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_002_scheduled_table\\rust\\server\\gpt-5\\llm)\nerror[E0599]: no method named `insert` found for reference `&tick_timer__TableHandle` in the current scope\n --> src\\lib.rs:20:25\n |\n20 | ctx.db.tick_timer().insert(TickTimer {\n | --------------------^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `try_insert` with a similar name\n |\n20 | ctx.db.tick_timer().try_insert(TickTimer {\n | ++++\n\nFor more information about this error, try `rustc --explain E0599`.\nerror: could not compile `spacetime-module` (lib) due to 1 previous error\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n",
"phase": "build_or_publish"
}
}
},
"vendor": "openai",
- "started_at": "2025-10-21T22:01:04.979448300Z",
- "finished_at": "2025-10-21T22:03:52.576952700Z"
+ "started_at": "2025-10-21T22:01:04.635852900Z",
+ "finished_at": "2025-10-21T22:03:52.544440900Z"
},
- "t_009_init": {
+ "t_003_struct_in_table": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_009_init",
+ "task": "t_003_struct_in_table",
"lang": "rust",
"golden_published": true,
"model_name": "GPT-5",
- "total_tests": 4,
- "passed_tests": 4,
- "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = users)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.users().insert(User {\n id: 1,\n name: \"Alice\".into(),\n age: 30,\n active: true,\n });\n ctx.db.users().insert(User {\n id: 2,\n name: \"Bob\".into(),\n age: 22,\n active: false,\n });\n}",
+ "total_tests": 1,
+ "passed_tests": 1,
+ "llm_output": "use spacetimedb::SpacetimeType;\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = entities)]\npub struct Entity {\n #[primary_key]\n id: i32,\n pos: Position,\n}",
"category": "basics",
"route_api_model": "gpt-5",
- "golden_db": "basics-t-009-init-golden",
- "llm_db": "basics-t-009-init-gpt-5-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_009_init\\rust\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_009_init\\rust\\server\\gpt-5\\llm",
+ "golden_db": "basics-t-003-struct-in-table-golden",
+ "llm_db": "basics-t-003-struct-in-table-gpt-5-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_003_struct_in_table\\rust\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_003_struct_in_table\\rust\\server\\gpt-5\\llm",
"scorer_details": {
- "init_total_two": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 2,
- "expected": 2,
- "sql": "SELECT COUNT(*) AS n FROM users"
- }
- },
- "init_seed_alice": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM users WHERE id=1 AND name='Alice' AND age=30 AND active=true"
- }
- },
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-009-init-golden",
- "llm_db": "basics-t-009-init-gpt-5-llm",
+ "golden_db": "basics-t-003-struct-in-table-golden",
+ "llm_db": "basics-t-003-struct-in-table-gpt-5-llm",
+ "reducers_diff": null,
+ "reducers_equal": true,
+ "server": "local",
+ "tables_diff": null,
+ "tables_equal": true
+ }
+ }
+ },
+ "vendor": "openai",
+ "started_at": "2025-10-21T22:01:04.701964800Z",
+ "finished_at": "2025-10-21T22:03:31.337265Z"
+ },
+ "t_004_insert": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_004_insert",
+ "lang": "rust",
+ "golden_published": true,
+ "model_name": "GPT-5",
+ "total_tests": 2,
+ "passed_tests": 0,
+ "llm_output": "use spacetimedb::{reducer, table, ReducerContext};\n\n#[spacetimedb::table(name = users)]\npub struct User {\n #[primary_key]\n pub id: i32,\n pub name: String,\n pub age: i32,\n pub active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn insert_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) -> Result<(), String> {\n ctx.db.users().try_insert(User { id, name, age, active })?;\n Ok(())\n}",
+ "category": "basics",
+ "route_api_model": "gpt-5",
+ "golden_db": "basics-t-004-insert-golden",
+ "llm_db": "basics-t-004-insert-gpt-5-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_004_insert\\rust\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_004_insert\\rust\\server\\gpt-5\\llm",
+ "scorer_details": {
+ "publish_error": {
+ "pass": false,
+ "partial": 0.0,
+ "notes": {
+ "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 67 packages to latest compatible versions\n Compiling proc-macro2 v1.0.101\n Compiling unicode-ident v1.0.20\n Compiling quote v1.0.41\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling cfg-if v1.0.4\n Compiling zerocopy v0.8.27\n Compiling either v1.15.0\n Compiling serde v1.0.228\n Compiling find-msvc-tools v0.1.4\n Compiling shlex v1.3.0\n Compiling nohash-hasher v0.2.0\n Compiling bitflags v2.10.0\n Compiling anyhow v1.0.100\n Compiling thiserror v1.0.69\n Compiling convert_case v0.4.0\n Compiling heck v0.5.0\n Compiling keccak v0.1.5\n Compiling heck v0.4.1\n Compiling humantime v2.3.0\n Compiling arrayvec v0.7.6\n Compiling constant_time_eq v0.3.1\n Compiling bytes v1.10.1\n Compiling smallvec v1.15.1\n Compiling hex v0.4.3\n Compiling spacetimedb-lib v1.6.0\n Compiling second-stack v0.3.5\n Compiling getrandom v0.2.16\n Compiling arrayref v0.3.9\n Compiling bytemuck v1.24.0\n Compiling itertools v0.12.1\n Compiling log v0.4.28\n Compiling cc v1.2.41\n Compiling rand_core v0.6.4\n Compiling scoped-tls v1.0.1\n Compiling generic-array v0.14.9\n Compiling num-traits v0.2.19\n Compiling blake3 v1.8.2\n Compiling syn v2.0.107\n Compiling spacetimedb-primitives v1.6.0\n Compiling approx v0.3.2\n Compiling chrono v0.4.42\n Compiling crypto-common v0.1.6\n Compiling block-buffer v0.10.4\n Compiling decorum v0.3.1\n Compiling digest v0.10.7\n Compiling spacetimedb-bindings-sys v1.6.0\n Compiling sha3 v0.10.8\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling ethnum v1.5.2\n Compiling thiserror-impl v1.0.69\n Compiling spacetimedb-bindings-macro v1.6.0\n Compiling enum-as-inner v0.6.1\n Compiling derive_more v0.99.20\n Compiling spacetimedb-sats v1.6.0\n Compiling spacetimedb v1.6.0\n Compiling spacetime-module v0.1.0 (E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_004_insert\\rust\\server\\gpt-5\\llm)\nwarning: unused imports: `reducer` and `table`\n --> src\\lib.rs:2:19\n |\n2 | use spacetimedb::{reducer, table, ReducerContext};\n | ^^^^^^^ ^^^^^\n |\n = note: `#[warn(unused_imports)]` on by default\n\nerror[E0599]: no method named `try_insert` found for reference `&users__TableHandle` in the current scope\n --> src\\lib.rs:15:20\n |\n15 | ctx.db.users().try_insert(User { id, name, age, active })?;\n | ^^^^^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `try_insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `insert` with a similar name\n |\n15 - ctx.db.users().try_insert(User { id, name, age, active })?;\n15 + ctx.db.users().insert(User { id, name, age, active })?;\n |\n\nFor more information about this error, try `rustc --explain E0599`.\nwarning: `spacetime-module` (lib) generated 1 warning\nerror: could not compile `spacetime-module` (lib) due to 1 previous error; 1 warning emitted\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n",
+ "phase": "build_or_publish"
+ }
+ }
+ },
+ "vendor": "openai",
+ "started_at": "2025-10-21T22:01:04.782126700Z",
+ "finished_at": "2025-10-21T22:02:20.360731800Z"
+ },
+ "t_005_update": {
+ "hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
+ "task": "t_005_update",
+ "lang": "rust",
+ "golden_published": true,
+ "model_name": "GPT-5",
+ "total_tests": 3,
+ "passed_tests": 3,
+ "llm_output": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(name = users)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) -> Result<(), String> {\n let user = User { id, name, age, active };\n ctx.db.users().id().update(user);\n Ok(())\n}",
+ "category": "basics",
+ "route_api_model": "gpt-5",
+ "golden_db": "basics-t-005-update-golden",
+ "llm_db": "basics-t-005-update-gpt-5-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_005_update\\rust\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_005_update\\rust\\server\\gpt-5\\llm",
+ "scorer_details": {
+ "schema_parity": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "golden_db": "basics-t-005-update-golden",
+ "llm_db": "basics-t-005-update-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -16177,19 +16232,36 @@
"tables_equal": true
}
},
- "init_seed_bob": {
+ "data_parity_update_user": {
"pass": true,
"partial": 1.0,
"notes": {
- "actual": 1,
- "expected": 1,
- "sql": "SELECT COUNT(*) AS n FROM users WHERE id=2 AND name='Bob' AND age=22 AND active=false"
+ "args": [
+ 1,
+ "Alice2",
+ 31,
+ false
+ ],
+ "golden_db": "basics-t-005-update-golden",
+ "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
+ "llm_db": "basics-t-005-update-gpt-5-llm",
+ "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false",
+ "query": "SELECT id, name, age, active FROM users WHERE id=1",
+ "reducer": "update_user",
+ "server": "local"
+ }
+ },
+ "seed_users_row": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "sql": "INSERT INTO users(id, name, age, active) VALUES (1, 'Alice', 30, true)"
}
}
},
"vendor": "openai",
- "started_at": "2025-10-21T22:01:05.093863600Z",
- "finished_at": "2025-10-21T22:03:55.340861800Z"
+ "started_at": "2025-10-21T22:01:04.855103100Z",
+ "finished_at": "2025-10-21T22:03:56.020393200Z"
},
"t_006_delete": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
@@ -16241,125 +16313,113 @@
"started_at": "2025-10-21T22:01:04.928492200Z",
"finished_at": "2025-10-21T22:03:26.113027400Z"
},
- "t_000_empty_reducers": {
- "hash": "00917b28ec8f75406cd8aac609eeafb254b65945d2bb0c709404ac09a9762fca",
- "task": "t_000_empty_reducers",
- "lang": "rust",
- "golden_published": true,
- "model_name": "GPT-5",
- "total_tests": 1,
- "passed_tests": 1,
- "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::reducer]\npub fn empty_reducer_no_args(_ctx: &ReducerContext) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_int(_ctx: &ReducerContext, _count: i32) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_string(_ctx: &ReducerContext, _name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_two_args(_ctx: &ReducerContext, _count: i32, _name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_three_args(_ctx: &ReducerContext, _active: bool, _ratio: f32, _label: String) {}",
- "category": "basics",
- "route_api_model": "gpt-5",
- "golden_db": "basics-t-000-empty-reducers-golden",
- "llm_db": "basics-t-000-empty-reducers-gpt-5-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_000_empty_reducers\\rust\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_000_empty_reducers\\rust\\server\\gpt-5\\llm",
- "scorer_details": {
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
- "notes": {
- "golden_db": "basics-t-000-empty-reducers-golden",
- "llm_db": "basics-t-000-empty-reducers-gpt-5-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
- }
- }
- },
- "vendor": "openai",
- "started_at": "2025-10-24T21:13:14.021183600Z",
- "finished_at": "2025-10-24T21:14:01.165125200Z"
- },
- "t_001_basic_tables": {
+ "t_007_crud": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_001_basic_tables",
+ "task": "t_007_crud",
"lang": "rust",
"golden_published": true,
"model_name": "GPT-5",
- "total_tests": 1,
- "passed_tests": 1,
- "llm_output": "use spacetimedb as _;\n\n#[spacetimedb::table(name = users, public)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = products, public)]\npub struct Product {\n #[primary_key]\n id: i32,\n title: String,\n price: f32,\n in_stock: bool,\n}\n\n#[spacetimedb::table(name = notes, public)]\npub struct Note {\n #[primary_key]\n id: i32,\n body: String,\n rating: i64,\n pinned: bool,\n}",
+ "total_tests": 4,
+ "passed_tests": 0,
+ "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::table(name = users)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn crud(ctx: &ReducerContext) {\n // insert (id=1, name=\"Alice\", age=30, active=true)\n ctx.db.users().insert(User {\n id: 1,\n name: \"Alice\".to_string(),\n age: 30,\n active: true,\n });\n\n // insert (id=2, name=\"Bob\", age=22, active=false)\n ctx.db.users().insert(User {\n id: 2,\n name: \"Bob\".to_string(),\n age: 22,\n active: false,\n });\n\n // update (id=1, name=\"Alice2\", age=31, active=false)\n ctx.db.users().id().update(User {\n id: 1,\n name: \"Alice2\".to_string(),\n age: 31,\n active: false,\n });\n\n // delete id=2\n ctx.db.users().id().delete(2);\n}",
"category": "basics",
"route_api_model": "gpt-5",
- "golden_db": "basics-t-001-basic-tables-golden",
- "llm_db": "basics-t-001-basic-tables-gpt-5-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_001_basic_tables\\rust\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_001_basic_tables\\rust\\server\\gpt-5\\llm",
+ "golden_db": "basics-t-007-crud-golden",
+ "llm_db": "basics-t-007-crud-gpt-5-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_007_crud\\rust\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_007_crud\\rust\\server\\gpt-5\\llm",
"scorer_details": {
- "schema_parity": {
- "pass": true,
- "partial": 1.0,
+ "publish_error": {
+ "pass": false,
+ "partial": 0.0,
"notes": {
- "golden_db": "basics-t-001-basic-tables-golden",
- "llm_db": "basics-t-001-basic-tables-gpt-5-llm",
- "reducers_diff": null,
- "reducers_equal": true,
- "server": "local",
- "tables_diff": null,
- "tables_equal": true
+ "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Blocking waiting for file lock on package cache\n Updating crates.io index\n Blocking waiting for file lock on package cache\n Locking 67 packages to latest compatible versions\n Blocking waiting for file lock on package cache\n Blocking waiting for file lock on package cache\n Blocking waiting for file lock on package cache\n Compiling proc-macro2 v1.0.101\n Compiling quote v1.0.41\n Compiling unicode-ident v1.0.20\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling cfg-if v1.0.4\n Compiling serde_core v1.0.228\n Compiling zerocopy v0.8.27\n Compiling find-msvc-tools v0.1.4\n Compiling serde v1.0.228\n Compiling shlex v1.3.0\n Compiling either v1.15.0\n Compiling nohash-hasher v0.2.0\n Compiling bitflags v2.10.0\n Compiling anyhow v1.0.100\n Compiling thiserror v1.0.69\n Compiling humantime v2.3.0\n Compiling arrayvec v0.7.6\n Compiling heck v0.4.1\n Compiling keccak v0.1.5\n Compiling convert_case v0.4.0\n Compiling heck v0.5.0\n Compiling hex v0.4.3\n Compiling bytemuck v1.24.0\n Compiling second-stack v0.3.5\n Compiling spacetimedb-lib v1.6.0\n Compiling constant_time_eq v0.3.1\n Compiling bytes v1.10.1\n Compiling getrandom v0.2.16\n Compiling arrayref v0.3.9\n Compiling cc v1.2.41\n Compiling itertools v0.12.1\n Compiling rand_core v0.6.4\n Compiling smallvec v1.15.1\n Compiling log v0.4.28\n Compiling scoped-tls v1.0.1\n Compiling generic-array v0.14.9\n Compiling num-traits v0.2.19\n Compiling spacetimedb-primitives v1.6.0\n Compiling blake3 v1.8.2\n Compiling spacetimedb-bindings-sys v1.6.0\n Compiling ppv-lite86 v0.2.21\n Compiling ethnum v1.5.2\n Compiling syn v2.0.107\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling crypto-common v0.1.6\n Compiling block-buffer v0.10.4\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling approx v0.3.2\n Compiling chrono v0.4.42\n Compiling decorum v0.3.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling enum-as-inner v0.6.1\n Compiling spacetimedb-bindings-macro v1.6.0\n Compiling spacetimedb-sats v1.6.0\n Compiling spacetimedb v1.6.0\n Compiling spacetime-module v0.1.0 (E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_007_crud\\rust\\server\\gpt-5\\llm)\nerror[E0599]: no method named `insert` found for reference `&users__TableHandle` in the current scope\n --> src\\lib.rs:16:20\n |\n16 | ctx.db.users().insert(User {\n | ---------------^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `try_insert` with a similar name\n |\n16 | ctx.db.users().try_insert(User {\n | ++++\n\nerror[E0599]: no method named `insert` found for reference `&users__TableHandle` in the current scope\n --> src\\lib.rs:24:20\n |\n24 | ctx.db.users().insert(User {\n | ---------------^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `try_insert` with a similar name\n |\n24 | ctx.db.users().try_insert(User {\n | ++++\n\nFor more information about this error, try `rustc --explain E0599`.\nerror: could not compile `spacetime-module` (lib) due to 2 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n",
+ "phase": "build_or_publish"
}
}
},
"vendor": "openai",
- "started_at": "2025-10-21T22:01:04.558378900Z",
- "finished_at": "2025-10-21T22:02:18.806241100Z"
+ "started_at": "2025-10-21T22:01:04.979448300Z",
+ "finished_at": "2025-10-21T22:03:52.576952700Z"
},
- "t_021_multi_column_index": {
+ "t_008_index_lookup": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_021_multi_column_index",
+ "task": "t_008_index_lookup",
"lang": "rust",
"golden_published": true,
"model_name": "GPT-5",
- "total_tests": 4,
+ "total_tests": 3,
"passed_tests": 0,
- "llm_output": "use spacetimedb::{reducer, ReducerContext};\n\n#[spacetimedb::table(name = logs, index(name = by_user_day, btree = [user_id, day]))]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.logs().insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".to_string() });\n ctx.db.logs().insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".to_string() });\n ctx.db.logs().insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".to_string() });\n}",
- "category": "schema",
+ "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::table(name = users)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = results)]\npub struct Result {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(user) = ctx.db.users().id().find(id) {\n let name = user.name.clone();\n if ctx.db.results().id().find(id).is_some() {\n ctx.db.results().id().update(Result { id, name });\n } else {\n ctx.db.results().insert(Result { id, name });\n }\n }\n}",
+ "category": "basics",
"route_api_model": "gpt-5",
- "golden_db": "schema-t-021-multi-column-index-golden",
- "llm_db": "schema-t-021-multi-column-index-gpt-5-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_021_multi_column_index\\rust\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_021_multi_column_index\\rust\\server\\gpt-5\\llm",
+ "golden_db": "basics-t-008-index-lookup-golden",
+ "llm_db": "basics-t-008-index-lookup-gpt-5-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_008_index_lookup\\rust\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_008_index_lookup\\rust\\server\\gpt-5\\llm",
"scorer_details": {
"publish_error": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 67 packages to latest compatible versions\n Blocking waiting for file lock on package cache\n Blocking waiting for file lock on package cache\n Blocking waiting for file lock on package cache\n Compiling proc-macro2 v1.0.101\n Compiling quote v1.0.41\n Compiling unicode-ident v1.0.20\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling cfg-if v1.0.4\n Compiling serde v1.0.228\n Compiling find-msvc-tools v0.1.4\n Compiling either v1.15.0\n Compiling zerocopy v0.8.27\n Compiling shlex v1.3.0\n Compiling anyhow v1.0.100\n Compiling nohash-hasher v0.2.0\n Compiling thiserror v1.0.69\n Compiling bitflags v2.10.0\n Compiling arrayvec v0.7.6\n Compiling heck v0.5.0\n Compiling humantime v2.3.0\n Compiling convert_case v0.4.0\n Compiling heck v0.4.1\n Compiling keccak v0.1.5\n Compiling constant_time_eq v0.3.1\n Compiling bytes v1.10.1\n Compiling spacetimedb-lib v1.6.0\n Compiling second-stack v0.3.5\n Compiling smallvec v1.15.1\n Compiling bytemuck v1.24.0\n Compiling cc v1.2.41\n Compiling itertools v0.12.1\n Compiling getrandom v0.2.16\n Compiling arrayref v0.3.9\n Compiling hex v0.4.3\n Compiling scoped-tls v1.0.1\n Compiling log v0.4.28\n Compiling rand_core v0.6.4\n Compiling spacetimedb-primitives v1.6.0\n Compiling num-traits v0.2.19\n Compiling generic-array v0.14.9\n Compiling blake3 v1.8.2\n Compiling spacetimedb-bindings-sys v1.6.0\n Compiling syn v2.0.107\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling ethnum v1.5.2\n Compiling rand v0.8.5\n Compiling crypto-common v0.1.6\n Compiling block-buffer v0.10.4\n Compiling approx v0.3.2\n Compiling chrono v0.4.42\n Compiling digest v0.10.7\n Compiling decorum v0.3.1\n Compiling sha3 v0.10.8\n Compiling thiserror-impl v1.0.69\n Compiling enum-as-inner v0.6.1\n Compiling spacetimedb-bindings-macro v1.6.0\n Compiling derive_more v0.99.20\n Compiling spacetimedb-sats v1.6.0\n Compiling spacetimedb v1.6.0\n Compiling spacetime-module v0.1.0 (E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_021_multi_column_index\\rust\\server\\gpt-5\\llm)\nerror: expected parentheses\n --> src\\lib.rs:4:67\n |\n4 | #[spacetimedb::table(name = logs, index(name = by_user_day, btree = [user_id, day]))]\n | ^\n\nerror[E0422]: cannot find struct, variant or union type `Log` in this scope\n --> src\\lib.rs:15:26\n |\n15 | ctx.db.logs().insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".to_string() });\n | ^^^ not found in this scope\n\nerror[E0422]: cannot find struct, variant or union type `Log` in this scope\n --> src\\lib.rs:16:26\n |\n16 | ctx.db.logs().insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".to_string() });\n | ^^^ not found in this scope\n\nerror[E0422]: cannot find struct, variant or union type `Log` in this scope\n --> src\\lib.rs:17:26\n |\n17 | ctx.db.logs().insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".to_string() });\n | ^^^ not found in this scope\n\nerror[E0599]: no method named `logs` found for struct `Local` in the current scope\n --> src\\lib.rs:15:12\n |\n15 | ctx.db.logs().insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".to_string() });\n | ^^^^ method not found in `Local`\n\nerror[E0599]: no method named `logs` found for struct `Local` in the current scope\n --> src\\lib.rs:16:12\n |\n16 | ctx.db.logs().insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".to_string() });\n | ^^^^ method not found in `Local`\n\nerror[E0599]: no method named `logs` found for struct `Local` in the current scope\n --> src\\lib.rs:17:12\n |\n17 | ctx.db.logs().insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".to_string() });\n | ^^^^ method not found in `Local`\n\nSome errors have detailed explanations: E0422, E0599.\nFor more information about an error, try `rustc --explain E0422`.\nerror: could not compile `spacetime-module` (lib) due to 7 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n",
+ "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Blocking waiting for file lock on package cache\n Locking 67 packages to latest compatible versions\n Blocking waiting for file lock on package cache\n Blocking waiting for file lock on package cache\n Blocking waiting for file lock on package cache\n Compiling proc-macro2 v1.0.101\n Compiling unicode-ident v1.0.20\n Compiling quote v1.0.41\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling cfg-if v1.0.4\n Compiling serde_core v1.0.228\n Compiling shlex v1.3.0\n Compiling zerocopy v0.8.27\n Compiling find-msvc-tools v0.1.4\n Compiling either v1.15.0\n Compiling serde v1.0.228\n Compiling nohash-hasher v0.2.0\n Compiling bitflags v2.10.0\n Compiling anyhow v1.0.100\n Compiling thiserror v1.0.69\n Compiling humantime v2.3.0\n Compiling heck v0.5.0\n Compiling arrayvec v0.7.6\n Compiling convert_case v0.4.0\n Compiling keccak v0.1.5\n Compiling heck v0.4.1\n Compiling arrayref v0.3.9\n Compiling spacetimedb-lib v1.6.0\n Compiling smallvec v1.15.1\n Compiling second-stack v0.3.5\n Compiling hex v0.4.3\n Compiling constant_time_eq v0.3.1\n Compiling getrandom v0.2.16\n Compiling bytes v1.10.1\n Compiling bytemuck v1.24.0\n Compiling scoped-tls v1.0.1\n Compiling rand_core v0.6.4\n Compiling log v0.4.28\n Compiling itertools v0.12.1\n Compiling cc v1.2.41\n Compiling generic-array v0.14.9\n Compiling num-traits v0.2.19\n Compiling spacetimedb-primitives v1.6.0\n Compiling blake3 v1.8.2\n Compiling ethnum v1.5.2\n Compiling spacetimedb-bindings-sys v1.6.0\n Compiling ppv-lite86 v0.2.21\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.6\n Compiling digest v0.10.7\n Compiling rand_chacha v0.3.1\n Compiling sha3 v0.10.8\n Compiling syn v2.0.107\n Compiling approx v0.3.2\n Compiling chrono v0.4.42\n Compiling rand v0.8.5\n Compiling decorum v0.3.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling enum-as-inner v0.6.1\n Compiling spacetimedb-bindings-macro v1.6.0\n Compiling spacetimedb-sats v1.6.0\n Compiling spacetimedb v1.6.0\n Compiling spacetime-module v0.1.0 (E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\rust\\server\\gpt-5\\llm)\nerror[E0107]: struct takes 0 generic arguments but 2 generic arguments were supplied\n --> src\\lib.rs:4:1\n |\n 4 | #[spacetimedb::table(name = users)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected 0 generic arguments\n |\nnote: struct defined here, with 0 generic parameters\n --> src\\lib.rs:14:12\n |\n14 | pub struct Result {\n | ^^^^^^\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `deserialize` has an incompatible type for trait\n --> src\\lib.rs:4:1\n |\n4 | #[spacetimedb::table(name = users)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `Result`, found `Result`\n |\n = note: expected signature `fn(_) -> std::result::Result>::Error>`\n found signature `fn(_) -> Result`\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `visit_seq_product` has an incompatible type for trait\n --> src\\lib.rs:4:1\n |\n4 | #[spacetimedb::table(name = users)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `Result`, found `Result`\n |\n = note: expected signature `fn(_::__ProductVisitor, _) -> std::result::Result>::Error>`\n found signature `fn(_::__ProductVisitor, _) -> Result`\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `visit_named_product` has an incompatible type for trait\n --> src\\lib.rs:4:1\n |\n4 | #[spacetimedb::table(name = users)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `Result`, found `Result`\n |\n = note: expected signature `fn(_::__ProductVisitor, _) -> std::result::Result>::Error>`\n found signature `fn(_::__ProductVisitor, _) -> Result`\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `visit` has an incompatible type for trait\n --> src\\lib.rs:4:1\n |\n4 | #[spacetimedb::table(name = users)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `Result<__ProductFieldIdent, __E>`, found `Result`\n |\n = note: expected signature `fn(_::__ProductVisitor, &_) -> std::result::Result<_::__ProductFieldIdent, __E>`\n found signature `fn(_::__ProductVisitor, &_) -> Result`\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `serialize` has an incompatible type for trait\n --> src\\lib.rs:4:1\n |\n4 | #[spacetimedb::table(name = users)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `Result<::Ok, ...>`, found `Result`\n |\n = note: expected signature `fn(&User, _) -> std::result::Result<::Ok, ::Error>`\n found signature `fn(&User, _) -> Result`\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0107]: struct takes 0 generic arguments but 2 generic arguments were supplied\n --> src\\lib.rs:13:1\n |\n13 | #[spacetimedb::table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected 0 generic arguments\n |\nnote: struct defined here, with 0 generic parameters\n --> src\\lib.rs:14:12\n |\n14 | pub struct Result {\n | ^^^^^^\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `deserialize` has an incompatible type for trait\n --> src\\lib.rs:13:1\n |\n13 | #[spacetimedb::table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `Result`, found `Result`\n |\n = note: expected signature `fn(_) -> std::result::Result>::Error>`\n found signature `fn(_) -> Result`\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `visit_seq_product` has an incompatible type for trait\n --> src\\lib.rs:13:1\n |\n13 | #[spacetimedb::table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `Result`, found `Result`\n |\n = note: expected signature `fn(_::__ProductVisitor, _) -> std::result::Result>::Error>`\n found signature `fn(_::__ProductVisitor, _) -> Result`\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `visit_named_product` has an incompatible type for trait\n --> src\\lib.rs:13:1\n |\n13 | #[spacetimedb::table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `Result`, found `Result`\n |\n = note: expected signature `fn(_::__ProductVisitor, _) -> std::result::Result>::Error>`\n found signature `fn(_::__ProductVisitor, _) -> Result`\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `visit` has an incompatible type for trait\n --> src\\lib.rs:13:1\n |\n13 | #[spacetimedb::table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `Result<__ProductFieldIdent, __E>`, found `Result`\n |\n = note: expected signature `fn(_::__ProductVisitor, &_) -> std::result::Result<_::__ProductFieldIdent, __E>`\n found signature `fn(_::__ProductVisitor, &_) -> Result`\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `serialize` has an incompatible type for trait\n --> src\\lib.rs:13:1\n |\n13 | #[spacetimedb::table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `Result<::Ok, ...>`, found `Result`\n |\n = note: expected signature `fn(&Result, _) -> std::result::Result<::Ok, ::Error>`\n found signature `fn(&Result, _) -> Result`\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0308]: mismatched types\n --> src\\lib.rs:4:1\n |\n 4 | #[spacetimedb::table(name = users)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n | |\n | expected `Result`, found `Result`\n | expected `Result` because of return type\n |\n = note: `Result` and `Result` have similar names, but are actually distinct types\nnote: `Result` is defined in crate `core`\n --> C:\\Users\\bradl\\.rustup\\toolchains\\1.90.0-x86_64-pc-windows-msvc\\lib/rustlib/src/rust\\library/core/src/result.rs:548:1\n |\n548 | pub enum Result {\n | ^^^^^^^^^^^^^^^^^^^^^\nnote: `Result` is defined in the current crate\n --> src\\lib.rs:14:1\n |\n 14 | pub struct Result {\n | ^^^^^^^^^^^^^^^^^\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0277]: the `?` operator can only be used in a method that returns `Result` or `Option` (or another type that implements `FromResidual`)\n --> src\\lib.rs:4:35\n |\n4 | #[spacetimedb::table(name = users)]\n | ----------------------------------^\n | | |\n | | cannot use the `?` operator in a method that returns `Result`\n | this function should return `Result` or `Option` to accept `?`\n |\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` which comes from the expansion of the attribute macro `spacetimedb::table` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0308]: mismatched types\n --> src\\lib.rs:4:1\n |\n 4 | #[spacetimedb::table(name = users)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n | |\n | expected `Result`, found `Result`\n | expected `Result` because of return type\n |\n = note: `std::result::Result` and `Result` have similar names, but are actually distinct types\nnote: `std::result::Result` is defined in crate `core`\n --> C:\\Users\\bradl\\.rustup\\toolchains\\1.90.0-x86_64-pc-windows-msvc\\lib/rustlib/src/rust\\library/core/src/result.rs:548:1\n |\n548 | pub enum Result {\n | ^^^^^^^^^^^^^^^^^^^^^\nnote: `Result` is defined in the current crate\n --> src\\lib.rs:14:1\n |\n 14 | pub struct Result {\n | ^^^^^^^^^^^^^^^^^\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0308]: mismatched types\n --> src\\lib.rs:4:1\n |\n 4 | #[spacetimedb::table(name = users)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n | |\n | expected `Result`, found `Result<_, _>`\n | expected `Result` because of return type\n |\n = note: `std::result::Result<_, _>` and `Result` have similar names, but are actually distinct types\nnote: `std::result::Result<_, _>` is defined in crate `core`\n --> C:\\Users\\bradl\\.rustup\\toolchains\\1.90.0-x86_64-pc-windows-msvc\\lib/rustlib/src/rust\\library/core/src/result.rs:548:1\n |\n548 | pub enum Result {\n | ^^^^^^^^^^^^^^^^^^^^^\nnote: `Result` is defined in the current crate\n --> src\\lib.rs:14:1\n |\n 14 | pub struct Result {\n | ^^^^^^^^^^^^^^^^^\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0308]: mismatched types\n --> src\\lib.rs:4:1\n |\n 4 | #[spacetimedb::table(name = users)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n | |\n | expected `Result`, found `Result<__ProductFieldIdent, _>`\n | expected `Result` because of return type\n |\n = note: `Result<__ProductFieldIdent, _>` and `Result` have similar names, but are actually distinct types\nnote: `Result<__ProductFieldIdent, _>` is defined in crate `core`\n --> C:\\Users\\bradl\\.rustup\\toolchains\\1.90.0-x86_64-pc-windows-msvc\\lib/rustlib/src/rust\\library/core/src/result.rs:548:1\n |\n548 | pub enum Result {\n | ^^^^^^^^^^^^^^^^^^^^^\nnote: `Result` is defined in the current crate\n --> src\\lib.rs:14:1\n |\n 14 | pub struct Result {\n | ^^^^^^^^^^^^^^^^^\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0308]: mismatched types\n --> src\\lib.rs:4:1\n |\n 4 | #[spacetimedb::table(name = users)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n | |\n | expected `Result`, found `Result<::Ok, ...>`\n | expected `Result` because of return type\n |\n = note: `Result<::Ok, ...>` and `Result` have similar names, but are actually distinct types\nnote: `Result<::Ok, ...>` is defined in crate `core`\n --> C:\\Users\\bradl\\.rustup\\toolchains\\1.90.0-x86_64-pc-windows-msvc\\lib/rustlib/src/rust\\library/core/src/result.rs:548:1\n |\n548 | pub enum Result {\n | ^^^^^^^^^^^^^^^^^^^^^\nnote: `Result` is defined in the current crate\n --> src\\lib.rs:14:1\n |\n 14 | pub struct Result {\n | ^^^^^^^^^^^^^^^^^\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0308]: mismatched types\n --> src\\lib.rs:13:1\n |\n 13 | #[spacetimedb::table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n | |\n | expected `Result`, found `Result`\n | expected `Result` because of return type\n |\n = note: `Result` and `Result` have similar names, but are actually distinct types\nnote: `Result` is defined in crate `core`\n --> C:\\Users\\bradl\\.rustup\\toolchains\\1.90.0-x86_64-pc-windows-msvc\\lib/rustlib/src/rust\\library/core/src/result.rs:548:1\n |\n548 | pub enum Result {\n | ^^^^^^^^^^^^^^^^^^^^^\nnote: `Result` is defined in the current crate\n --> src\\lib.rs:14:1\n |\n 14 | pub struct Result {\n | ^^^^^^^^^^^^^^^^^\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\nhelp: consider using `Result::expect` to unwrap the `std::result::Result>::Error>` value, panicking if the value is a `Result::Err`\n |\n 13 | #[spacetimedb::table(name = results)].expect(\"REASON\")\n | +++++++++++++++++\n\nerror[E0277]: the `?` operator can only be used in a method that returns `Result` or `Option` (or another type that implements `FromResidual`)\n --> src\\lib.rs:13:37\n |\n13 | #[spacetimedb::table(name = results)]\n | ------------------------------------^\n | | |\n | | cannot use the `?` operator in a method that returns `Result`\n | this function should return `Result` or `Option` to accept `?`\n |\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` which comes from the expansion of the attribute macro `spacetimedb::table` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0308]: mismatched types\n --> src\\lib.rs:13:1\n |\n 13 | #[spacetimedb::table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n | |\n | expected `Result`, found `Result`\n | expected `Result` because of return type\n |\n = note: `std::result::Result` and `Result` have similar names, but are actually distinct types\nnote: `std::result::Result` is defined in crate `core`\n --> C:\\Users\\bradl\\.rustup\\toolchains\\1.90.0-x86_64-pc-windows-msvc\\lib/rustlib/src/rust\\library/core/src/result.rs:548:1\n |\n548 | pub enum Result {\n | ^^^^^^^^^^^^^^^^^^^^^\nnote: `Result` is defined in the current crate\n --> src\\lib.rs:14:1\n |\n 14 | pub struct Result {\n | ^^^^^^^^^^^^^^^^^\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0308]: mismatched types\n --> src\\lib.rs:13:1\n |\n 13 | #[spacetimedb::table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n | |\n | expected `Result`, found `Result<_, _>`\n | expected `Result` because of return type\n |\n = note: `std::result::Result<_, _>` and `Result` have similar names, but are actually distinct types\nnote: `std::result::Result<_, _>` is defined in crate `core`\n --> C:\\Users\\bradl\\.rustup\\toolchains\\1.90.0-x86_64-pc-windows-msvc\\lib/rustlib/src/rust\\library/core/src/result.rs:548:1\n |\n548 | pub enum Result {\n | ^^^^^^^^^^^^^^^^^^^^^\nnote: `Result` is defined in the current crate\n --> src\\lib.rs:14:1\n |\n 14 | pub struct Result {\n | ^^^^^^^^^^^^^^^^^\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0308]: mismatched types\n --> src\\lib.rs:13:1\n |\n 13 | #[spacetimedb::table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n | |\n | expected `Result`, found `Result<__ProductFieldIdent, _>`\n | expected `Result` because of return type\n |\n = note: `Result<__ProductFieldIdent, _>` and `Result` have similar names, but are actually distinct types\nnote: `Result<__ProductFieldIdent, _>` is defined in crate `core`\n --> C:\\Users\\bradl\\.rustup\\toolchains\\1.90.0-x86_64-pc-windows-msvc\\lib/rustlib/src/rust\\library/core/src/result.rs:548:1\n |\n548 | pub enum Result {\n | ^^^^^^^^^^^^^^^^^^^^^\nnote: `Result` is defined in the current crate\n --> src\\lib.rs:14:1\n |\n 14 | pub struct Result {\n | ^^^^^^^^^^^^^^^^^\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0308]: mismatched types\n --> src\\lib.rs:13:1\n |\n 13 | #[spacetimedb::table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n | |\n | expected `Result`, found `Result<::Ok, ...>`\n | expected `Result` because of return type\n |\n = note: `Result<::Ok, ...>` and `Result` have similar names, but are actually distinct types\nnote: `Result<::Ok, ...>` is defined in crate `core`\n --> C:\\Users\\bradl\\.rustup\\toolchains\\1.90.0-x86_64-pc-windows-msvc\\lib/rustlib/src/rust\\library/core/src/result.rs:548:1\n |\n548 | pub enum Result {\n | ^^^^^^^^^^^^^^^^^^^^^\nnote: `Result` is defined in the current crate\n --> src\\lib.rs:14:1\n |\n 14 | pub struct Result {\n | ^^^^^^^^^^^^^^^^^\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0599]: no method named `insert` found for reference `&results__TableHandle` in the current scope\n --> src\\lib.rs:27:30\n |\n27 | ctx.db.results().insert(Result { id, name });\n | ^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `try_insert` with a similar name\n |\n27 | ctx.db.results().try_insert(Result { id, name });\n | ++++\n\nSome errors have detailed explanations: E0053, E0107, E0277, E0308, E0599.\nFor more information about an error, try `rustc --explain E0053`.\nerror: could not compile `spacetime-module` (lib) due to 29 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n",
"phase": "build_or_publish"
}
}
},
"vendor": "openai",
- "started_at": "2025-10-21T22:02:18.830117100Z",
- "finished_at": "2025-10-21T22:03:58.105565200Z"
+ "started_at": "2025-10-21T22:01:05.033897200Z",
+ "finished_at": "2025-10-21T22:03:49.234671200Z"
},
- "t_003_struct_in_table": {
+ "t_009_init": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_003_struct_in_table",
+ "task": "t_009_init",
"lang": "rust",
"golden_published": true,
"model_name": "GPT-5",
- "total_tests": 1,
- "passed_tests": 1,
- "llm_output": "use spacetimedb::SpacetimeType;\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = entities)]\npub struct Entity {\n #[primary_key]\n id: i32,\n pos: Position,\n}",
+ "total_tests": 4,
+ "passed_tests": 4,
+ "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = users)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.users().insert(User {\n id: 1,\n name: \"Alice\".into(),\n age: 30,\n active: true,\n });\n ctx.db.users().insert(User {\n id: 2,\n name: \"Bob\".into(),\n age: 22,\n active: false,\n });\n}",
"category": "basics",
"route_api_model": "gpt-5",
- "golden_db": "basics-t-003-struct-in-table-golden",
- "llm_db": "basics-t-003-struct-in-table-gpt-5-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_003_struct_in_table\\rust\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_003_struct_in_table\\rust\\server\\gpt-5\\llm",
+ "golden_db": "basics-t-009-init-golden",
+ "llm_db": "basics-t-009-init-gpt-5-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_009_init\\rust\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_009_init\\rust\\server\\gpt-5\\llm",
"scorer_details": {
+ "init_seed_alice": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM users WHERE id=1 AND name='Alice' AND age=30 AND active=true"
+ }
+ },
+ "init_total_two": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 2,
+ "expected": 2,
+ "sql": "SELECT COUNT(*) AS n FROM users"
+ }
+ },
+ "init_seed_bob": {
+ "pass": true,
+ "partial": 1.0,
+ "notes": {
+ "actual": 1,
+ "expected": 1,
+ "sql": "SELECT COUNT(*) AS n FROM users WHERE id=2 AND name='Bob' AND age=22 AND active=false"
+ }
+ },
"schema_parity": {
"pass": true,
"partial": 1.0,
"notes": {
- "golden_db": "basics-t-003-struct-in-table-golden",
- "llm_db": "basics-t-003-struct-in-table-gpt-5-llm",
+ "golden_db": "basics-t-009-init-golden",
+ "llm_db": "basics-t-009-init-gpt-5-llm",
"reducers_diff": null,
"reducers_equal": true,
"server": "local",
@@ -16369,242 +16429,182 @@
}
},
"vendor": "openai",
- "started_at": "2025-10-21T22:01:04.701964800Z",
- "finished_at": "2025-10-21T22:03:31.337265Z"
+ "started_at": "2025-10-21T22:01:05.093863600Z",
+ "finished_at": "2025-10-21T22:03:55.340861800Z"
},
- "t_011_helper_function": {
+ "t_010_connect": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_011_helper_function",
+ "task": "t_010_connect",
"lang": "rust",
"golden_published": true,
"model_name": "GPT-5",
- "total_tests": 3,
+ "total_tests": 1,
"passed_tests": 0,
- "llm_output": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[spacetimedb::table(name = results)]\npub struct Result {\n #[primary_key]\n pub id: i32,\n pub sum: i32,\n}\n\nfn add(a: i32, b: i32) -> i32 {\n a + b\n}\n\n#[spacetimedb::reducer]\npub fn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) {\n let sum = add(a, b);\n ctx.db.results().insert(Result { id, sum });\n}",
+ "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::table(name = events)]\npub struct Event {\n #[primary_key]\n #[auto_inc]\n id: i32,\n kind: String,\n}\n\n#[spacetimedb::reducer(client_connected)]\npub fn client_connected(ctx: &ReducerContext) {\n ctx.db.events().insert(Event {\n id: 0,\n kind: \"connected\".to_string(),\n });\n}\n\n#[spacetimedb::reducer(client_disconnected)]\npub fn client_disconnected(ctx: &ReducerContext) {\n ctx.db.events().insert(Event {\n id: 0,\n kind: \"disconnected\".to_string(),\n });\n}",
"category": "basics",
"route_api_model": "gpt-5",
- "golden_db": "basics-t-011-helper-function-golden",
- "llm_db": "basics-t-011-helper-function-gpt-5-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_011_helper_function\\rust\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_011_helper_function\\rust\\server\\gpt-5\\llm",
+ "golden_db": "basics-t-010-connect-golden",
+ "llm_db": "basics-t-010-connect-gpt-5-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_010_connect\\rust\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_010_connect\\rust\\server\\gpt-5\\llm",
"scorer_details": {
"publish_error": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 67 packages to latest compatible versions\n Compiling proc-macro2 v1.0.101\n Compiling quote v1.0.41\n Compiling unicode-ident v1.0.20\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling cfg-if v1.0.4\n Compiling serde_core v1.0.228\n Compiling either v1.15.0\n Compiling serde v1.0.228\n Compiling zerocopy v0.8.27\n Compiling shlex v1.3.0\n Compiling find-msvc-tools v0.1.4\n Compiling nohash-hasher v0.2.0\n Compiling thiserror v1.0.69\n Compiling bitflags v2.10.0\n Compiling anyhow v1.0.100\n Compiling convert_case v0.4.0\n Compiling keccak v0.1.5\n Compiling heck v0.4.1\n Compiling humantime v2.3.0\n Compiling arrayvec v0.7.6\n Compiling heck v0.5.0\n Compiling arrayref v0.3.9\n Compiling bytemuck v1.24.0\n Compiling hex v0.4.3\n Compiling second-stack v0.3.5\n Compiling bytes v1.10.1\n Compiling spacetimedb-lib v1.6.0\n Compiling getrandom v0.2.16\n Compiling smallvec v1.15.1\n Compiling constant_time_eq v0.3.1\n Compiling itertools v0.12.1\n Compiling rand_core v0.6.4\n Compiling log v0.4.28\n Compiling scoped-tls v1.0.1\n Compiling cc v1.2.41\n Compiling generic-array v0.14.9\n Compiling num-traits v0.2.19\n Compiling spacetimedb-primitives v1.6.0\n Compiling syn v2.0.107\n Compiling blake3 v1.8.2\n Compiling spacetimedb-bindings-sys v1.6.0\n Compiling crypto-common v0.1.6\n Compiling block-buffer v0.10.4\n Compiling ppv-lite86 v0.2.21\n Compiling digest v0.10.7\n Compiling ethnum v1.5.2\n Compiling rand_chacha v0.3.1\n Compiling approx v0.3.2\n Compiling chrono v0.4.42\n Compiling decorum v0.3.1\n Compiling rand v0.8.5\n Compiling sha3 v0.10.8\n Compiling thiserror-impl v1.0.69\n Compiling spacetimedb-bindings-macro v1.6.0\n Compiling enum-as-inner v0.6.1\n Compiling derive_more v0.99.20\n Compiling spacetimedb-sats v1.6.0\n Compiling spacetimedb v1.6.0\n Compiling spacetime-module v0.1.0 (E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_011_helper_function\\rust\\server\\gpt-5\\llm)\nwarning: unused imports: `reducer` and `table`\n --> src\\lib.rs:2:19\n |\n2 | use spacetimedb::{reducer, table, ReducerContext, Table};\n | ^^^^^^^ ^^^^^\n |\n = note: `#[warn(unused_imports)]` on by default\n\nerror[E0107]: struct takes 0 generic arguments but 2 generic arguments were supplied\n --> src\\lib.rs:4:1\n |\n4 | #[spacetimedb::table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected 0 generic arguments\n |\nnote: struct defined here, with 0 generic parameters\n --> src\\lib.rs:5:12\n |\n5 | pub struct Result {\n | ^^^^^^\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `deserialize` has an incompatible type for trait\n --> src\\lib.rs:4:1\n |\n4 | #[spacetimedb::table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `Result`, found `Result`\n |\n = note: expected signature `fn(_) -> std::result::Result>::Error>`\n found signature `fn(_) -> Result`\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `visit_seq_product` has an incompatible type for trait\n --> src\\lib.rs:4:1\n |\n4 | #[spacetimedb::table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `Result`, found `Result`\n |\n = note: expected signature `fn(__ProductVisitor, _) -> std::result::Result>::Error>`\n found signature `fn(__ProductVisitor, _) -> Result`\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `visit_named_product` has an incompatible type for trait\n --> src\\lib.rs:4:1\n |\n4 | #[spacetimedb::table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `Result`, found `Result`\n |\n = note: expected signature `fn(__ProductVisitor, _) -> std::result::Result>::Error>`\n found signature `fn(__ProductVisitor, _) -> Result`\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `visit` has an incompatible type for trait\n --> src\\lib.rs:4:1\n |\n4 | #[spacetimedb::table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `Result<__ProductFieldIdent, __E>`, found `Result`\n |\n = note: expected signature `fn(__ProductVisitor, &_) -> std::result::Result<__ProductFieldIdent, __E>`\n found signature `fn(__ProductVisitor, &_) -> Result`\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `serialize` has an incompatible type for trait\n --> src\\lib.rs:4:1\n |\n4 | #[spacetimedb::table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `Result<::Ok, ...>`, found `Result`\n |\n = note: expected signature `fn(&Result, _) -> std::result::Result<::Ok, ::Error>`\n found signature `fn(&Result, _) -> Result`\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0308]: mismatched types\n --> src\\lib.rs:4:1\n |\n 4 | #[spacetimedb::table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n | |\n | expected `Result`, found `Result`\n | expected `Result` because of return type\n |\n = note: `Result` and `Result` have similar names, but are actually distinct types\nnote: `Result` is defined in crate `core`\n --> C:\\Users\\bradl\\.rustup\\toolchains\\1.90.0-x86_64-pc-windows-msvc\\lib/rustlib/src/rust\\library/core/src/result.rs:548:1\n |\n548 | pub enum Result {\n | ^^^^^^^^^^^^^^^^^^^^^\nnote: `Result` is defined in the current crate\n --> src\\lib.rs:5:1\n |\n 5 | pub struct Result {\n | ^^^^^^^^^^^^^^^^^\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\nhelp: consider using `Result::expect` to unwrap the `std::result::Result>::Error>` value, panicking if the value is a `Result::Err`\n |\n 4 | #[spacetimedb::table(name = results)].expect(\"REASON\")\n | +++++++++++++++++\n\nerror[E0277]: the `?` operator can only be used in a method that returns `Result` or `Option` (or another type that implements `FromResidual`)\n --> src\\lib.rs:4:37\n |\n4 | #[spacetimedb::table(name = results)]\n | ------------------------------------^\n | | |\n | | cannot use the `?` operator in a method that returns `Result`\n | this function should return `Result` or `Option` to accept `?`\n |\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` which comes from the expansion of the attribute macro `spacetimedb::table` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0308]: mismatched types\n --> src\\lib.rs:4:1\n |\n 4 | #[spacetimedb::table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n | |\n | expected `Result`, found `Result`\n | expected `Result` because of return type\n |\n = note: `std::result::Result` and `Result` have similar names, but are actually distinct types\nnote: `std::result::Result` is defined in crate `core`\n --> C:\\Users\\bradl\\.rustup\\toolchains\\1.90.0-x86_64-pc-windows-msvc\\lib/rustlib/src/rust\\library/core/src/result.rs:548:1\n |\n548 | pub enum Result {\n | ^^^^^^^^^^^^^^^^^^^^^\nnote: `Result` is defined in the current crate\n --> src\\lib.rs:5:1\n |\n 5 | pub struct Result {\n | ^^^^^^^^^^^^^^^^^\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0308]: mismatched types\n --> src\\lib.rs:4:1\n |\n 4 | #[spacetimedb::table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n | |\n | expected `Result`, found `Result<_, _>`\n | expected `Result` because of return type\n |\n = note: `std::result::Result<_, _>` and `Result` have similar names, but are actually distinct types\nnote: `std::result::Result<_, _>` is defined in crate `core`\n --> C:\\Users\\bradl\\.rustup\\toolchains\\1.90.0-x86_64-pc-windows-msvc\\lib/rustlib/src/rust\\library/core/src/result.rs:548:1\n |\n548 | pub enum Result {\n | ^^^^^^^^^^^^^^^^^^^^^\nnote: `Result` is defined in the current crate\n --> src\\lib.rs:5:1\n |\n 5 | pub struct Result {\n | ^^^^^^^^^^^^^^^^^\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0308]: mismatched types\n --> src\\lib.rs:4:1\n |\n 4 | #[spacetimedb::table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n | |\n | expected `Result`, found `Result<__ProductFieldIdent, _>`\n | expected `Result` because of return type\n |\n = note: `Result<__ProductFieldIdent, _>` and `Result` have similar names, but are actually distinct types\nnote: `Result<__ProductFieldIdent, _>` is defined in crate `core`\n --> C:\\Users\\bradl\\.rustup\\toolchains\\1.90.0-x86_64-pc-windows-msvc\\lib/rustlib/src/rust\\library/core/src/result.rs:548:1\n |\n548 | pub enum Result {\n | ^^^^^^^^^^^^^^^^^^^^^\nnote: `Result` is defined in the current crate\n --> src\\lib.rs:5:1\n |\n 5 | pub struct Result {\n | ^^^^^^^^^^^^^^^^^\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0308]: mismatched types\n --> src\\lib.rs:4:1\n |\n 4 | #[spacetimedb::table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n | |\n | expected `Result`, found `Result<::Ok, ...>`\n | expected `Result` because of return type\n |\n = note: `Result<::Ok, ...>` and `Result` have similar names, but are actually distinct types\nnote: `Result<::Ok, ...>` is defined in crate `core`\n --> C:\\Users\\bradl\\.rustup\\toolchains\\1.90.0-x86_64-pc-windows-msvc\\lib/rustlib/src/rust\\library/core/src/result.rs:548:1\n |\n548 | pub enum Result {\n | ^^^^^^^^^^^^^^^^^^^^^\nnote: `Result` is defined in the current crate\n --> src\\lib.rs:5:1\n |\n 5 | pub struct Result {\n | ^^^^^^^^^^^^^^^^^\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nSome errors have detailed explanations: E0053, E0107, E0277, E0308.\nFor more information about an error, try `rustc --explain E0053`.\nwarning: `spacetime-module` (lib) generated 1 warning\nerror: could not compile `spacetime-module` (lib) due to 14 previous errors; 1 warning emitted\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n",
+ "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Blocking waiting for file lock on package cache\n Locking 67 packages to latest compatible versions\n Blocking waiting for file lock on package cache\n Blocking waiting for file lock on package cache\n Blocking waiting for file lock on package cache\n Compiling proc-macro2 v1.0.101\n Compiling unicode-ident v1.0.20\n Compiling quote v1.0.41\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling cfg-if v1.0.4\n Compiling serde_core v1.0.228\n Compiling either v1.15.0\n Compiling zerocopy v0.8.27\n Compiling find-msvc-tools v0.1.4\n Compiling serde v1.0.228\n Compiling shlex v1.3.0\n Compiling bitflags v2.10.0\n Compiling anyhow v1.0.100\n Compiling thiserror v1.0.69\n Compiling nohash-hasher v0.2.0\n Compiling heck v0.4.1\n Compiling humantime v2.3.0\n Compiling heck v0.5.0\n Compiling keccak v0.1.5\n Compiling arrayvec v0.7.6\n Compiling convert_case v0.4.0\n Compiling smallvec v1.15.1\n Compiling second-stack v0.3.5\n Compiling hex v0.4.3\n Compiling arrayref v0.3.9\n Compiling bytemuck v1.24.0\n Compiling spacetimedb-lib v1.6.0\n Compiling getrandom v0.2.16\n Compiling itertools v0.12.1\n Compiling cc v1.2.41\n Compiling bytes v1.10.1\n Compiling constant_time_eq v0.3.1\n Compiling rand_core v0.6.4\n Compiling scoped-tls v1.0.1\n Compiling log v0.4.28\n Compiling num-traits v0.2.19\n Compiling generic-array v0.14.9\n Compiling spacetimedb-primitives v1.6.0\n Compiling spacetimedb-bindings-sys v1.6.0\n Compiling blake3 v1.8.2\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling syn v2.0.107\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.6\n Compiling digest v0.10.7\n Compiling rand v0.8.5\n Compiling sha3 v0.10.8\n Compiling ethnum v1.5.2\n Compiling approx v0.3.2\n Compiling chrono v0.4.42\n Compiling decorum v0.3.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-bindings-macro v1.6.0\n Compiling enum-as-inner v0.6.1\n Compiling spacetimedb-sats v1.6.0\n Compiling spacetimedb v1.6.0\n Compiling spacetime-module v0.1.0 (E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_010_connect\\rust\\server\\gpt-5\\llm)\nerror[E0599]: no method named `insert` found for reference `&events__TableHandle` in the current scope\n --> src\\lib.rs:14:21\n |\n14 | ctx.db.events().insert(Event {\n | ----------------^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `try_insert` with a similar name\n |\n14 | ctx.db.events().try_insert(Event {\n | ++++\n\nerror[E0599]: no method named `insert` found for reference `&events__TableHandle` in the current scope\n --> src\\lib.rs:22:21\n |\n22 | ctx.db.events().insert(Event {\n | ----------------^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `try_insert` with a similar name\n |\n22 | ctx.db.events().try_insert(Event {\n | ++++\n\nFor more information about this error, try `rustc --explain E0599`.\nerror: could not compile `spacetime-module` (lib) due to 2 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n",
"phase": "build_or_publish"
}
}
},
"vendor": "openai",
- "started_at": "2025-10-21T22:01:05.213008300Z",
- "finished_at": "2025-10-21T22:02:46.665697200Z"
+ "started_at": "2025-10-21T22:01:05.157567300Z",
+ "finished_at": "2025-10-21T22:03:17.764056200Z"
},
- "t_015_product_type_columns": {
+ "t_011_helper_function": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_015_product_type_columns",
+ "task": "t_011_helper_function",
"lang": "rust",
"golden_published": true,
"model_name": "GPT-5",
"total_tests": 3,
"passed_tests": 0,
- "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = profiles, public)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) -> Result<(), String> {\n ctx.db.profiles().insert(Profile {\n id: 1,\n home: Address {\n street: \"1 Main\".to_string(),\n zip: 11111,\n },\n work: Address {\n street: \"2 Broad\".to_string(),\n zip: 22222,\n },\n pos: Position { x: 7, y: 9 },\n });\n Ok(())\n}",
- "category": "schema",
+ "llm_output": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[spacetimedb::table(name = results)]\npub struct Result {\n #[primary_key]\n pub id: i32,\n pub sum: i32,\n}\n\nfn add(a: i32, b: i32) -> i32 {\n a + b\n}\n\n#[spacetimedb::reducer]\npub fn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) {\n let sum = add(a, b);\n ctx.db.results().insert(Result { id, sum });\n}",
+ "category": "basics",
"route_api_model": "gpt-5",
- "golden_db": "schema-t-015-product-type-columns-golden",
- "llm_db": "schema-t-015-product-type-columns-gpt-5-llm",
- "work_dir_golden": "target\\llm-runs\\schema\\t_015_product_type_columns\\rust\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\schema\\t_015_product_type_columns\\rust\\server\\gpt-5\\llm",
+ "golden_db": "basics-t-011-helper-function-golden",
+ "llm_db": "basics-t-011-helper-function-gpt-5-llm",
+ "work_dir_golden": "target\\llm-runs\\basics\\t_011_helper_function\\rust\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\basics\\t_011_helper_function\\rust\\server\\gpt-5\\llm",
"scorer_details": {
"publish_error": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 67 packages to latest compatible versions\n Compiling proc-macro2 v1.0.101\n Compiling unicode-ident v1.0.20\n Compiling quote v1.0.41\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling cfg-if v1.0.4\n Compiling either v1.15.0\n Compiling zerocopy v0.8.27\n Compiling find-msvc-tools v0.1.4\n Compiling serde v1.0.228\n Compiling shlex v1.3.0\n Compiling anyhow v1.0.100\n Compiling bitflags v2.10.0\n Compiling nohash-hasher v0.2.0\n Compiling thiserror v1.0.69\n Compiling heck v0.4.1\n Compiling arrayvec v0.7.6\n Compiling keccak v0.1.5\n Compiling convert_case v0.4.0\n Compiling heck v0.5.0\n Compiling humantime v2.3.0\n Compiling smallvec v1.15.1\n Compiling bytes v1.10.1\n Compiling spacetimedb-lib v1.6.0\n Compiling second-stack v0.3.5\n Compiling hex v0.4.3\n Compiling bytemuck v1.24.0\n Compiling itertools v0.12.1\n Compiling getrandom v0.2.16\n Compiling constant_time_eq v0.3.1\n Compiling rand_core v0.6.4\n Compiling generic-array v0.14.9\n Compiling arrayref v0.3.9\n Compiling scoped-tls v1.0.1\n Compiling log v0.4.28\n Compiling cc v1.2.41\n Compiling num-traits v0.2.19\n Compiling spacetimedb-primitives v1.6.0\n Compiling blake3 v1.8.2\n Compiling spacetimedb-bindings-sys v1.6.0\n Compiling crypto-common v0.1.6\n Compiling block-buffer v0.10.4\n Compiling syn v2.0.107\n Compiling ppv-lite86 v0.2.21\n Compiling digest v0.10.7\n Compiling approx v0.3.2\n Compiling chrono v0.4.42\n Compiling sha3 v0.10.8\n Compiling rand_chacha v0.3.1\n Compiling decorum v0.3.1\n Compiling ethnum v1.5.2\n Compiling rand v0.8.5\n Compiling thiserror-impl v1.0.69\n Compiling spacetimedb-bindings-macro v1.6.0\n Compiling enum-as-inner v0.6.1\n Compiling derive_more v0.99.20\n Compiling spacetimedb-sats v1.6.0\n Compiling spacetimedb v1.6.0\n Compiling spacetime-module v0.1.0 (E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\rust\\server\\gpt-5\\llm)\nerror[E0599]: no method named `insert` found for reference `&profiles__TableHandle` in the current scope\n --> src\\lib.rs:27:23\n |\n27 | ctx.db.profiles().insert(Profile {\n | ------------------^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `try_insert` with a similar name\n |\n27 | ctx.db.profiles().try_insert(Profile {\n | ++++\n\nFor more information about this error, try `rustc --explain E0599`.\nerror: could not compile `spacetime-module` (lib) due to 1 previous error\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n",
+ "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 67 packages to latest compatible versions\n Compiling proc-macro2 v1.0.101\n Compiling quote v1.0.41\n Compiling unicode-ident v1.0.20\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling cfg-if v1.0.4\n Compiling serde_core v1.0.228\n Compiling either v1.15.0\n Compiling serde v1.0.228\n Compiling zerocopy v0.8.27\n Compiling shlex v1.3.0\n Compiling find-msvc-tools v0.1.4\n Compiling nohash-hasher v0.2.0\n Compiling thiserror v1.0.69\n Compiling bitflags v2.10.0\n Compiling anyhow v1.0.100\n Compiling convert_case v0.4.0\n Compiling keccak v0.1.5\n Compiling heck v0.4.1\n Compiling humantime v2.3.0\n Compiling arrayvec v0.7.6\n Compiling heck v0.5.0\n Compiling arrayref v0.3.9\n Compiling bytemuck v1.24.0\n Compiling hex v0.4.3\n Compiling second-stack v0.3.5\n Compiling bytes v1.10.1\n Compiling spacetimedb-lib v1.6.0\n Compiling getrandom v0.2.16\n Compiling smallvec v1.15.1\n Compiling constant_time_eq v0.3.1\n Compiling itertools v0.12.1\n Compiling rand_core v0.6.4\n Compiling log v0.4.28\n Compiling scoped-tls v1.0.1\n Compiling cc v1.2.41\n Compiling generic-array v0.14.9\n Compiling num-traits v0.2.19\n Compiling spacetimedb-primitives v1.6.0\n Compiling syn v2.0.107\n Compiling blake3 v1.8.2\n Compiling spacetimedb-bindings-sys v1.6.0\n Compiling crypto-common v0.1.6\n Compiling block-buffer v0.10.4\n Compiling ppv-lite86 v0.2.21\n Compiling digest v0.10.7\n Compiling ethnum v1.5.2\n Compiling rand_chacha v0.3.1\n Compiling approx v0.3.2\n Compiling chrono v0.4.42\n Compiling decorum v0.3.1\n Compiling rand v0.8.5\n Compiling sha3 v0.10.8\n Compiling thiserror-impl v1.0.69\n Compiling spacetimedb-bindings-macro v1.6.0\n Compiling enum-as-inner v0.6.1\n Compiling derive_more v0.99.20\n Compiling spacetimedb-sats v1.6.0\n Compiling spacetimedb v1.6.0\n Compiling spacetime-module v0.1.0 (E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_011_helper_function\\rust\\server\\gpt-5\\llm)\nwarning: unused imports: `reducer` and `table`\n --> src\\lib.rs:2:19\n |\n2 | use spacetimedb::{reducer, table, ReducerContext, Table};\n | ^^^^^^^ ^^^^^\n |\n = note: `#[warn(unused_imports)]` on by default\n\nerror[E0107]: struct takes 0 generic arguments but 2 generic arguments were supplied\n --> src\\lib.rs:4:1\n |\n4 | #[spacetimedb::table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected 0 generic arguments\n |\nnote: struct defined here, with 0 generic parameters\n --> src\\lib.rs:5:12\n |\n5 | pub struct Result {\n | ^^^^^^\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `deserialize` has an incompatible type for trait\n --> src\\lib.rs:4:1\n |\n4 | #[spacetimedb::table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `Result`, found `Result`\n |\n = note: expected signature `fn(_) -> std::result::Result>::Error>`\n found signature `fn(_) -> Result`\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `visit_seq_product` has an incompatible type for trait\n --> src\\lib.rs:4:1\n |\n4 | #[spacetimedb::table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `Result`, found `Result`\n |\n = note: expected signature `fn(__ProductVisitor, _) -> std::result::Result>::Error>`\n found signature `fn(__ProductVisitor, _) -> Result`\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `visit_named_product` has an incompatible type for trait\n --> src\\lib.rs:4:1\n |\n4 | #[spacetimedb::table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `Result`, found `Result`\n |\n = note: expected signature `fn(__ProductVisitor, _) -> std::result::Result>::Error>`\n found signature `fn(__ProductVisitor, _) -> Result`\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `visit` has an incompatible type for trait\n --> src\\lib.rs:4:1\n |\n4 | #[spacetimedb::table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `Result<__ProductFieldIdent, __E>`, found `Result`\n |\n = note: expected signature `fn(__ProductVisitor, &_) -> std::result::Result<__ProductFieldIdent, __E>`\n found signature `fn(__ProductVisitor, &_) -> Result`\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `serialize` has an incompatible type for trait\n --> src\\lib.rs:4:1\n |\n4 | #[spacetimedb::table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `Result<::Ok, ...>`, found `Result`\n |\n = note: expected signature `fn(&Result, _) -> std::result::Result<::Ok, ::Error>`\n found signature `fn(&Result, _) -> Result`\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0308]: mismatched types\n --> src\\lib.rs:4:1\n |\n 4 | #[spacetimedb::table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n | |\n | expected `Result`, found `Result`\n | expected `Result` because of return type\n |\n = note: `Result` and `Result` have similar names, but are actually distinct types\nnote: `Result` is defined in crate `core`\n --> C:\\Users\\bradl\\.rustup\\toolchains\\1.90.0-x86_64-pc-windows-msvc\\lib/rustlib/src/rust\\library/core/src/result.rs:548:1\n |\n548 | pub enum Result {\n | ^^^^^^^^^^^^^^^^^^^^^\nnote: `Result` is defined in the current crate\n --> src\\lib.rs:5:1\n |\n 5 | pub struct Result {\n | ^^^^^^^^^^^^^^^^^\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\nhelp: consider using `Result::expect` to unwrap the `std::result::Result>::Error>` value, panicking if the value is a `Result::Err`\n |\n 4 | #[spacetimedb::table(name = results)].expect(\"REASON\")\n | +++++++++++++++++\n\nerror[E0277]: the `?` operator can only be used in a method that returns `Result` or `Option` (or another type that implements `FromResidual`)\n --> src\\lib.rs:4:37\n |\n4 | #[spacetimedb::table(name = results)]\n | ------------------------------------^\n | | |\n | | cannot use the `?` operator in a method that returns `Result`\n | this function should return `Result` or `Option` to accept `?`\n |\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` which comes from the expansion of the attribute macro `spacetimedb::table` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0308]: mismatched types\n --> src\\lib.rs:4:1\n |\n 4 | #[spacetimedb::table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n | |\n | expected `Result`, found `Result`\n | expected `Result` because of return type\n |\n = note: `std::result::Result` and `Result` have similar names, but are actually distinct types\nnote: `std::result::Result` is defined in crate `core`\n --> C:\\Users\\bradl\\.rustup\\toolchains\\1.90.0-x86_64-pc-windows-msvc\\lib/rustlib/src/rust\\library/core/src/result.rs:548:1\n |\n548 | pub enum Result {\n | ^^^^^^^^^^^^^^^^^^^^^\nnote: `Result` is defined in the current crate\n --> src\\lib.rs:5:1\n |\n 5 | pub struct Result {\n | ^^^^^^^^^^^^^^^^^\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0308]: mismatched types\n --> src\\lib.rs:4:1\n |\n 4 | #[spacetimedb::table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n | |\n | expected `Result`, found `Result<_, _>`\n | expected `Result` because of return type\n |\n = note: `std::result::Result<_, _>` and `Result` have similar names, but are actually distinct types\nnote: `std::result::Result<_, _>` is defined in crate `core`\n --> C:\\Users\\bradl\\.rustup\\toolchains\\1.90.0-x86_64-pc-windows-msvc\\lib/rustlib/src/rust\\library/core/src/result.rs:548:1\n |\n548 | pub enum Result {\n | ^^^^^^^^^^^^^^^^^^^^^\nnote: `Result` is defined in the current crate\n --> src\\lib.rs:5:1\n |\n 5 | pub struct Result {\n | ^^^^^^^^^^^^^^^^^\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0308]: mismatched types\n --> src\\lib.rs:4:1\n |\n 4 | #[spacetimedb::table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n | |\n | expected `Result`, found `Result<__ProductFieldIdent, _>`\n | expected `Result` because of return type\n |\n = note: `Result<__ProductFieldIdent, _>` and `Result` have similar names, but are actually distinct types\nnote: `Result<__ProductFieldIdent, _>` is defined in crate `core`\n --> C:\\Users\\bradl\\.rustup\\toolchains\\1.90.0-x86_64-pc-windows-msvc\\lib/rustlib/src/rust\\library/core/src/result.rs:548:1\n |\n548 | pub enum Result {\n | ^^^^^^^^^^^^^^^^^^^^^\nnote: `Result` is defined in the current crate\n --> src\\lib.rs:5:1\n |\n 5 | pub struct Result {\n | ^^^^^^^^^^^^^^^^^\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0308]: mismatched types\n --> src\\lib.rs:4:1\n |\n 4 | #[spacetimedb::table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n | |\n | expected `Result`, found `Result<::Ok, ...>`\n | expected `Result` because of return type\n |\n = note: `Result<::Ok, ...>` and `Result` have similar names, but are actually distinct types\nnote: `Result<::Ok, ...>` is defined in crate `core`\n --> C:\\Users\\bradl\\.rustup\\toolchains\\1.90.0-x86_64-pc-windows-msvc\\lib/rustlib/src/rust\\library/core/src/result.rs:548:1\n |\n548 | pub enum Result {\n | ^^^^^^^^^^^^^^^^^^^^^\nnote: `Result` is defined in the current crate\n --> src\\lib.rs:5:1\n |\n 5 | pub struct Result {\n | ^^^^^^^^^^^^^^^^^\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nSome errors have detailed explanations: E0053, E0107, E0277, E0308.\nFor more information about an error, try `rustc --explain E0053`.\nwarning: `spacetime-module` (lib) generated 1 warning\nerror: could not compile `spacetime-module` (lib) due to 14 previous errors; 1 warning emitted\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n",
"phase": "build_or_publish"
}
}
},
"vendor": "openai",
- "started_at": "2025-10-21T22:01:05.430917300Z",
- "finished_at": "2025-10-21T22:02:51.908916800Z"
+ "started_at": "2025-10-21T22:01:05.213008300Z",
+ "finished_at": "2025-10-21T22:02:46.665697200Z"
},
- "t_002_scheduled_table": {
+ "t_012_spacetime_product_type": {
"hash": "e14a4c9b74a1bcb23078c80458a07ab1250d718b8723ed80b471c193028b701f",
- "task": "t_002_scheduled_table",
+ "task": "t_012_spacetime_product_type",
"lang": "rust",
"golden_published": true,
"model_name": "GPT-5",
- "total_tests": 1,
+ "total_tests": 3,
"passed_tests": 0,
- "llm_output": "use std::time::Duration;\nuse spacetimedb::{ReducerContext, ScheduleAt};\n\n#[spacetimedb::table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n pub scheduled_id: u64,\n pub scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n let interval = Duration::from_micros(50_000);\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::Interval(interval.into()),\n });\n}",
- "category": "basics",
+ "llm_output": "use spacetimedb::{reducer, table, ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Debug, Clone)]\npub struct Score {\n pub left: i32,\n pub right: i32,\n}\n\n#[table(name = results)]\npub struct Result {\n #[primary_key]\n pub id: i32,\n pub value: Score,\n}\n\n#[reducer]\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db.results().insert(Result {\n id,\n value: Score { left, right },\n });\n}",
+ "category": "schema",
"route_api_model": "gpt-5",
- "golden_db": "basics-t-002-scheduled-table-golden",
- "llm_db": "basics-t-002-scheduled-table-gpt-5-llm",
- "work_dir_golden": "target\\llm-runs\\basics\\t_002_scheduled_table\\rust\\server\\golden",
- "work_dir_llm": "target\\llm-runs\\basics\\t_002_scheduled_table\\rust\\server\\gpt-5\\llm",
+ "golden_db": "schema-t-012-spacetime-product-type-golden",
+ "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm",
+ "work_dir_golden": "target\\llm-runs\\schema\\t_012_spacetime_product_type\\rust\\server\\golden",
+ "work_dir_llm": "target\\llm-runs\\schema\\t_012_spacetime_product_type\\rust\\server\\gpt-5\\llm",
"scorer_details": {
"publish_error": {
"pass": false,
"partial": 0.0,
"notes": {
- "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Blocking waiting for file lock on package cache\n Updating crates.io index\n Blocking waiting for file lock on package cache\n Locking 67 packages to latest compatible versions\n Blocking waiting for file lock on package cache\n Blocking waiting for file lock on package cache\n Compiling proc-macro2 v1.0.101\n Compiling unicode-ident v1.0.20\n Compiling quote v1.0.41\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling cfg-if v1.0.4\n Compiling either v1.15.0\n Compiling serde v1.0.228\n Compiling zerocopy v0.8.27\n Compiling shlex v1.3.0\n Compiling find-msvc-tools v0.1.4\n Compiling nohash-hasher v0.2.0\n Compiling thiserror v1.0.69\n Compiling bitflags v2.10.0\n Compiling anyhow v1.0.100\n Compiling convert_case v0.4.0\n Compiling arrayvec v0.7.6\n Compiling heck v0.4.1\n Compiling keccak v0.1.5\n Compiling humantime v2.3.0\n Compiling heck v0.5.0\n Compiling second-stack v0.3.5\n Compiling constant_time_eq v0.3.1\n Compiling bytemuck v1.24.0\n Compiling arrayref v0.3.9\n Compiling spacetimedb-lib v1.6.0\n Compiling bytes v1.10.1\n Compiling smallvec v1.15.1\n Compiling itertools v0.12.1\n Compiling generic-array v0.14.9\n Compiling cc v1.2.41\n Compiling spacetimedb-primitives v1.6.0\n Compiling getrandom v0.2.16\n Compiling hex v0.4.3\n Compiling log v0.4.28\n Compiling scoped-tls v1.0.1\n Compiling num-traits v0.2.19\n Compiling rand_core v0.6.4\n Compiling blake3 v1.8.2\n Compiling spacetimedb-bindings-sys v1.6.0\n Compiling ppv-lite86 v0.2.21\n Compiling ethnum v1.5.2\n Compiling rand_chacha v0.3.1\n Compiling crypto-common v0.1.6\n Compiling block-buffer v0.10.4\n Compiling syn v2.0.107\n Compiling digest v0.10.7\n Compiling rand v0.8.5\n Compiling sha3 v0.10.8\n Compiling approx v0.3.2\n Compiling chrono v0.4.42\n Compiling decorum v0.3.1\n Compiling thiserror-impl v1.0.69\n Compiling spacetimedb-bindings-macro v1.6.0\n Compiling enum-as-inner v0.6.1\n Compiling derive_more v0.99.20\n Compiling spacetimedb-sats v1.6.0\n Compiling spacetimedb v1.6.0\n Compiling spacetime-module v0.1.0 (E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_002_scheduled_table\\rust\\server\\gpt-5\\llm)\nerror[E0599]: no method named `insert` found for reference `&tick_timer__TableHandle` in the current scope\n --> src\\lib.rs:20:25\n |\n20 | ctx.db.tick_timer().insert(TickTimer {\n | --------------------^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `try_insert` with a similar name\n |\n20 | ctx.db.tick_timer().try_insert(TickTimer {\n | ++++\n\nFor more information about this error, try `rustc --explain E0599`.\nerror: could not compile `spacetime-module` (lib) due to 1 previous error\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n",
+ "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 67 packages to latest compatible versions\n Compiling proc-macro2 v1.0.101\n Compiling unicode-ident v1.0.20\n Compiling quote v1.0.41\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling cfg-if v1.0.4\n Compiling serde_core v1.0.228\n Compiling zerocopy v0.8.27\n Compiling either v1.15.0\n Compiling serde v1.0.228\n Compiling shlex v1.3.0\n Compiling find-msvc-tools v0.1.4\n Compiling anyhow v1.0.100\n Compiling nohash-hasher v0.2.0\n Compiling bitflags v2.10.0\n Compiling thiserror v1.0.69\n Compiling heck v0.4.1\n Compiling humantime v2.3.0\n Compiling arrayvec v0.7.6\n Compiling heck v0.5.0\n Compiling convert_case v0.4.0\n Compiling keccak v0.1.5\n Compiling spacetimedb-lib v1.6.0\n Compiling bytemuck v1.24.0\n Compiling bytes v1.10.1\n Compiling second-stack v0.3.5\n Compiling hex v0.4.3\n Compiling smallvec v1.15.1\n Compiling getrandom v0.2.16\n Compiling arrayref v0.3.9\n Compiling constant_time_eq v0.3.1\n Compiling log v0.4.28\n Compiling cc v1.2.41\n Compiling itertools v0.12.1\n Compiling rand_core v0.6.4\n Compiling scoped-tls v1.0.1\n Compiling generic-array v0.14.9\n Compiling num-traits v0.2.19\n Compiling blake3 v1.8.2\n Compiling spacetimedb-primitives v1.6.0\n Compiling spacetimedb-bindings-sys v1.6.0\n Compiling syn v2.0.107\n Compiling ppv-lite86 v0.2.21\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.6\n Compiling digest v0.10.7\n Compiling ethnum v1.5.2\n Compiling rand_chacha v0.3.1\n Compiling approx v0.3.2\n Compiling chrono v0.4.42\n Compiling decorum v0.3.1\n Compiling sha3 v0.10.8\n Compiling rand v0.8.5\n Compiling thiserror-impl v1.0.69\n Compiling spacetimedb-bindings-macro v1.6.0\n Compiling derive_more v0.99.20\n Compiling enum-as-inner v0.6.1\n Compiling spacetimedb-sats v1.6.0\n Compiling spacetimedb v1.6.0\n Compiling spacetime-module v0.1.0 (E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_012_spacetime_product_type\\rust\\server\\gpt-5\\llm)\nerror[E0107]: struct takes 0 generic arguments but 2 generic arguments were supplied\n --> src\\lib.rs:4:10\n |\n 4 | #[derive(SpacetimeType, Debug, Clone)]\n | ^^^^^^^^^^^^^ expected 0 generic arguments\n |\nnote: struct defined here, with 0 generic parameters\n --> src\\lib.rs:11:12\n |\n11 | pub struct Result {\n | ^^^^^^\n = note: this error originates in the derive macro `SpacetimeType` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `deserialize` has an incompatible type for trait\n --> src\\lib.rs:4:10\n |\n4 | #[derive(SpacetimeType, Debug, Clone)]\n | ^^^^^^^^^^^^^ expected `Result`, found `Result`\n |\n = note: expected signature `fn(_) -> std::result::Result>::Error>`\n found signature `fn(_) -> Result`\n = note: this error originates in the derive macro `SpacetimeType` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `visit_seq_product` has an incompatible type for trait\n --> src\\lib.rs:4:10\n |\n4 | #[derive(SpacetimeType, Debug, Clone)]\n | ^^^^^^^^^^^^^ expected `Result`, found `Result`\n |\n = note: expected signature `fn(_::__ProductVisitor, _) -> std::result::Result>::Error>`\n found signature `fn(_::__ProductVisitor, _) -> Result`\n = note: this error originates in the derive macro `SpacetimeType` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `visit_named_product` has an incompatible type for trait\n --> src\\lib.rs:4:10\n |\n4 | #[derive(SpacetimeType, Debug, Clone)]\n | ^^^^^^^^^^^^^ expected `Result`, found `Result`\n |\n = note: expected signature `fn(_::__ProductVisitor, _) -> std::result::Result>::Error>`\n found signature `fn(_::__ProductVisitor, _) -> Result`\n = note: this error originates in the derive macro `SpacetimeType` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `visit` has an incompatible type for trait\n --> src\\lib.rs:4:10\n |\n4 | #[derive(SpacetimeType, Debug, Clone)]\n | ^^^^^^^^^^^^^ expected `Result<__ProductFieldIdent, __E>`, found `Result`\n |\n = note: expected signature `fn(_::__ProductVisitor, &_) -> std::result::Result<_::__ProductFieldIdent, __E>`\n found signature `fn(_::__ProductVisitor, &_) -> Result`\n = note: this error originates in the derive macro `SpacetimeType` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `serialize` has an incompatible type for trait\n --> src\\lib.rs:4:10\n |\n4 | #[derive(SpacetimeType, Debug, Clone)]\n | ^^^^^^^^^^^^^ expected `Result<::Ok, ...>`, found `Result`\n |\n = note: expected signature `fn(&Score, _) -> std::result::Result<::Ok, ::Error>`\n found signature `fn(&Score, _) -> Result`\n = note: this error originates in the derive macro `SpacetimeType` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0107]: struct takes 0 generic arguments but 2 generic arguments were supplied\n --> src\\lib.rs:10:1\n |\n10 | #[table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^ expected 0 generic arguments\n |\nnote: struct defined here, with 0 generic parameters\n --> src\\lib.rs:11:12\n |\n11 | pub struct Result {\n | ^^^^^^\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `deserialize` has an incompatible type for trait\n --> src\\lib.rs:10:1\n |\n10 | #[table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^ expected `Result`, found `Result`\n |\n = note: expected signature `fn(_) -> std::result::Result>::Error>`\n found signature `fn(_) -> Result`\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `visit_seq_product` has an incompatible type for trait\n --> src\\lib.rs:10:1\n |\n10 | #[table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^ expected `Result`, found `Result`\n |\n = note: expected signature `fn(_::__ProductVisitor, _) -> std::result::Result>::Error>`\n found signature `fn(_::__ProductVisitor, _) -> Result`\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `visit_named_product` has an incompatible type for trait\n --> src\\lib.rs:10:1\n |\n10 | #[table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^ expected `Result`, found `Result`\n |\n = note: expected signature `fn(_::__ProductVisitor, _) -> std::result::Result>::Error>`\n found signature `fn(_::__ProductVisitor, _) -> Result`\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `visit` has an incompatible type for trait\n --> src\\lib.rs:10:1\n |\n10 | #[table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^ expected `Result<__ProductFieldIdent, __E>`, found `Result`\n |\n = note: expected signature `fn(_::__ProductVisitor, &_) -> std::result::Result<_::__ProductFieldIdent, __E>`\n found signature `fn(_::__ProductVisitor, &_) -> Result`\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `serialize` has an incompatible type for trait\n --> src\\lib.rs:10:1\n |\n10 | #[table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^ expected `Result<::Ok, ...>`, found `Result`\n |\n = note: expected signature `fn(&Result, _) -> std::result::Result<::Ok, ::Error>`\n found signature `fn(&Result, _) -> Result`\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0308]: mismatched types\n --> src\\lib.rs:4:10\n |\n 4 | #[derive(SpacetimeType, Debug, Clone)]\n | ^^^^^^^^^^^^^\n | |\n | expected `Result`, found `Result`\n | expected `Result` because of return type\n |\n = note: `Result` and `Result` have similar names, but are actually distinct types\nnote: `Result` is defined in crate `core`\n --> C:\\Users\\bradl\\.rustup\\toolchains\\1.90.0-x86_64-pc-windows-msvc\\lib/rustlib/src/rust\\library/core/src/result.rs:548:1\n |\n548 | pub enum Result {\n | ^^^^^^^^^^^^^^^^^^^^^\nnote: `Result` is defined in the current crate\n --> src\\lib.rs:11:1\n |\n 11 | pub struct Result {\n | ^^^^^^^^^^^^^^^^^\n = note: this error originates in the derive macro `SpacetimeType` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0277]: the `?` operator can only be used in a method that returns `Result` or `Option` (or another type that implements `FromResidual`)\n --> src\\lib.rs:4:22\n |\n4 | #[derive(SpacetimeType, Debug, Clone)]\n | ------------^\n | | |\n | | cannot use the `?` operator in a method that returns `Result`\n | this function should return `Result` or `Option` to accept `?`\n |\n = note: this error originates in the derive macro `SpacetimeType` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0308]: mismatched types\n --> src\\lib.rs:4:10\n |\n 4 | #[derive(SpacetimeType, Debug, Clone)]\n | ^^^^^^^^^^^^^\n | |\n | expected `Result`, found `Result`\n | expected `Result` because of return type\n |\n = note: `std::result::Result