feat(cli): fast and no more JIT openapi spec

.github/workflows/ci.yml

@@ -36,7 +36,16 @@ jobs:
           bun-version: 1.3.14
 
       - name: Install dependencies
-        run: bun install
+        run: bun ci
+
+      - name: Typecheck
+        run: bun run typecheck
 
       - name: Run tests
         run: bun test
+
+      - name: Fake-call every endpoint in every API snapshot
+        run: bun run conformance:all
+
+      - name: Smoke-test built CLI
+        run: bun bin/langfuse.mjs --help

.gitignore

@@ -1,7 +1,6 @@
 node_modules
 dist
 *.tgz
-bun.lock
 bun.lockb
 .env
 .env.local

bin/langfuse.mjs

@@ -1,3 +1,3 @@
-#!/usr/bin/env node
+#!/usr/bin/env bun
 import { run } from "../dist/cli.js";
-run(process.argv);
+await run(process.argv);

conformance/README.md

@@ -0,0 +1,113 @@
+# Langfuse CLI OpenAPI conformance suite
+
+Language-neutral, version-pinned acceptance tests for the native TypeScript CLI. The oracle does not import the runtime request builder.
+
+## What is tested
+
+The primary test fake-calls every operation in every cataloged spec through the real CLI. For each operation it:
+
+- generates one minimally valid invocation from the untouched OpenAPI source
+- starts a local mock HTTP server with a response generated from that operation
+- runs the CLI as a subprocess against the mock host
+- compares the received method, path, query, headers, authentication, and JSON body
+- compares the CLI's response status, body, and exit status
+
+The black-box oracle does not share request-building code with the CLI. `bun test` currently attempts all 678 operations across all eight v3 snapshots using the historical field-flag adapter. The 18 operations that require lossless JSON bodies remain an explicit compatibility baseline; any additional failure fails the test. The native `contract-v1` adapter passes all 678 operations through `--body-json`.
+
+Supporting unit tests verify immutable spec hashes, valid sampling, serialization, naming, adapters, and the capture runner itself.
+
+OpenAPI cannot describe database setup, generated IDs, cross-request bindings, cleanup, licenses, or feature flags. Those stateful live workflows require a small reviewed scenario overlay; they are not silently invented by this generator.
+
+## Pinned specs
+
+| Langfuse | Paths | Operations |
+|---|---:|---:|
+| 3.0.0 | 29 | 39 |
+| 3.50.0 | 44 | 68 |
+| 3.100.0 | 49 | 77 |
+| 3.150.0 | 55 | 86 |
+| 3.176.0 | 60 | 96 |
+| 3.200.0 | 61 | 98 |
+| 3.212.0 | 64 | 101 |
+| 3.216.0 | 69 | 113 |
+
+Each source file is downloaded by immutable commit and verified against the SHA-256 in `catalog.json`. Tests are network-free after sync.
+
+Langfuse 3.200.0, 3.212.0, and 3.216.0 use the JSON Schema `const` keyword while declaring OpenAPI 3.0.1. `swagger-parser` correctly reports those sources as invalid OAS 3.0 documents. The catalog records this as `oas3.0-const-keyword`; request sampling still preserves and tests the constraint with the independent JSON Schema validator.
+
+## Files
+
+```text
+catalog.json                 immutable Git refs, commits, hashes, known source issues
+policy.json                  implementation adapters; not API truth
+specs/<version>/openapi.yml  untouched upstream snapshots
+src/                         compiler, serializers, adapters, capture runner
+tests/                       compiler, validator, and runner tests
+```
+
+## Commands
+
+```sh
+# Generator, schema, serializer, capture, and compatibility tests
+bun test
+
+# Build and fake-call every endpoint through the lossless native CLI
+bun run conformance:all
+
+# Re-download pinned bytes and verify their hashes
+bun run conformance:sync
+```
+
+## CI gate
+
+GitHub Actions runs both commands for every pull request, merge queue entry, and push to `main`:
+
+```sh
+bun test
+bun run conformance:all
+```
+
+The required check name is **Test and verify OpenAPI conformance**. The interactive release script repeats the checks before building or publishing.
+
+## Run a focused current-CLI check
+
+```sh
+bun run conformance:run -- \
+  --version 3.212.0 \
+  --adapter specli-v0 \
+  --current-cli
+```
+
+The adapter name is retained because it describes the old field-flag grammar. The runner builds the native current source and compiles the selected untouched spec into a temporary runtime contract.
+
+Useful filters:
+
+```sh
+--operation prompts_create
+--max 20
+--fail-fast
+```
+
+Failures identify current limitations inline: raw union bodies, complex body flags, response exit codes, naming mismatches, or missing version selection.
+
+## Run the lossless native contract
+
+The native adapter uses lossless JSON body input via `--body-json`:
+
+```sh
+bun run conformance:run -- \
+  --version 3.216.0 \
+  --adapter contract-v1 \
+  -- bun bin/langfuse.mjs --api-version 3.216.0
+```
+
+The command after the second `--` is treated as an external black-box executable.
+
+## Add a version
+
+1. Resolve the release tag to its immutable commit.
+2. Add the tag, commit, and SHA-256 to `catalog.json`.
+3. Run `bun run conformance:sync -- --version <version>`.
+4. Run `bun test` and `bun run conformance:all`.
+
+Never point a committed catalog entry at mutable `main` or `latest`.

conformance/src/all.ts

@@ -0,0 +1,43 @@
+import { loadCatalog } from "./catalog";
+import { generateCorpus } from "./generator";
+import { runConformance } from "./runner";
+
+const build = Bun.spawn(["bun", "run", "build"], {
+  cwd: import.meta.dir + "/../..",
+  stdout: "inherit",
+  stderr: "inherit",
+});
+if ((await build.exited) !== 0) process.exit(1);
+
+const catalog = await loadCatalog();
+let passed = 0;
+let total = 0;
+let failed = false;
+
+for (const entry of catalog.versions) {
+  const corpus = await generateCorpus(entry);
+  const results = await runConformance({
+    entry,
+    manifest: corpus.compiled.manifest,
+    vectors: corpus.vectors,
+    adapter: "contract-v1",
+    command: ["bun", "bin/langfuse.mjs", "--api-version", entry.version],
+    quiet: true,
+  });
+  const versionPassed = results.filter((result) => result.passed).length;
+  passed += versionPassed;
+  total += results.length;
+  process.stdout.write(`${entry.version}: ${versionPassed}/${results.length}\n`);
+
+  for (const result of results.filter((candidate) => !candidate.passed)) {
+    failed = true;
+    process.stderr.write(`FAIL ${result.id}\n`);
+    for (const failure of result.failures) {
+      process.stderr.write(`  ${failure}\n`);
+    }
+    if (result.process.stderr) process.stderr.write(result.process.stderr);
+  }
+}
+
+process.stdout.write(`Total: ${passed}/${total}\n`);
+if (failed) process.exit(1);

conformance/src/runner.ts

@@ -1,4 +1,4 @@
-import { mkdir, mkdtemp, rm, symlink } from "node:fs/promises";
+import { mkdir, mkdtemp, rm } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join, resolve } from "node:path";
 
@@ -9,6 +9,7 @@ import {
 } from "./adapters";
 import { CaptureServer, requestDiff, sameJson } from "./capture";
 import { POLICY_PATH, REPOSITORY_ROOT, readVerifiedSpec } from "./catalog";
+import { compileApiContract } from "../../src/contracts/compiler";
 import type {
   CatalogEntry,
   ConformanceVector,
@@ -95,11 +96,6 @@ async function currentCliCommand(entry: CatalogEntry): Promise<{
   const bin = resolve(directory, "bin");
   await mkdir(dist, { recursive: true });
   await mkdir(bin, { recursive: true });
-  await symlink(
-    resolve(REPOSITORY_ROOT, "node_modules"),
-    resolve(directory, "node_modules"),
-    "dir",
-  );
   const build = Bun.spawn(
     [
       "bun",
@@ -108,7 +104,7 @@ async function currentCliCommand(entry: CatalogEntry): Promise<{
       "--outfile",
       resolve(dist, "cli.js"),
       "--target",
-      "node",
+      "bun",
       "--format",
       "esm",
     ],
@@ -125,7 +121,21 @@ async function currentCliCommand(entry: CatalogEntry): Promise<{
     await rm(directory, { recursive: true, force: true });
     throw new Error(`Current CLI build failed:\n${stdout}${stderr}`);
   }
-  await Bun.write(resolve(directory, "openapi.yml"), await readVerifiedSpec(entry));
+  const raw = await readVerifiedSpec(entry);
+  const contracts = resolve(dist, "contracts");
+  await mkdir(contracts, { recursive: true });
+  await Bun.write(
+    resolve(contracts, `${entry.version}.json`),
+    `${JSON.stringify(compileApiContract(entry, raw))}\n`,
+  );
+  await Bun.write(
+    resolve(contracts, "catalog.json"),
+    `${JSON.stringify({
+      schemaVersion: 1,
+      latest: entry.version,
+      versions: [{ version: entry.version, sourceSha256: entry.sha256 }],
+    })}\n`,
+  );
   await Bun.write(
     resolve(bin, "langfuse.mjs"),
     await Bun.file(resolve(REPOSITORY_ROOT, "bin/langfuse.mjs")).text(),