diff --git a/.github/workflows/build-iceberg-engine.yml b/.github/workflows/build-iceberg-engine.yml new file mode 100644 index 00000000..612196db --- /dev/null +++ b/.github/workflows/build-iceberg-engine.yml @@ -0,0 +1,100 @@ +name: Build Iceberg engine (WASM) + +# Builds the Iceberg Mode Explorer's engine — Redpanda's real C++ Avro->Iceberg +# schema mapper compiled to WebAssembly — for a specific Redpanda release, so +# each docs major.minor loads the engine built from that release's source. +# +# This is the analog of the Bloblang playground's update-go-modules workflow, +# keyed on Redpanda release tags instead of Go module bumps. It produces +# iceberg-engine-.js/.wasm as build artifacts; publish those to the +# versioned docs content repo under +# modules//assets/attachments/ so the tool loads the version-matched +# engine at runtime (see src/js/27-iceberg-explorer.js loader). + +on: + workflow_dispatch: + inputs: + redpanda_ref: + description: 'Redpanda git ref/tag to build the engine from (e.g. v26.2.1 or dev)' + required: true + default: 'dev' + # Trigger from the redpanda repo on a new release via repository_dispatch: + # curl -X POST .../dispatches -d '{"event_type":"redpanda-release","client_payload":{"ref":"v26.2.1"}}' + repository_dispatch: + types: [redpanda-release] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Resolve Redpanda ref and docs version + id: ver + run: | + REF="${{ github.event.inputs.redpanda_ref || github.event.client_payload.ref }}" + echo "ref=$REF" >> "$GITHUB_OUTPUT" + # major.minor from a vX.Y.Z tag; 'dev' stays 'dev'. + if [[ "$REF" =~ ^v?([0-9]+)\.([0-9]+) ]]; then + echo "version=${BASH_REMATCH[1]}.${BASH_REMATCH[2]}" >> "$GITHUB_OUTPUT" + else + echo "version=$REF" >> "$GITHUB_OUTPUT" + fi + + - name: Set up Emscripten + uses: mymindstorm/setup-emsdk@v14 + with: + version: 6.0.3 + + - name: Fetch Redpanda source + third-party deps + working-directory: iceberg-editor/wasm-spike + env: + REDPANDA_REF: ${{ steps.ver.outputs.ref }} + run: | + ./fetch-sources.sh + ./third_party/fetch-deps.sh + + - name: Build engine (web target) + working-directory: iceberg-editor/wasm + run: ./build.sh web + + - name: Stage versioned artifact + id: stage + run: | + V="${{ steps.ver.outputs.version }}" + mkdir -p dist + cp src/static/iceberg-engine.js "dist/iceberg-engine-$V.js" + cp src/static/iceberg-engine.wasm "dist/iceberg-engine-$V.wasm" + # The emscripten glue references the .wasm by its build name; rewrite + # it to the versioned name so both files can coexist per version. + sed -i "s/iceberg-engine\.wasm/iceberg-engine-$V.wasm/g" "dist/iceberg-engine-$V.js" + ls -la dist + + - name: Smoke-test the built engine under Node + working-directory: iceberg-editor/wasm + run: | + ./build.sh node + node - <<'EOF' + const create = require('./build/iceberg-engine-node.js') + create().then(m => { + const out = m.avroToIcebergJson(JSON.stringify({ + type:'record', name:'t', fields:[{name:'a',type:'string'}] + })) + const p = JSON.parse(out) + if (!p.fields || p.fields[0].type !== 'string') { console.error('FAIL', out); process.exit(1) } + console.log('engine smoke test OK:', out) + }) + EOF + + - name: Upload versioned engine artifact + uses: actions/upload-artifact@v4 + with: + name: iceberg-engine-${{ steps.ver.outputs.version }} + path: dist/ + if-no-files-found: error + + # Publishing step (commented): open a PR to the versioned content repo + # placing dist/* under modules//assets/attachments/ for the + # matching docs version. Wire this to your content-repo bot token. + # - name: Publish to content repo + # run: ./ci/publish-iceberg-engine.sh "${{ steps.ver.outputs.version }}" dist/ diff --git a/.github/workflows/iceberg-dsl-conformance.yml b/.github/workflows/iceberg-dsl-conformance.yml new file mode 100644 index 00000000..d9c01d49 --- /dev/null +++ b/.github/workflows/iceberg-dsl-conformance.yml @@ -0,0 +1,65 @@ +name: Iceberg DSL conformance + +# Verifies the Iceberg Mode Explorer's config-string builder (pure JS in +# src/js/27-iceberg-explorer.js) still matches Redpanda's authoritative DSL +# serialization, whose expectations live in +# src/v/model/tests/iceberg_mode_test.cc in the redpanda repo. +# +# The unit test (tests/iceberg-dsl/conformance-test.js) always runs. On a +# schedule / manual dispatch it also fetches the current C++ test file from a +# chosen redpanda ref and greps for the expected format strings, so upstream +# DSL changes surface as a failing check even before anyone updates the JS. + +on: + pull_request: + paths: + - 'src/js/27-iceberg-explorer.js' + - 'tests/iceberg-dsl/**' + workflow_dispatch: + inputs: + redpanda_ref: + description: 'redpanda ref to diff the DSL vectors against' + required: false + default: 'dev' + schedule: + - cron: '0 6 * * 1' # weekly: catch upstream DSL drift + +jobs: + conformance: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Run DSL conformance vectors + run: npm run test:iceberg-dsl + + - name: Cross-check vectors against upstream iceberg_mode_test.cc + if: github.event_name != 'pull_request' + env: + REF: ${{ github.event.inputs.redpanda_ref || 'dev' }} + run: | + set -euo pipefail + url="https://raw.githubusercontent.com/redpanda-data/redpanda/$REF/src/v/model/tests/iceberg_mode_test.cc" + curl -fsSL "$url" -o upstream_iceberg_mode_test.cc + # Merge C++ adjacent string-literal concatenation ("a" "b" -> "ab"), + # including across line breaks, so long expected strings that the test + # file wraps still match. + norm="$(tr -d '\n' < upstream_iceberg_mode_test.cc | sed 's/"[[:space:]]*"//g')" + # Every expected string in our conformance vectors must still appear + # verbatim in the upstream C++ format tests. + missing=0 + while IFS= read -r expected; do + [ -z "$expected" ] && continue + if ! printf '%s' "$norm" | grep -qF -- "$expected"; then + echo "::warning::Expected DSL string not found upstream ($REF): $expected" + missing=$((missing+1)) + fi + done < <(grep -oE "expect: '[^']+'" tests/iceberg-dsl/conformance-test.js | sed "s/expect: '//; s/'$//") + if [ "$missing" -gt 0 ]; then + echo "$missing expected DSL string(s) no longer present upstream — the DSL may have changed; re-sync tests/iceberg-dsl vectors and update buildConfigString." + exit 1 + fi + echo "All DSL vectors still present in upstream iceberg_mode_test.cc @ $REF" diff --git a/iceberg-editor/README.md b/iceberg-editor/README.md new file mode 100644 index 00000000..ebafa7ce --- /dev/null +++ b/iceberg-editor/README.md @@ -0,0 +1,28 @@ +# iceberg-editor + +WASM engine tooling for the interactive **Iceberg Mode Explorer** docs tool, +mirroring the layout of `blobl-editor/` (the Bloblang playground's Go→WASM +engine). + +Unlike Bloblang — whose engine is Go and wraps `benthos` directly — Iceberg +mode translation is implemented in **C++** in the Redpanda core +(`redpanda/src/v/iceberg/conversion/` + `src/v/datalake/`). To keep the docs +tool faithful to production behavior (and to stay current automatically, one +build per `major.minor` release), the engine is the **real C++ translation code +compiled to WebAssembly with Emscripten**, not a re-implementation. + +## Status + +- `wasm-spike/` — a **feasibility spike** that proves the hardest dependency + (Apache Avro C++ + a thin Seastar shim + the `iceberg/conversion` schema + mapper) compiles and links under Emscripten. Run this **before** investing in + the full engine module. See `wasm-spike/README.md`. + +## Why a spike first + +The schema/value mappers are synchronous and free of Seastar's reactor, but the +shared Iceberg type model (`iceberg/datatypes.h`) transitively includes a few +Seastar utility types (`ss::sstring`, `chunked_vector`) and Redpanda base utils. +The spike confirms those can be shimmed and that Avro C++ builds under +Emscripten. If the spike passes, the remaining work (value mapper, DSL, JSON +bindings, per-release CI) is mechanical. diff --git a/iceberg-editor/wasm-spike/.gitignore b/iceberg-editor/wasm-spike/.gitignore new file mode 100644 index 00000000..25254b82 --- /dev/null +++ b/iceberg-editor/wasm-spike/.gitignore @@ -0,0 +1,11 @@ +# Everything fetched or built is reproducible from the scripts — never commit it. + +# Fetched Redpanda source (fetch-sources.sh, pinned to $REDPANDA_REF) +vendor/ + +# Downloaded third-party deps (fmt, boost, avro fork) — keep only the fetcher +third_party/* +!third_party/fetch-deps.sh + +# Build outputs (objects, wasm, js glue) +build/ diff --git a/iceberg-editor/wasm-spike/README.md b/iceberg-editor/wasm-spike/README.md new file mode 100644 index 00000000..4020fbe9 --- /dev/null +++ b/iceberg-editor/wasm-spike/README.md @@ -0,0 +1,128 @@ +# Iceberg engine — Emscripten feasibility spike + +**Goal:** prove that Redpanda's real C++ Iceberg **schema** mapper +(`iceberg::type_to_iceberg`) compiles and links to WebAssembly under Emscripten, +against an Emscripten build of **Avro C++**, using a thin **Seastar shim** in +place of the reactor. If this passes, the full C++→WASM engine (value mapper, +DSL parser, JSON bindings, per-release CI) is mechanical. + +## ✅ RESULT: PASS (verified 2026-07-22) + +The real production mapper runs in wasm. `stage2-schema-avro.sh` prints: + +``` +OK: converted Avro record to iceberg::struct_type with 3 field(s) +``` + +**Verified toolchain (the coherent combo that works — versions matter):** + +| Component | Version / ref | Note | +| --- | --- | --- | +| Emscripten | 6.0.3 | `brew install emscripten` | +| fmt | **12.1.0** | matches redpanda `MODULE.bazel`; fmt 11 fails | +| Avro C++ | **redpanda-data/avro** @ `6821e2b4…` + `avro-fmt-const.patch` | upstream apache/avro 1.12 does **not** compile under fmt 12 | +| Boost | 1.85.0 (headers) | for `boost::outcome` | +| Redpanda source | `dev` | `iceberg/datatypes.cc`, `conversion/{avro_utils,schema_avro}.cc` | + +**What the shim had to provide** (all small, in `shim/`): `ss::sstring`→ +`std::string`, `ss::chunked_vector`→`std::vector`, `ss::bool_class` (+ its fmt +formatter), `ss::visit`/`make_visitor`, `ssx::sformat`, no-op `ss::logger`, +`ss::defer`, `vassert`, an fmt formatter for `std::exception_ptr`, and a +schema-only `conversion_outcome.h` that drops the `values.h` (iobuf/absl) +include the schema path doesn't need. + +**Green light** to build the full engine. Remaining work is mechanical and +additive: the **value** mapper (`values_avro.cc` → pulls in `values.h` → +`iobuf`/`absl`, the next dependency to size), the **Protobuf** path +(`schema_protobuf.cc` + libprotobuf-wasm), the tiny **DSL** parser +(`model/model.cc`, spec'd by `model/tests/iceberg_mode_test.cc`), Emscripten +**bindings** for `iceberg_translate(config, schema, record) → JSON`, and the +**per-release CI** that emits `iceberg-engine-.wasm`. + +This directory contains **no compiled artifacts** and **no vendored Redpanda +source** — real source is fetched from a pinned `redpanda` ref at run time +(`fetch-sources.sh`), which also mirrors the per-release build model. Nothing +here has been compiled in-repo; it is meant to be run where `emcc` is available. + +## Prerequisites + +- [emsdk / Emscripten](https://emscripten.org/docs/getting_started/downloads.html) + (`emcc`, `emcmake`, `em++`) on `PATH`. +- `cmake`, `git`, `curl`, and `gh` (for fetching pinned Redpanda source; falls + back to raw HTTPS if `gh` is unavailable). +- `node` (to run the resulting `.wasm`). + +## What gets proven, in two stages + +The spike is deliberately staged so a cheap, high-confidence check runs before +the expensive external-library step. + +### Stage 1 — `stage1-datatypes.sh` (no external libraries) + +Compiles `iceberg/datatypes.cc` to a wasm object with `emcc -c`, using only: +`fmt` (header-only), the Seastar shim (`shim/`), `named_type.h`, and a `vassert` +shim. **Proves** the Iceberg type model + Redpanda base utils compile under +Emscripten with the shim. If this fails, the shim is wrong — fix it here before +touching Avro. + +### Stage 2 — `stage2-schema-avro.sh` (the real unknown) + +Builds Apache Avro C++ for wasm (`build-avro-cpp.sh`), then compiles +`datatypes.cc` + `schema_avro.cc` + `poc_main.cc` and links them against the +Avro C++ wasm library into `build/iceberg-schema-poc.wasm`. `poc_main.cc` +parses a small Avro schema and calls `iceberg::type_to_iceberg`, so a successful +`node build/iceberg-schema-poc.js` run proves the whole path works end to end. + +To sever the schema mapper from `iobuf`/`absl` (which are only pulled in +transitively via `conversion_outcome.h` → `values.h`, and are needed only by the +*value* mapper), the spike puts a **schema-only** `conversion_outcome.h` on the +include path (`shim/iceberg/conversion/conversion_outcome.h`) that keeps +`result<>` and `conversion_exception` but drops the `values.h` include. This is +a legitimate simplification and documents that the schema and value paths have +different dependency footprints. + +## Run it + +```bash +export REDPANDA_REF=dev # or a release tag, e.g. v26.2.1 +./fetch-sources.sh # pulls real source into vendor/ at $REDPANDA_REF +./stage1-datatypes.sh # cheap proof (no Avro) +./build-avro-cpp.sh # builds Avro C++ for wasm (slow, one-time) +./stage2-schema-avro.sh # the real proof; runs the wasm via node +``` + +## Interpreting results (this is the decision the spike exists to inform) + +- **Stage 1 fails to compile** → the Seastar/base shim is incomplete. Extend + `shim/` until `datatypes.cc` compiles. Low risk; expected to be quick. +- **`build-avro-cpp.sh` fails** → Avro C++ (or its deps: Boost, Snappy) doesn't + build cleanly under Emscripten. This is the biggest risk. If it's only + optional features (codecs), disable them (`-DAVRO_*`), since the docs tool only + needs schema parsing, not container files/compression. +- **Stage 2 compiles + `node` prints `OK`** → **green light.** The real + translation code runs in wasm. Proceed to the full engine module: add the + value mapper (`values_avro.cc`, which pulls in `values.h` → `iobuf`/`absl`; the + next dependency to size), the Protobuf path, the DSL parser (tiny; specified by + `model/tests/iceberg_mode_test.cc`), Emscripten bindings for + `iceberg_translate(config, schema, record) → JSON`, and the release-tag CI. + +## Files + +| Path | Purpose | +| --- | --- | +| `fetch-sources.sh` | Fetch real Redpanda source at `$REDPANDA_REF` into `vendor/` | +| `shim/base/seastarx.h` | `ss` = `seastar` namespace alias (matches real) | +| `shim/base/vassert.h` | `vassert(...)` → `assert`/`abort` | +| `shim/seastar/core/sstring.hh` | `ss::sstring` → `std::string` | +| `shim/seastar/core/chunked_vector.hh` | `ss::chunked_vector` → `std::vector` | +| `shim/seastar/util/bool_class.hh` | `ss::bool_class` strong-typedef bool | +| `shim/seastar/util/defer.hh` | Minimal `ss::defer` | +| `shim/seastar/util/log.hh` | No-op `ss::logger` | +| `shim/iceberg/conversion/conversion_outcome.h` | Schema-only outcome (no `values.h`) | +| `poc_main.cc` | Parses an Avro schema, calls `type_to_iceberg` | +| `stage1-datatypes.sh` | Compile-only proof of the shim closure | +| `build-avro-cpp.sh` | Build Apache Avro C++ for wasm | +| `stage2-schema-avro.sh` | Compile + link + run the schema-mapper wasm | +| `third_party/` | Downloaded deps (fmt, boost.outcome, avro-cpp); git-ignored | +| `vendor/` | Fetched Redpanda source; git-ignored | +| `build/` | Build outputs; git-ignored | diff --git a/iceberg-editor/wasm-spike/build-avro-cpp.sh b/iceberg-editor/wasm-spike/build-avro-cpp.sh new file mode 100755 index 00000000..4a65a40f --- /dev/null +++ b/iceberg-editor/wasm-spike/build-avro-cpp.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Compile Apache Avro C++ (Redpanda's fork) to WebAssembly objects with emcc. +# +# We compile the sources directly rather than via CMake: the docs tool only +# needs schema PARSING (compileJsonSchemaFromString + the Node/Schema API), so +# we skip the codegen CLI (avrogencpp.cc) and container-file I/O (DataFile.cc, +# which needs Snappy/zlib codecs). This proved simpler and more portable under +# Emscripten than driving Avro's CMake. +# +# Objects land in $BUILD_DIR/wasm-obj/ and are linked by stage2-schema-avro.sh. +set -euo pipefail +source "$(dirname "${BASH_SOURCE[0]}")/env.sh" + +require emcc +[ -d "$AVRO_SRC/include/avro" ] || { + echo "Avro fork missing. Run ./third_party/fetch-deps.sh first." >&2; exit 1; } +[ -d "$FMT_INCLUDE" ] || { + echo "fmt missing. Run ./third_party/fetch-deps.sh first." >&2; exit 1; } +[ -d "$BOOST_INCLUDE/boost" ] || { + echo "boost missing. Run ./third_party/fetch-deps.sh first." >&2; exit 1; } + +OBJ="$BUILD_DIR/wasm-obj" +mkdir -p "$OBJ" + +# Avro's own .cc files include their headers unqualified (e.g. "Decoder.hh"), +# so both include/ and include/avro/ must be on the path. +AVRO_INCS=(-I "$FMT_INCLUDE" -I "$BOOST_INCLUDE" + -I "$AVRO_SRC/include" -I "$AVRO_SRC/include/avro") + +echo "Compiling Avro C++ (fork) sources to wasm objects ..." +n=0 +for f in "$AVRO_SRC"/impl/*.cc "$AVRO_SRC"/impl/parsing/*.cc \ + "$AVRO_SRC"/impl/json/*.cc; do + base="$(basename "$f")" + # Skip the codegen CLI (own main) and container-file I/O (codec deps). + case "$base" in avrogencpp.cc | DataFile.cc) continue ;; esac + emcc -std=c++20 -c -DFMT_HEADER_ONLY=1 "${AVRO_INCS[@]}" \ + "$f" -o "$OBJ/${base%.cc}.o" + n=$((n + 1)) +done +echo "PASS: compiled $n Avro C++ sources to $OBJ/" diff --git a/iceberg-editor/wasm-spike/env.sh b/iceberg-editor/wasm-spike/env.sh new file mode 100755 index 00000000..d89e8f24 --- /dev/null +++ b/iceberg-editor/wasm-spike/env.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Shared configuration for the spike scripts. Sourced by the others. +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Which Redpanda ref to pull real source from. Use `dev` while iterating; pin to +# a release tag (e.g. v26.2.1) to reproduce a specific version's behavior — this +# is the same knob the per-release CI will drive. +export REDPANDA_REF="${REDPANDA_REF:-dev}" +export REDPANDA_REPO="${REDPANDA_REPO:-redpanda-data/redpanda}" + +export SHIM_DIR="$HERE/shim" +export VENDOR_DIR="$HERE/vendor" +export TP_DIR="$HERE/third_party" +export BUILD_DIR="$HERE/build" + +# Dependency include roots. Override any of these to point at a preinstalled +# copy (recommended in CI). If unset, third_party/fetch-deps.sh populates them. +export FMT_INCLUDE="${FMT_INCLUDE:-$TP_DIR/fmt/include}" # fmt 12.1.0 +export BOOST_INCLUDE="${BOOST_INCLUDE:-$TP_DIR/boost}" # boost/ header root +export AVRO_SRC="${AVRO_SRC:-$TP_DIR/avro-cpp}" # -> redpanda avro fork lang/c++ + +mkdir -p "$VENDOR_DIR" "$TP_DIR" "$BUILD_DIR" + +require() { + command -v "$1" >/dev/null 2>&1 || { + echo "ERROR: '$1' not found on PATH. See README prerequisites." >&2 + exit 127 + } +} diff --git a/iceberg-editor/wasm-spike/fetch-sources.sh b/iceberg-editor/wasm-spike/fetch-sources.sh new file mode 100755 index 00000000..6acf8bed --- /dev/null +++ b/iceberg-editor/wasm-spike/fetch-sources.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# Fetch the exact Redpanda source files the spike compiles, at $REDPANDA_REF, +# into vendor/ preserving the include layout (so `#include "iceberg/..."` etc. +# resolve). Real source is NOT committed to this repo — this keeps the spike +# small and ties the build to a specific Redpanda ref, mirroring the eventual +# per-release build model. +# +# Uses `gh api` when available (respects auth/rate limits), else raw HTTPS. +set -euo pipefail +source "$(dirname "${BASH_SOURCE[0]}")/env.sh" + +require curl + +# Files to fetch, relative to redpanda's `src/v/`. Shimmed headers +# (seastarx.h, vassert.h, container/chunked_vector.h, conversion_outcome.h) are +# deliberately NOT fetched — the shim/ copies take precedence on the -I path. +FILES=( + "iceberg/datatypes.h" + "iceberg/datatypes.cc" + "iceberg/conversion/schema_avro.h" + "iceberg/conversion/schema_avro.cc" + "iceberg/conversion/avro_utils.h" + "iceberg/conversion/avro_utils.cc" + "base/format_to.h" + "base/outcome.h" + "utils/named_type.h" + "container/chunked_vector.h" +) + +fetch_file() { + local rel="$1" + local src="src/v/$rel" + local dst="$VENDOR_DIR/$rel" + mkdir -p "$(dirname "$dst")" + if command -v gh >/dev/null 2>&1; then + gh api "repos/$REDPANDA_REPO/contents/$src?ref=$REDPANDA_REF" \ + -q '.content' 2>/dev/null | base64 -d >"$dst" + else + curl -fsSL \ + "https://raw.githubusercontent.com/$REDPANDA_REPO/$REDPANDA_REF/$src" \ + -o "$dst" + fi + [ -s "$dst" ] || { echo "ERROR: failed to fetch $src" >&2; exit 1; } + echo " vendor/$rel" +} + +echo "Fetching Redpanda source @ $REDPANDA_REF ($REDPANDA_REPO):" +for f in "${FILES[@]}"; do fetch_file "$f"; done +echo "Done. Vendored $(echo "${#FILES[@]}") files into $VENDOR_DIR" diff --git a/iceberg-editor/wasm-spike/poc_main.cc b/iceberg-editor/wasm-spike/poc_main.cc new file mode 100644 index 00000000..1dc38844 --- /dev/null +++ b/iceberg-editor/wasm-spike/poc_main.cc @@ -0,0 +1,51 @@ +// Spike driver: parse a small Avro schema and run the REAL Redpanda schema +// mapper (iceberg::type_to_iceberg) on it. Success here means the production +// C++ translation code compiled and linked to wasm and executed under node. +// +// This intentionally exercises the whole stage-2 path: Avro C++ (schema +// parsing) + the iceberg conversion mapper + the Iceberg type model. + +#include "iceberg/conversion/schema_avro.h" +#include "iceberg/datatypes.h" + +#include +#include + +#include +#include + +int main() { + // A minimal Avro record schema, like a Kafka value would carry. + const std::string schema_json = R"({ + "type": "record", + "name": "order", + "fields": [ + {"name": "order_id", "type": "string"}, + {"name": "quantity", "type": "int"}, + {"name": "price", "type": "double"} + ] + })"; + + avro::ValidSchema schema; + try { + schema = avro::compileJsonSchemaFromString(schema_json); + } catch (const std::exception& e) { + std::printf("FAIL: could not parse Avro schema: %s\n", e.what()); + return 2; + } + + auto outcome = iceberg::type_to_iceberg(schema.root()); + if (outcome.has_error()) { + std::printf( + "FAIL: type_to_iceberg returned an error: %s\n", + outcome.error().what()); + return 1; + } + + // Count top-level fields as a cheap sanity check on the produced struct. + const auto& struct_type = outcome.value(); + std::printf( + "OK: converted Avro record to iceberg::struct_type with %zu field(s)\n", + struct_type.fields.size()); + return 0; +} diff --git a/iceberg-editor/wasm-spike/shim/base/seastarx.h b/iceberg-editor/wasm-spike/shim/base/seastarx.h new file mode 100644 index 00000000..bcb136bb --- /dev/null +++ b/iceberg-editor/wasm-spike/shim/base/seastarx.h @@ -0,0 +1,10 @@ +// Spike shim for redpanda's base/seastarx.h — matches the real header: make +// `ss` an alias for `seastar`. The actual Seastar types the Iceberg code uses +// (sstring, chunked_vector, bool_class) are provided by the dedicated +// shim headers under shim/seastar/, which the source includes +// directly. +#pragma once + +namespace seastar {} // namespace seastar + +namespace ss = seastar; diff --git a/iceberg-editor/wasm-spike/shim/base/vassert.h b/iceberg-editor/wasm-spike/shim/base/vassert.h new file mode 100644 index 00000000..8001fbe3 --- /dev/null +++ b/iceberg-editor/wasm-spike/shim/base/vassert.h @@ -0,0 +1,18 @@ +// Spike shim for redpanda's base/vassert.h. +// +// The real macro formats a rich fmt message and aborts. For the spike we only +// need it to compile and to abort on failure; the fmt-style trailing arguments +// are accepted and discarded (they are never evaluated). +#pragma once + +#include +#include + +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +#define vassert(cond, ...) \ + do { \ + if (!(cond)) { \ + std::fprintf(stderr, "vassert failed: %s\n", #cond); \ + std::abort(); \ + } \ + } while (0) diff --git a/iceberg-editor/wasm-spike/shim/iceberg/conversion/conversion_outcome.h b/iceberg-editor/wasm-spike/shim/iceberg/conversion/conversion_outcome.h new file mode 100644 index 00000000..049045bd --- /dev/null +++ b/iceberg-editor/wasm-spike/shim/iceberg/conversion/conversion_outcome.h @@ -0,0 +1,56 @@ +// Spike shim for iceberg/conversion/conversion_outcome.h — SCHEMA path only. +// +// The real header additionally includes iceberg/values.h to declare the value +// conversion outcome aliases (value_outcome, etc.). values.h transitively pulls +// in iobuf and absl int128, which the *schema* mapper (type_to_iceberg) does +// not need. To keep the spike's dependency footprint minimal we drop that +// include here and keep only what the schema mapper uses: result<> from +// base/outcome.h plus the conversion_exception type. +// +// The full engine will restore the value aliases (and size the iobuf/absl deps) +// when the value mapper is added. +#pragma once + +#include "base/outcome.h" + +#include + +#include +#include + +// The real Redpanda base provides an fmt formatter for std::exception_ptr +// (used when logging conversion failures). Provide an equivalent so the schema +// mapper compiles: format as the contained exception's message. +template<> +struct fmt::formatter : fmt::formatter { + auto format(std::exception_ptr ep, fmt::format_context& ctx) const { + std::string msg = "unknown exception"; + if (ep) { + try { + std::rethrow_exception(ep); + } catch (const std::exception& e) { + msg = e.what(); + } catch (...) { + } + } + return fmt::formatter::format(msg, ctx); + } +}; + +namespace iceberg { + +class conversion_exception final : public std::exception { +public: + explicit conversion_exception(std::string msg) noexcept + : msg_(std::move(msg)) {} + + const char* what() const noexcept final { return msg_.c_str(); } + +private: + std::string msg_; +}; + +template +using conversion_outcome = result; + +} // namespace iceberg diff --git a/iceberg-editor/wasm-spike/shim/seastar/core/chunked_vector.hh b/iceberg-editor/wasm-spike/shim/seastar/core/chunked_vector.hh new file mode 100644 index 00000000..94a4f9b9 --- /dev/null +++ b/iceberg-editor/wasm-spike/shim/seastar/core/chunked_vector.hh @@ -0,0 +1,15 @@ +// Spike shim for . +// std::vector is a drop-in for the translation logic (order-preserving, +// size(), iteration, push_back). The second template parameter mirrors +// Seastar's max-contiguous-allocation knob and is ignored. +#pragma once + +#include +#include + +namespace seastar { +template +using chunked_vector = std::vector; +} // namespace seastar + +namespace ss = seastar; diff --git a/iceberg-editor/wasm-spike/shim/seastar/core/sstring.hh b/iceberg-editor/wasm-spike/shim/seastar/core/sstring.hh new file mode 100644 index 00000000..ae7558b1 --- /dev/null +++ b/iceberg-editor/wasm-spike/shim/seastar/core/sstring.hh @@ -0,0 +1,12 @@ +// Spike shim for . +// Iceberg field names/values are plain text; std::string is behaviorally +// equivalent for the translation logic. +#pragma once + +#include + +namespace seastar { +using sstring = std::string; +} // namespace seastar + +namespace ss = seastar; diff --git a/iceberg-editor/wasm-spike/shim/seastar/util/bool_class.hh b/iceberg-editor/wasm-spike/shim/seastar/util/bool_class.hh new file mode 100644 index 00000000..86470bd4 --- /dev/null +++ b/iceberg-editor/wasm-spike/shim/seastar/util/bool_class.hh @@ -0,0 +1,62 @@ +// Spike shim for . +// A strong-typedef boolean (prevents mixing up e.g. `field_required` with a +// plain bool). Provides the subset the Iceberg code uses. +#pragma once + +#include +#include + +namespace seastar { + +template +class bool_class { + bool _value{false}; + +public: + constexpr bool_class() noexcept = default; + constexpr explicit bool_class(bool v) noexcept + : _value(v) {} + + constexpr explicit operator bool() const noexcept { return _value; } + + constexpr bool_class operator!() const noexcept { + return bool_class(!_value); + } + + friend constexpr bool + operator==(bool_class a, bool_class b) noexcept { + return a._value == b._value; + } + friend constexpr bool + operator!=(bool_class a, bool_class b) noexcept { + return a._value != b._value; + } + + static const bool_class yes; + static const bool_class no; +}; + +// Static data members of a class template may be defined in a header. +template +const bool_class bool_class::yes{true}; +template +const bool_class bool_class::no{false}; + +template +std::ostream& operator<<(std::ostream& os, bool_class v) { + return os << (static_cast(v) ? "true" : "false"); +} + +} // namespace seastar + +namespace ss = seastar; + +// The real Seastar ships an fmt formatter for bool_class; the Iceberg code +// relies on it (e.g. formatting a field's `required` flag). Format as the +// underlying bool ("true"/"false") — exact text is irrelevant to the spike. +template +struct fmt::formatter> : fmt::formatter { + auto format(seastar::bool_class v, fmt::format_context& ctx) const { + return fmt::formatter::format(static_cast(v), ctx); + } +}; diff --git a/iceberg-editor/wasm-spike/shim/seastar/util/defer.hh b/iceberg-editor/wasm-spike/shim/seastar/util/defer.hh new file mode 100644 index 00000000..eaeed754 --- /dev/null +++ b/iceberg-editor/wasm-spike/shim/seastar/util/defer.hh @@ -0,0 +1,39 @@ +// Spike shim for . +// Minimal scope-guard: runs the action on destruction unless moved-from. +#pragma once + +#include + +namespace seastar { + +template +class deferred_action { +public: + explicit deferred_action(Func f) + : _f(std::move(f)) {} + deferred_action(deferred_action&& o) noexcept + : _f(std::move(o._f)) + , _active(o._active) { + o._active = false; + } + deferred_action(const deferred_action&) = delete; + deferred_action& operator=(const deferred_action&) = delete; + ~deferred_action() { + if (_active) { + _f(); + } + } + +private: + Func _f; + bool _active{true}; +}; + +template +deferred_action defer(Func&& f) { + return deferred_action(std::forward(f)); +} + +} // namespace seastar + +namespace ss = seastar; diff --git a/iceberg-editor/wasm-spike/shim/seastar/util/log.hh b/iceberg-editor/wasm-spike/shim/seastar/util/log.hh new file mode 100644 index 00000000..cc43fa3e --- /dev/null +++ b/iceberg-editor/wasm-spike/shim/seastar/util/log.hh @@ -0,0 +1,28 @@ +// Spike shim for . +// No-op logger: accepts any fmt-style call and discards it. Sufficient because +// the schema mapper only logs diagnostics; it does not depend on log output. +#pragma once + +#include + +namespace seastar { + +class logger { +public: + explicit logger(std::string /*name*/) {} + + template + void info(const char* /*fmt*/, Args&&...) {} + template + void warn(const char* /*fmt*/, Args&&...) {} + template + void error(const char* /*fmt*/, Args&&...) {} + template + void debug(const char* /*fmt*/, Args&&...) {} + template + void trace(const char* /*fmt*/, Args&&...) {} +}; + +} // namespace seastar + +namespace ss = seastar; diff --git a/iceberg-editor/wasm-spike/shim/seastar/util/variant_utils.hh b/iceberg-editor/wasm-spike/shim/seastar/util/variant_utils.hh new file mode 100644 index 00000000..ea8479cd --- /dev/null +++ b/iceberg-editor/wasm-spike/shim/seastar/util/variant_utils.hh @@ -0,0 +1,32 @@ +// Spike shim for . +// Provides ss::visit (a thin wrapper over std::visit with the visitor first) +// and make_visitor (an overload set), the subset the Iceberg code uses. +#pragma once + +#include +#include + +namespace seastar { + +template +struct overloaded_functor : Ts... { + using Ts::operator()...; +}; +template +overloaded_functor(Ts...) -> overloaded_functor; + +template +auto make_visitor(Ts&&... fs) { + return overloaded_functor...>{std::forward(fs)...}; +} + +// Seastar's visit takes the variant first, then the visitor functors. +template +decltype(auto) visit(Variant&& v, Funcs&&... fs) { + return std::visit( + make_visitor(std::forward(fs)...), std::forward(v)); +} + +} // namespace seastar + +namespace ss = seastar; diff --git a/iceberg-editor/wasm-spike/shim/ssx/sformat.h b/iceberg-editor/wasm-spike/shim/ssx/sformat.h new file mode 100644 index 00000000..33f811b7 --- /dev/null +++ b/iceberg-editor/wasm-spike/shim/ssx/sformat.h @@ -0,0 +1,17 @@ +// Spike shim for redpanda's ssx/sformat.h. +// ssx::sformat is a thin wrapper that formats with fmt and returns an +// ss::sstring. Since our sstring is std::string, forward to fmt::format. +#pragma once + +#include + +#include + +namespace ssx { + +template +std::string sformat(fmt::format_string fmt, Args&&... args) { + return fmt::format(fmt, std::forward(args)...); +} + +} // namespace ssx diff --git a/iceberg-editor/wasm-spike/stage1-datatypes.sh b/iceberg-editor/wasm-spike/stage1-datatypes.sh new file mode 100755 index 00000000..63cac3f7 --- /dev/null +++ b/iceberg-editor/wasm-spike/stage1-datatypes.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# STAGE 1 — cheap, high-confidence proof (no external libraries). +# Compile iceberg/datatypes.cc to a wasm object using only fmt + the shim. +# If this fails, the Seastar/base shim is incomplete; fix shim/ before Avro. +set -euo pipefail +source "$(dirname "${BASH_SOURCE[0]}")/env.sh" + +require emcc +[ -f "$VENDOR_DIR/iceberg/datatypes.cc" ] || { + echo "Run ./fetch-sources.sh first." >&2; exit 1; } +[ -d "$FMT_INCLUDE" ] || { + echo "Run ./third_party/fetch-deps.sh first (fmt missing)." >&2; exit 1; } + +mkdir -p "$BUILD_DIR" +echo "Stage 1: compiling iceberg/datatypes.cc -> wasm object ..." + +# Include order matters: shim/ first so its headers win over any real ones. +emcc -std=c++20 -c \ + -DFMT_HEADER_ONLY=1 \ + -I "$SHIM_DIR" \ + -I "$VENDOR_DIR" \ + -I "$FMT_INCLUDE" \ + "$VENDOR_DIR/iceberg/datatypes.cc" \ + -o "$BUILD_DIR/datatypes.o" + +echo "PASS: datatypes.cc compiled to $BUILD_DIR/datatypes.o" +echo "The Iceberg type model + base utils compile under Emscripten with the shim." diff --git a/iceberg-editor/wasm-spike/stage2-schema-avro.sh b/iceberg-editor/wasm-spike/stage2-schema-avro.sh new file mode 100755 index 00000000..d90345de --- /dev/null +++ b/iceberg-editor/wasm-spike/stage2-schema-avro.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# STAGE 2 — the real proof. Compile the iceberg conversion TUs (datatypes.cc, +# avro_utils.cc, schema_avro.cc) + the driver to wasm, link against the Avro C++ +# wasm objects from build-avro-cpp.sh, and run under node. A printed "OK:" line +# means the production C++ schema mapper (iceberg::type_to_iceberg) runs in wasm. +set -euo pipefail +source "$(dirname "${BASH_SOURCE[0]}")/env.sh" + +require emcc +require node +[ -f "$VENDOR_DIR/iceberg/conversion/schema_avro.cc" ] || { + echo "Run ./fetch-sources.sh first." >&2; exit 1; } + +OBJ="$BUILD_DIR/wasm-obj" +[ -n "$(ls "$OBJ"/*.o 2>/dev/null)" ] || { + echo "No Avro objects. Run ./build-avro-cpp.sh first." >&2; exit 1; } + +ICE_INCS=(-I "$SHIM_DIR" -I "$VENDOR_DIR" -I "$FMT_INCLUDE") +AVRO_INCS=(-I "$BOOST_INCLUDE" -I "$AVRO_SRC/include" -I "$AVRO_SRC/include/avro") + +echo "Compiling iceberg conversion TUs + driver to wasm ..." +# datatypes.cc needs no Avro/Boost; the mappers and driver do. +emcc -std=c++20 -c -DFMT_HEADER_ONLY=1 "${ICE_INCS[@]}" \ + "$VENDOR_DIR/iceberg/datatypes.cc" -o "$OBJ/ice_datatypes.o" +emcc -std=c++20 -c -DFMT_HEADER_ONLY=1 "${ICE_INCS[@]}" "${AVRO_INCS[@]}" \ + "$VENDOR_DIR/iceberg/conversion/avro_utils.cc" -o "$OBJ/ice_avro_utils.o" +emcc -std=c++20 -c -DFMT_HEADER_ONLY=1 "${ICE_INCS[@]}" "${AVRO_INCS[@]}" \ + "$VENDOR_DIR/iceberg/conversion/schema_avro.cc" -o "$OBJ/ice_schema_avro.o" +emcc -std=c++20 -c -DFMT_HEADER_ONLY=1 "${ICE_INCS[@]}" "${AVRO_INCS[@]}" \ + "$(dirname "${BASH_SOURCE[0]}")/poc_main.cc" -o "$OBJ/poc_main.o" + +OUT="$BUILD_DIR/iceberg-schema-poc.js" +echo "Linking -> $OUT ..." +emcc -std=c++20 "$OBJ"/*.o \ + -s ALLOW_MEMORY_GROWTH=1 -s EXIT_RUNTIME=1 \ + -o "$OUT" + +echo "Running under node ..." +node "$OUT" +echo "" +echo "If you see 'OK:' above, the real C++ schema mapper runs in wasm." +echo "Green light to build the full engine module (value mapper, DSL, bindings)." diff --git a/iceberg-editor/wasm-spike/third_party/fetch-deps.sh b/iceberg-editor/wasm-spike/third_party/fetch-deps.sh new file mode 100755 index 00000000..77b9ed53 --- /dev/null +++ b/iceberg-editor/wasm-spike/third_party/fetch-deps.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# Fetch the dependencies the spike links against, PINNED to the same versions +# Redpanda's real build uses (so the toolchain is coherent): +# - fmt 12.1.0 (redpanda MODULE.bazel: bazel_dep fmt 12.1.0) +# - Boost headers (for boost::outcome, used by base/outcome.h) +# - Avro C++ (redpanda-data/avro fork @ AVRO_SHA + redpanda's +# avro-fmt-const.patch / avro-snappy-includes.patch) +# +# The Avro *fork + patches* matter: upstream apache/avro 1.12 ships an +# fmt::formatter with a non-const format() method that fmt 12 rejects; the +# fork's patch fixes exactly that. Using upstream Avro will fail to compile. +set -euo pipefail +source "$(dirname "${BASH_SOURCE[0]}")/../env.sh" + +require git +require curl + +FMT_TAG="${FMT_TAG:-12.1.0}" +BOOST_VER="${BOOST_VER:-1.85.0}" +# Redpanda's Avro fork commit (bazel/repositories.bzl -> name = "avro"). +AVRO_SHA="${AVRO_SHA:-6821e2b454401308d4e3819c0569d0fe7f2a66fa}" +AVRO_PATCH_REF="${AVRO_PATCH_REF:-dev}" # redpanda ref to pull patches from + +# --- fmt 12.1.0 (header-only via FMT_HEADER_ONLY) --- +if [ ! -d "$FMT_INCLUDE" ]; then + echo "Fetching fmt $FMT_TAG ..." + git clone --depth 1 --branch "$FMT_TAG" https://github.com/fmtlib/fmt \ + "$TP_DIR/fmt" +fi + +# --- Boost headers (Boost.Outcome is header-only) --- +if [ ! -d "$BOOST_INCLUDE/boost" ]; then + echo "Fetching Boost $BOOST_VER headers ..." + ver_us="${BOOST_VER//./_}" + url="https://archives.boost.io/release/$BOOST_VER/source/boost_${ver_us}.tar.gz" + curl -fsSL "$url" -o "$TP_DIR/boost.tar.gz" + tar -xzf "$TP_DIR/boost.tar.gz" -C "$TP_DIR" + rm -rf "$BOOST_INCLUDE" + mkdir -p "$BOOST_INCLUDE" + mv "$TP_DIR/boost_${ver_us}/boost" "$BOOST_INCLUDE/boost" +fi + +# --- Avro C++ (Redpanda fork @ AVRO_SHA, patched) --- +AVRO_EXTRACT="$TP_DIR/avro-$AVRO_SHA" +if [ ! -e "$AVRO_SRC/include/avro/Node.hh" ]; then + echo "Fetching redpanda-data/avro @ $AVRO_SHA ..." + curl -fsSL "https://github.com/redpanda-data/avro/archive/$AVRO_SHA.tar.gz" \ + -o "$TP_DIR/avro.tar.gz" + # Ignore tar's harmless CRC warnings on Java test-resource files. + tar -xzf "$TP_DIR/avro.tar.gz" -C "$TP_DIR" || true + ln -sfn "$AVRO_EXTRACT/lang/c++" "$AVRO_SRC" + + echo "Applying redpanda Avro patches ..." + for p in avro-fmt-const.patch avro-snappy-includes.patch; do + if command -v gh >/dev/null 2>&1; then + gh api "repos/$REDPANDA_REPO/contents/bazel/thirdparty/$p?ref=$AVRO_PATCH_REF" \ + -q '.content' | base64 -d >"$TP_DIR/$p" + else + curl -fsSL \ + "https://raw.githubusercontent.com/$REDPANDA_REPO/$AVRO_PATCH_REF/bazel/thirdparty/$p" \ + -o "$TP_DIR/$p" + fi + (cd "$AVRO_EXTRACT" && patch -p1 <"$TP_DIR/$p") + done +fi + +echo "third_party ready:" +echo " fmt -> $FMT_INCLUDE ($FMT_TAG)" +echo " boost -> $BOOST_INCLUDE ($BOOST_VER)" +echo " avro -> $AVRO_SRC (redpanda fork @ ${AVRO_SHA:0:12}, patched)" diff --git a/iceberg-editor/wasm/.gitignore b/iceberg-editor/wasm/.gitignore new file mode 100644 index 00000000..dbc65241 --- /dev/null +++ b/iceberg-editor/wasm/.gitignore @@ -0,0 +1,2 @@ +# Build outputs (objects, node test module, wasm) — reproducible via build.sh +build/ diff --git a/iceberg-editor/wasm/build.sh b/iceberg-editor/wasm/build.sh new file mode 100755 index 00000000..558f71a6 --- /dev/null +++ b/iceberg-editor/wasm/build.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# Build the Iceberg engine WASM module (real C++ Avro->Iceberg schema mapper +# + Embind bindings) for a JS host. +# +# Reuses the vendored Redpanda source, Seastar shim, and third-party deps that +# the feasibility spike fetches (../wasm-spike). Run the spike's +# fetch-sources.sh + third_party/fetch-deps.sh first (build-avro-cpp.sh is NOT +# required — this script compiles Avro itself, with exceptions enabled so the +# binding's try/catch can return a JSON error instead of aborting). +# +# Usage: +# ./build.sh node -> build/iceberg-engine-node.js (test under node) +# ./build.sh web -> ../../src/static/iceberg-engine.js(+.wasm) (docs-ui) +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SPIKE="$HERE/../wasm-spike" +TARGET="${1:-web}" + +SHIM="$SPIKE/shim" +VENDOR="$SPIKE/vendor" +TP="$SPIKE/third_party" +AVRO="$TP/avro-cpp" + +command -v emcc >/dev/null 2>&1 || { echo "emcc not on PATH" >&2; exit 127; } +[ -f "$VENDOR/iceberg/conversion/schema_avro.cc" ] || { + echo "Redpanda source missing. Run ../wasm-spike/fetch-sources.sh first." >&2 + exit 1 +} +[ -d "$AVRO/include/avro" ] || { + echo "Avro missing. Run ../wasm-spike/third_party/fetch-deps.sh first." >&2 + exit 1 +} + +# -fexceptions everywhere: Avro throws on malformed input and the binding +# catches it; both sides must share the same exception ABI. +EXC="-fexceptions" +ICE_INCS=(-I "$SHIM" -I "$VENDOR" -I "$TP/fmt/include") +AVRO_INCS=(-I "$TP/fmt/include" -I "$TP/boost" -I "$AVRO/include" -I "$AVRO/include/avro") + +OBJ="$HERE/build/obj" +mkdir -p "$OBJ" + +echo "Compiling Avro C++ (schema subset) with exceptions ..." +for f in "$AVRO"/impl/*.cc "$AVRO"/impl/parsing/*.cc "$AVRO"/impl/json/*.cc; do + base="$(basename "$f")" + case "$base" in avrogencpp.cc | DataFile.cc) continue ;; esac + o="$OBJ/avro_${base%.cc}.o" + [ -f "$o" ] && [ "$o" -nt "$f" ] && continue + emcc -std=c++20 -c $EXC -DFMT_HEADER_ONLY=1 "${AVRO_INCS[@]}" "$f" -o "$o" +done + +echo "Compiling iceberg conversion TUs + binding ..." +emcc -std=c++20 -c $EXC -DFMT_HEADER_ONLY=1 "${ICE_INCS[@]}" \ + "$VENDOR/iceberg/datatypes.cc" -o "$OBJ/ice_datatypes.o" +emcc -std=c++20 -c $EXC -DFMT_HEADER_ONLY=1 "${ICE_INCS[@]}" "${AVRO_INCS[@]}" \ + "$VENDOR/iceberg/conversion/avro_utils.cc" -o "$OBJ/ice_avro_utils.o" +emcc -std=c++20 -c $EXC -DFMT_HEADER_ONLY=1 "${ICE_INCS[@]}" "${AVRO_INCS[@]}" \ + "$VENDOR/iceberg/conversion/schema_avro.cc" -o "$OBJ/ice_schema_avro.o" +emcc -std=c++20 -c $EXC -DFMT_HEADER_ONLY=1 "${ICE_INCS[@]}" "${AVRO_INCS[@]}" \ + "$HERE/iceberg_wasm.cc" -o "$OBJ/iceberg_wasm.o" + +LINK=(-std=c++20 $EXC --bind -O2 "$OBJ"/*.o -s ALLOW_MEMORY_GROWTH=1 + -s MODULARIZE=1 -s EXPORT_NAME=createIcebergEngine) + +if [ "$TARGET" = "node" ]; then + echo "Linking Node test module ..." + emcc "${LINK[@]}" -s ENVIRONMENT=node -o "$HERE/build/iceberg-engine-node.js" + echo "PASS: $HERE/build/iceberg-engine-node.js" +else + OUTDIR="$HERE/../../src/static" + echo "Linking browser module -> $OUTDIR/iceberg-engine.js ..." + emcc "${LINK[@]}" -s "ENVIRONMENT=web,worker" \ + -o "$OUTDIR/iceberg-engine.js" + echo "PASS: $OUTDIR/iceberg-engine.js (+ .wasm)" +fi diff --git a/iceberg-editor/wasm/iceberg_wasm.cc b/iceberg-editor/wasm/iceberg_wasm.cc new file mode 100644 index 00000000..0805a4a9 --- /dev/null +++ b/iceberg-editor/wasm/iceberg_wasm.cc @@ -0,0 +1,144 @@ +// Browser/Node entry point for the Iceberg engine WASM module. +// +// Exposes a single Embind function, `avroToIcebergJson(avroSchemaJson)`, that +// runs Redpanda's REAL C++ Avro->Iceberg schema mapper (iceberg::type_to_iceberg) +// and serializes the resulting iceberg::struct_type to a compact JSON shape the +// docs-ui hydration module already knows how to render: +// +// { "fields": [ { "name": "...", "type": "", "required": bool, +// "fields": [ ... ] /* only when type == "struct" */ } ] } +// // or on failure: { "error": "..." } +// +// This is the production translation code compiled to WebAssembly — not a +// re-implementation. See iceberg-editor/wasm-spike for the feasibility proof +// and the pinned toolchain (fmt 12.1.0, redpanda-data/avro fork + patches). + +#include "iceberg/conversion/schema_avro.h" +#include "iceberg/datatypes.h" + +#include +#include +#include + +#include +#include + +namespace { + +template +struct overloaded : Ts... { + using Ts::operator()...; +}; +template +overloaded(Ts...) -> overloaded; + +void escape_json(const std::string& s, std::string& out) { + for (char c : s) { + switch (c) { + case '"': + out += "\\\""; + break; + case '\\': + out += "\\\\"; + break; + case '\n': + out += "\\n"; + break; + default: + out += c; + } + } +} + +std::string struct_fields_json(const iceberg::struct_type& s); + +// Render a field_type as the display type string used by the UI, plus (for +// structs) the nested fields JSON via the out-param. +std::string type_string(const iceberg::field_type& t, std::string& nestedOut) { + std::string result; + std::visit( + overloaded{ + [&](const iceberg::primitive_type& p) { + result = fmt::format("{}", p); + }, + [&](const iceberg::struct_type& s) { + result = "struct"; + nestedOut = struct_fields_json(s); + }, + [&](const iceberg::list_type& l) { + std::string ignored; + result = "list<" + type_string(l.element_field->type, ignored) + + ">"; + }, + [&](const iceberg::map_type& m) { + std::string ik, iv; + result = "map<" + type_string(m.key_field->type, ik) + ", " + + type_string(m.value_field->type, iv) + ">"; + }, + }, + t); + return result; +} + +std::string field_json(const iceberg::nested_field& f) { + std::string out = "{\"name\":\""; + escape_json(f.name, out); + out += "\",\"required\":"; + out += (static_cast(f.required) ? "true" : "false"); + std::string nested; + std::string ty = type_string(f.type, nested); + out += ",\"type\":\""; + escape_json(ty, out); + out += "\""; + if (!nested.empty()) { + out += ",\"fields\":"; + out += nested; + } + out += "}"; + return out; +} + +std::string struct_fields_json(const iceberg::struct_type& s) { + std::string out = "["; + bool first = true; + for (const auto& fptr : s.fields) { + if (!first) { + out += ","; + } + first = false; + out += field_json(*fptr); + } + out += "]"; + return out; +} + +std::string avro_to_iceberg_json(const std::string& avro_schema_json) { + avro::ValidSchema schema; + try { + schema = avro::compileJsonSchemaFromString(avro_schema_json); + } catch (const std::exception& e) { + std::string out = "{\"error\":\"invalid Avro schema: "; + escape_json(e.what(), out); + out += "\"}"; + return out; + } + + auto outcome = iceberg::type_to_iceberg(schema.root()); + if (outcome.has_error()) { + std::string out = "{\"error\":\""; + escape_json(outcome.error().what(), out); + out += "\"}"; + return out; + } + + std::string out = "{\"fields\":"; + out += struct_fields_json(outcome.value()); + out += "}"; + return out; +} + +} // namespace + +EMSCRIPTEN_BINDINGS(iceberg_engine) { + emscripten::function("avroToIcebergJson", &avro_to_iceberg_json); +} diff --git a/package.json b/package.json index fea318c2..9bb93066 100644 --- a/package.json +++ b/package.json @@ -79,6 +79,7 @@ "test:playground": "npm run build:wasm && npm run test:headless", "test:interactive": "node tests/bloblang-interactive/test-runner.js", "test:all": "npm run test:playground && npm run test:interactive", + "test:iceberg-dsl": "node tests/iceberg-dsl/conformance-test.js", "build:wasm": "cd blobl-editor/wasm && GOOS=js GOARCH=wasm go build -o ../../src/static/blobl.wasm .", "copy:wasm-exec": "cp \"$(go env GOROOT)/lib/wasm/wasm_exec.js\" src/js/vendor/", "serve:playground": "npx serve ." diff --git a/preview-src/iceberg-explorer-test.adoc b/preview-src/iceberg-explorer-test.adoc new file mode 100644 index 00000000..7ce07223 --- /dev/null +++ b/preview-src/iceberg-explorer-test.adoc @@ -0,0 +1,22 @@ += Iceberg Mode Explorer +:page-layout: default +:page-no-toc: true +:description: Interactive explorer for Redpanda Iceberg mode key/value/header translation. + +Pick a configuration and see the DSL config string, the resulting Iceberg table +schema, and a sample translated record update live. + +== Default + +[iceberg-explorer] +-- +-- + +== With an author-supplied initial config + +[iceberg-explorer] +---- +{ + "config": "key:mode=schema_id_prefix;value:mode=schema_latest,layout=nested;headers:value_type=string" +} +---- diff --git a/preview-src/ui-model.yml b/preview-src/ui-model.yml index 3add08d6..d906c94f 100644 --- a/preview-src/ui-model.yml +++ b/preview-src/ui-model.yml @@ -1,4 +1,10 @@ antoraVersion: '1.0.0' +# Register Asciidoctor macros for the preview build (loaded by +# gulp.d/tasks/build-preview-pages.js). The iceberg-explorer block is provided +# by the docs-extensions-and-macros package (npm-linked locally for preview). +asciidoc: + extensions: + - '@redpanda-data/docs-extensions-and-macros/macros/iceberg-explorer' site: url: https://deploy-preview-327--docs-ui.netlify.app title: UI Preview diff --git a/src/css/iceberg-explorer.css b/src/css/iceberg-explorer.css new file mode 100644 index 00000000..b0d91651 --- /dev/null +++ b/src/css/iceberg-explorer.css @@ -0,0 +1,183 @@ +/* + * Iceberg Mode Explorer — interactive docs tool. + * Built on the site's semantic theme tokens (vars.css / dark-mode.css) so it + * follows the docs light/dark theme automatically. Rendered by + * src/js/27-iceberg-explorer.js into the `[iceberg-explorer]` block mount point. + */ + +.iceberg-explorer { + --ice-accent: var(--product-color, #e2401b); + --ice-muted: var(--color-gray-70, #6b7280); + --ice-border: var(--panel-border-color, #d0d3d9); + --ice-panel: var(--panel-background, #f7f8fa); + --ice-pre: var(--pre-background, #f0f1f4); + --ice-code: var(--code-font-color, var(--body-font-color)); + --ice-green: #1a7f5a; + --ice-blue: #2563c9; + --ice-yellow: #9a6b00; + --ice-purple: #7c3aed; + --ice-cyan: #0e7490; + font-size: var(--secondary-font-size, 0.9rem); + margin: 1.25rem 0; +} +[data-theme="dark"] .iceberg-explorer { + --ice-green: #4ade80; + --ice-blue: #60a5fa; + --ice-yellow: #facc15; + --ice-purple: #c084fc; + --ice-cyan: #22d3ee; +} + +.iceberg-explorer .ice-grid { + display: grid; + grid-template-columns: 340px 1fr; + gap: 1.25rem; + align-items: start; +} +@media (max-width: 900px) { + .iceberg-explorer .ice-grid { grid-template-columns: 1fr; } +} + +.iceberg-explorer .ice-card { + background: var(--ice-panel); + border: 1px solid var(--ice-border); + border-radius: 8px; + padding: 1rem 1.1rem; + margin-bottom: 1rem; +} +.iceberg-explorer .ice-card h3 { + font-size: 0.8rem; + text-transform: uppercase; + letter-spacing: 0.05em; + color: var(--ice-muted); + margin: 0 0 0.75rem; + padding-bottom: 0.4rem; + border-bottom: 1px solid var(--ice-border); +} + +.iceberg-explorer .ice-label { + font-size: 0.75rem; + font-weight: 600; + text-transform: uppercase; + letter-spacing: 0.05em; + color: var(--ice-cyan); + margin-bottom: 0.4rem; +} + +.iceberg-explorer .ice-radios { + display: flex; + flex-wrap: wrap; + gap: 0.35rem; + margin-bottom: 0.4rem; +} +.iceberg-explorer .ice-radio { + display: inline-flex; + align-items: center; + gap: 0.35rem; + font-size: 0.8rem; + padding: 0.3rem 0.6rem; + border: 1px solid var(--ice-border); + border-radius: 6px; + cursor: pointer; + user-select: none; + transition: border-color 0.15s, background 0.15s; +} +.iceberg-explorer .ice-radio:hover { border-color: var(--ice-accent); } +.iceberg-explorer .ice-radio input { accent-color: var(--ice-accent); margin: 0; } + +.iceberg-explorer .ice-textrow { + display: flex; + align-items: center; + gap: 0.5rem; + margin-top: 0.4rem; +} +.iceberg-explorer .ice-textrow label { + font-size: 0.75rem; + color: var(--ice-muted); + min-width: 88px; + text-align: right; +} +.iceberg-explorer .ice-textrow input { + flex: 1; + background: var(--ice-pre); + border: 1px solid var(--ice-border); + border-radius: 5px; + padding: 0.3rem 0.5rem; + color: var(--body-font-color); + font-family: var(--code-font-family, monospace); + font-size: 0.78rem; +} +.iceberg-explorer .ice-textrow input:disabled { opacity: 0.4; cursor: not-allowed; } + +.iceberg-explorer .ice-configbar { + display: flex; + align-items: center; + gap: 0.85rem; + background: var(--ice-panel); + border: 1px solid var(--ice-border); + border-radius: 8px; + padding: 0.85rem 1rem; + margin-bottom: 1rem; +} +.iceberg-explorer .ice-configlabel { + font-size: 0.7rem; + text-transform: uppercase; + letter-spacing: 0.05em; + color: var(--ice-muted); + white-space: nowrap; +} +.iceberg-explorer .ice-configstr { + flex: 1; + font-family: var(--code-font-family, monospace); + font-size: 0.9rem; + color: var(--ice-green); + word-break: break-all; + background: none; + padding: 0; +} +.iceberg-explorer .ice-copy { + flex-shrink: 0; + background: transparent; + border: 1px solid var(--ice-border); + color: var(--ice-muted); + border-radius: 6px; + padding: 0.35rem 0.7rem; + font-size: 0.75rem; + cursor: pointer; + transition: border-color 0.15s, color 0.15s; +} +.iceberg-explorer .ice-copy:hover { border-color: var(--ice-accent); color: var(--ice-accent); } +.iceberg-explorer .ice-copy.ice-copied { border-color: var(--ice-green); color: var(--ice-green); } + +.iceberg-explorer .ice-tree { + font-family: var(--code-font-family, monospace); + font-size: 0.8rem; + line-height: 1.65; + white-space: pre; + overflow-x: auto; + margin: 0; + background: none; + color: var(--ice-code); +} +.iceberg-explorer .ice-tree .field-name { color: var(--ice-blue); } +.iceberg-explorer .ice-tree .field-type { color: var(--ice-green); } +.iceberg-explorer .ice-tree .field-req { color: var(--ice-muted); font-size: 0.72rem; } +.iceberg-explorer .ice-tree .struct-brace { color: var(--ice-muted); } +.iceberg-explorer .ice-tree .comment { color: var(--ice-muted); font-style: italic; } +.iceberg-explorer .ice-tree .field-val-str { color: var(--ice-green); } +.iceberg-explorer .ice-tree .field-val-num { color: var(--ice-yellow); } +.iceberg-explorer .ice-tree .field-val-bin { color: var(--ice-purple); } + +.iceberg-explorer .ice-tag { + display: inline-block; + font-size: 0.62rem; + text-transform: uppercase; + letter-spacing: 0.04em; + padding: 0.1rem 0.4rem; + border-radius: 4px; + vertical-align: middle; +} +.iceberg-explorer .ice-tag.ice-new { background: rgba(124, 58, 237, 0.15); color: var(--ice-purple); } +.iceberg-explorer .ice-tag.ice-legacy { background: rgba(154, 107, 0, 0.15); color: var(--ice-yellow); } +.iceberg-explorer .ice-tag.ice-engine-real { background: rgba(26, 127, 90, 0.15); color: var(--ice-green); text-transform: none; letter-spacing: 0; } +.iceberg-explorer .ice-tag.ice-engine-preview { background: rgba(107, 114, 128, 0.15); color: var(--ice-muted); text-transform: none; letter-spacing: 0; } diff --git a/src/css/site.css b/src/css/site.css index 06e8c60f..32f674a4 100644 --- a/src/css/site.css +++ b/src/css/site.css @@ -56,3 +56,4 @@ @import "product-switcher.css"; @import "labs-home.css"; @import "hero-light-mode.css"; +@import "iceberg-explorer.css"; diff --git a/src/js/27-iceberg-explorer.js b/src/js/27-iceberg-explorer.js new file mode 100644 index 00000000..50a25241 --- /dev/null +++ b/src/js/27-iceberg-explorer.js @@ -0,0 +1,540 @@ +;(function () { + 'use strict' + + // Iceberg Mode Explorer — docs-ui hydration module. + // + // The `[iceberg-explorer]` block (docs-extensions-and-macros) emits an empty + // `
` mount point. This module finds + // every such mount point and renders the interactive tool into it. + // + // ENGINE SEAM: all translation logic lives behind `translate(cfg)` below. + // Today that is an in-browser port of the rules in Redpanda's + // `model/iceberg_mode` + `iceberg/conversion` (an interim engine). It is + // structured so a per-version WASM build of the real C++ engine can be + // dropped in later without touching the UI — see docs-ui/iceberg-editor/. + + // ── Fixed sample data (illustrative). A future iteration swaps these for + // user-supplied schema/record via editors; the engine seam stays the same. + var KEY_SCHEMA = [ + { name: 'user_id', type: 'long' }, + { name: 'region', type: 'string' }, + ] + var VAL_SCHEMA = [ + { name: 'order_id', type: 'string' }, + { name: 'product', type: 'string' }, + { name: 'quantity', type: 'int' }, + { name: 'price', type: 'double' }, + { name: 'shipped', type: 'boolean' }, + { name: 'shipping_address', type: 'struct', fields: [ + { name: 'street', type: 'string' }, + { name: 'city', type: 'string' }, + { name: 'zip', type: 'string' }, + { name: 'tags', type: 'list' }, + ] }, + ] + var RECORD = { + partition: 3, offset: 42, timestamp: 1720000000000, + value: { + order_id: 'ORD-98765', product: 'Redpanda Plushie', quantity: 3, + price: 29.99, shipped: false, + shipping_address: { street: '123 Main St', city: 'Seattle', zip: '98101', tags: ['residential', 'priority'] }, + }, + headers: [ + { key: 'content-type', value: 'application/json' }, + { key: 'x-request-id', value: 'req-abc-123' }, + { key: 'x-trace-id', value: 'trace-00f1e2d3' }, + ], + } + + // The sample value schema expressed as real Avro JSON. When the WASM engine + // is available, this is fed to the REAL Redpanda C++ mapper to compute the + // Iceberg value fields (see below); otherwise the tool falls back to the + // VAL_SCHEMA rules above (interim engine). + var AVRO_VALUE_SCHEMA = JSON.stringify({ + type: 'record', name: 'order', fields: [ + { name: 'order_id', type: 'string' }, + { name: 'product', type: 'string' }, + { name: 'quantity', type: 'int' }, + { name: 'price', type: 'double' }, + { name: 'shipped', type: 'boolean' }, + { name: 'shipping_address', type: { type: 'record', name: 'shipping_address', fields: [ + { name: 'street', type: 'string' }, + { name: 'city', type: 'string' }, + { name: 'zip', type: 'string' }, + { name: 'tags', type: { type: 'array', items: 'string' } } + ] } } + ] + }) + + // ── Real WASM engine loader (lazy singleton). Resolves to the parsed value + // fields computed by Redpanda's actual C++ Avro->Iceberg mapper, or null + // if no engine can be loaded (then the interim JS engine is used). + // + // The engine is versioned: schema-translation rules can change per release, + // so each docs version loads the engine built from that release tag. We try + // candidates in priority order and fall back gracefully: + // 1. data-engine-base override on the mount (CI/author points this at the + // versioned content-attachments dir where release-tag CI publishes + // iceberg-engine-.js/.wasm). + // 2. versioned filename in the UI root: /iceberg-engine-.js + // 3. unversioned /iceberg-engine.js (single-bundle / preview). + // 4. none -> interim JS engine. + var enginePromise = null + function uiRoot () { + // Derive the UI root (…/_) from the already-loaded site.js script src so + // this works at any doc version/path. + var s = document.querySelector('script[src*="/js/site.js"]') + if (s) return s.src.replace(/\/js\/site\.js.*$/, '') + return '/_' + } + function engineCandidates (version, overrideBase) { + var root = uiRoot() + var list = [] + if (overrideBase && version) { + list.push({ dir: overrideBase, file: 'iceberg-engine-' + version + '.js' }) + } + if (version) { + list.push({ dir: root, file: 'iceberg-engine-' + version + '.js' }) + } + list.push({ dir: root, file: 'iceberg-engine.js' }) + return list + } + // Load one candidate; resolve to value fields, or null on any failure so the + // caller can try the next candidate. + function tryCandidate (cand) { + return new Promise(function (resolve) { + var url = cand.dir.replace(/\/$/, '') + '/' + cand.file + var script = document.createElement('script') + script.src = url + script.onload = function () { + if (typeof createIcebergEngine !== 'function') { resolve(null); return } + createIcebergEngine({ locateFile: function (p) { return cand.dir.replace(/\/$/, '') + '/' + p } }) + .then(function (mod) { + try { + var parsed = JSON.parse(mod.avroToIcebergJson(AVRO_VALUE_SCHEMA)) + resolve(parsed.error ? null : parsed.fields) + } catch (e) { resolve(null) } + }, function () { resolve(null) }) + } + script.onerror = function () { resolve(null) } + document.head.appendChild(script) + }) + } + function loadEngineValueFields (version, overrideBase) { + if (enginePromise) return enginePromise + var cands = engineCandidates(version, overrideBase) + enginePromise = (function next (i) { + if (i >= cands.length) return Promise.resolve(null) + return tryCandidate(cands[i]).then(function (fields) { + return fields || next(i + 1) + }) + })(0) + return enginePromise + } + + function esc (s) { + return String(s).replace(/&/g, '&').replace(//g, '>') + } + function isSchemaMode (mode) { + return mode === 'schema_id_prefix' || mode === 'schema_latest' + } + + // ── Config-string builder — mirrors model/model.cc format_to(): + // legacy strings when legacy-compatible, else the section-based DSL, + // always serializing all sections (matches PR #31035 behavior). + function buildConfigString (cfg) { + var keyDefault = cfg.keyMode === 'binary' + var hdrDefault = cfg.hdrType === 'binary' + var valLayoutDefault = cfg.valLayout === 'flat' + var valNotString = cfg.valMode !== 'string' + var legacyCompat = keyDefault && hdrDefault && valLayoutDefault && valNotString + + if (legacyCompat) { + if (cfg.valMode === 'binary') return { str: 'key_value', legacy: true } + if (cfg.valMode === 'schema_id_prefix') return { str: 'value_schema_id_prefix', legacy: true } + if (cfg.valMode === 'schema_latest') { + // Legacy option order (model.cc format_to): protobuf_name then subject. + var s = 'value_schema_latest' + var opts = [] + if (cfg.valProto) opts.push('protobuf_name=' + cfg.valProto) + if (cfg.valSubject) opts.push('subject=' + cfg.valSubject) + if (opts.length) s += ':' + opts.join(',') + return { str: s, legacy: true } + } + } + + // Section format (model.cc): always emit all sections and all options so + // defaults are frozen at set-time. Key: mode, subject?, protobuf_name?. + // Value: mode, subject?, protobuf_name?, layout (always, last). + var sections = [] + var keyOpts = ['mode=' + cfg.keyMode] + if (cfg.keySubject) keyOpts.push('subject=' + cfg.keySubject) + if (cfg.keyProto) keyOpts.push('protobuf_name=' + cfg.keyProto) + sections.push('key:' + keyOpts.join(',')) + + var valOpts = ['mode=' + cfg.valMode] + if (cfg.valSubject) valOpts.push('subject=' + cfg.valSubject) + if (cfg.valProto) valOpts.push('protobuf_name=' + cfg.valProto) + valOpts.push('layout=' + cfg.valLayout) + sections.push('value:' + valOpts.join(',')) + + sections.push('headers:value_type=' + cfg.hdrType) + return { str: sections.join(';'), legacy: false } + } + + function renderSchemaFields (lines, fields, depth, S, indent) { + for (var i = 0; i < fields.length; i++) { + var f = fields[i] + // Honor an explicit `required` flag (present on real-engine output); + // default to optional for the interim JS schema. + var req = f.required === true ? 'required' : 'optional' + if (f.type === 'struct' && f.fields) { + lines.push(indent(depth) + S('field-name', f.name) + ': ' + S('struct-brace', '{') + ' ' + S('field-req', req)) + renderSchemaFields(lines, f.fields, depth + 1, S, indent) + lines.push(indent(depth) + S('struct-brace', '}')) + } else { + lines.push(indent(depth) + S('field-name', f.name) + ': ' + S('field-type', f.type) + ' ' + S('field-req', req)) + } + } + } + function renderRecordFields (lines, fields, values, depth, S, indent) { + for (var i = 0; i < fields.length; i++) { + var f = fields[i] + if (f.type === 'struct' && f.fields && values[f.name]) { + lines.push(indent(depth) + S('field-name', f.name) + ': ' + S('struct-brace', '{')) + renderRecordFields(lines, f.fields, values[f.name], depth + 1, S, indent) + lines.push(indent(depth) + S('struct-brace', '}')) + } else { + var v = values[f.name] + if (Array.isArray(v)) { + var items = v.map(function (x) { return S('field-val-str', '"' + x + '"') }).join(', ') + lines.push(indent(depth) + S('field-name', f.name) + ': [' + items + ']') + } else if (typeof v === 'string') { + lines.push(indent(depth) + S('field-name', f.name) + ': ' + S('field-val-str', '"' + v + '"')) + } else { + lines.push(indent(depth) + S('field-name', f.name) + ': ' + S('field-val-num', String(v))) + } + } + } + } + + function renderSchema (cfg, valFields) { + var lines = [] + var S = function (cls, text) { return '' + esc(text) + '' } + var indent = function (n) { var o = ''; for (var i = 0; i < n; i++) o += ' '; return o } + var useKeySchema = isSchemaMode(cfg.keyMode) + var useValSchema = isSchemaMode(cfg.valMode) + var valSchema = valFields || VAL_SCHEMA + + lines.push(S('struct-brace', 'row {')) + lines.push(indent(1) + S('field-name', 'redpanda') + ': ' + S('struct-brace', '{') + ' ' + S('field-req', 'optional')) + lines.push(indent(2) + S('field-name', 'partition') + ': ' + S('field-type', 'int') + ' ' + S('field-req', 'optional')) + lines.push(indent(2) + S('field-name', 'offset') + ': ' + S('field-type', 'long') + ' ' + S('field-req', 'optional')) + lines.push(indent(2) + S('field-name', 'timestamp') + ': ' + S('field-type', 'timestamptz') + ' ' + S('field-req', 'optional')) + var hdrValType = cfg.hdrType === 'string' ? 'string' : 'binary' + lines.push(indent(2) + S('field-name', 'headers') + ': ' + S('struct-brace', 'list<{') + ' ' + S('field-req', 'optional')) + lines.push(indent(3) + S('field-name', 'key') + ': ' + S('field-type', 'string') + ' ' + S('field-req', 'required')) + lines.push(indent(3) + S('field-name', 'value') + ': ' + S('field-type', hdrValType) + ' ' + S('field-req', 'optional')) + lines.push(indent(2) + S('struct-brace', '}>')) + if (useKeySchema) { + lines.push(indent(2) + S('field-name', 'key') + ': ' + S('struct-brace', '{') + ' ' + S('field-req', 'optional')) + for (var i = 0; i < KEY_SCHEMA.length; i++) { + lines.push(indent(3) + S('field-name', KEY_SCHEMA[i].name) + ': ' + S('field-type', KEY_SCHEMA[i].type) + ' ' + S('field-req', 'optional')) + } + lines.push(indent(2) + S('struct-brace', '}')) + } else { + var keyType = cfg.keyMode === 'string' ? 'string' : 'binary' + lines.push(indent(2) + S('field-name', 'key') + ': ' + S('field-type', keyType) + ' ' + S('field-req', 'optional')) + } + lines.push(indent(2) + S('field-name', 'timestamp_type') + ': ' + S('field-type', 'int') + ' ' + S('field-req', 'optional')) + lines.push(indent(1) + S('struct-brace', '}')) + + if (useValSchema) { + var baseIndent = cfg.valLayout === 'flat' ? 1 : 2 + if (cfg.valLayout === 'flat') { + lines.push(indent(1) + S('comment', '// value fields (flat layout)')) + } else { + lines.push(indent(1) + S('field-name', 'value') + ': ' + S('struct-brace', '{') + ' ' + S('field-req', 'optional')) + } + renderSchemaFields(lines, valSchema, baseIndent, S, indent) + if (cfg.valLayout !== 'flat') lines.push(indent(1) + S('struct-brace', '}')) + } else { + var valType = cfg.valMode === 'string' ? 'string' : 'binary' + lines.push(indent(1) + S('field-name', 'value') + ': ' + S('field-type', valType) + ' ' + S('field-req', 'optional')) + } + lines.push(S('struct-brace', '}')) + return lines.join('\n') + } + + function renderRecord (cfg) { + var lines = [] + var S = function (cls, text) { return '' + esc(text) + '' } + var indent = function (n) { var o = ''; for (var i = 0; i < n; i++) o += ' '; return o } + var useKeySchema = isSchemaMode(cfg.keyMode) + var useValSchema = isSchemaMode(cfg.valMode) + var tsUs = RECORD.timestamp * 1000 + + lines.push(S('struct-brace', '{')) + lines.push(indent(1) + S('field-name', 'redpanda') + ': ' + S('struct-brace', '{')) + lines.push(indent(2) + S('field-name', 'partition') + ': ' + S('field-val-num', '3')) + lines.push(indent(2) + S('field-name', 'offset') + ': ' + S('field-val-num', '42')) + lines.push(indent(2) + S('field-name', 'timestamp') + ': ' + S('field-val-num', String(tsUs))) + lines.push(indent(2) + S('field-name', 'headers') + ': [') + for (var i = 0; i < RECORD.headers.length; i++) { + var h = RECORD.headers[i] + var valCls = cfg.hdrType === 'string' ? 'field-val-str' : 'field-val-bin' + var valPrefix = cfg.hdrType === 'string' ? '' : 'b' + lines.push(indent(3) + S('struct-brace', '{') + ' ' + S('field-name', 'key') + ': ' + S('field-val-str', '"' + h.key + '"') + + ', ' + S('field-name', 'value') + ': ' + S(valCls, valPrefix + '"' + h.value + '"') + ' ' + S('struct-brace', '}')) + } + lines.push(indent(2) + ']') + if (useKeySchema) { + lines.push(indent(2) + S('field-name', 'key') + ': ' + S('struct-brace', '{')) + lines.push(indent(3) + S('field-name', 'user_id') + ': ' + S('field-val-num', '12345')) + lines.push(indent(3) + S('field-name', 'region') + ': ' + S('field-val-str', '"us-west-2"')) + lines.push(indent(2) + S('struct-brace', '}')) + } else if (cfg.keyMode === 'string') { + lines.push(indent(2) + S('field-name', 'key') + ': ' + S('field-val-str', '"\\x00\\x00...us-west-2"')) + } else { + lines.push(indent(2) + S('field-name', 'key') + ': ' + S('field-val-bin', 'b"\\x00\\x00\\x00\\x00\\x01\\xf2\\xc0\\x01\\x12us-west-2"')) + } + lines.push(indent(2) + S('field-name', 'timestamp_type') + ': ' + S('field-val-num', '0')) + lines.push(indent(1) + S('struct-brace', '}')) + if (useValSchema) { + var baseIndent = cfg.valLayout === 'flat' ? 1 : 2 + if (cfg.valLayout !== 'flat') lines.push(indent(1) + S('field-name', 'value') + ': ' + S('struct-brace', '{')) + renderRecordFields(lines, VAL_SCHEMA, RECORD.value, baseIndent, S, indent) + if (cfg.valLayout !== 'flat') lines.push(indent(1) + S('struct-brace', '}')) + } else if (cfg.valMode === 'string') { + lines.push(indent(1) + S('field-name', 'value') + ': ' + S('field-val-str', '"\\x00\\x00...ORD-98765..."')) + } else { + lines.push(indent(1) + S('field-name', 'value') + ': ' + S('field-val-bin', 'b"\\x00\\x00\\x00\\x00\\x02\\x12ORD-98765..."')) + } + lines.push(S('struct-brace', '}')) + return lines.join('\n') + } + + // ── Engine seam. Returns the config string, table schema, and translated + // record for a given config. `valFields`, when provided, are the Iceberg + // value fields computed by the real C++ WASM engine; otherwise the interim + // JS rules (VAL_SCHEMA) are used. + function translate (cfg, valFields) { + return { + config: buildConfigString(cfg), + schema: renderSchema(cfg, valFields), + record: renderRecord(cfg), + } + } + + // ── UI markup injected into each mount point. `nid` is a per-instance id so + // radio `name` attributes are unique — browsers group radios by name + // document-wide, so two explorers on one page would otherwise fight over + // the checked state of same-named groups. + function template (nid) { + var r = function (group, value, checked) { return radio(nid, group, value, checked) } + return '' + + '
' + + '
' + + '

Key configuration

' + + '
Mode
' + + '
' + + r('key-mode', 'binary', true) + r('key-mode', 'string') + r('key-mode', 'schema_id_prefix') + r('key-mode', 'schema_latest') + + '
' + + textRow('key-subject', 'subject', '(default: <topic>-key)') + + textRow('key-proto', 'protobuf_name', '(default: first message)') + + '
' + + '

Value configuration

' + + '
Mode
' + + '
' + + r('val-mode', 'binary', true) + r('val-mode', 'string') + r('val-mode', 'schema_id_prefix') + r('val-mode', 'schema_latest') + + '
' + + '
Layout
' + + '
' + + r('val-layout', 'flat', true) + r('val-layout', 'nested') + + '
' + + textRow('val-subject', 'subject', '(default: <topic>-value)') + + textRow('val-proto', 'protobuf_name', '(default: first message)') + + '
' + + '

Headers configuration

' + + '
Value type
' + + '
' + + r('hdr-type', 'binary', true) + r('hdr-type', 'string') + + '
' + + '
' + + '
' + + '
' + + '
Config string' + + '' + + '' + + '
' + + '

Iceberg table schema 

' +
+        '
Translated record
' +
+      '
' +
+    '
'
+  }
+  // A radio's `name` is namespaced by the instance id (nid); its logical group
+  // is kept in `data-group` so per-instance queries can find it without knowing
+  // the prefix.
+  function radio (nid, group, value, checked) {
+    return ''
+  }
+  var instanceCounter = 0
+  function textRow (id, label, ph) {
+    return '
' +
+      '
'
+  }
+
+  function hydrate (root) {
+    var nid = 'ice' + (++instanceCounter)
+    var engineValFields = null // real-engine value fields, set when WASM loads
+    root.innerHTML = template(nid)
+    var q = function (sel) { return root.querySelector(sel) }
+    // Query radios within this mount by their logical group (data-group),
+    // never by the global name attribute.
+    var radios = function (group) { return root.querySelectorAll('input[data-group="' + group + '"]') }
+    var val = function (group) { var el = root.querySelector('input[data-group="' + group + '"]:checked'); return el ? el.value : '' }
+    var setRadioValue = function (group, value) {
+      var el = root.querySelector('input[data-group="' + group + '"][value="' + value + '"]')
+      if (el) el.checked = true
+    }
+    var field = function (id) { var el = root.querySelector('[data-field="' + id + '"]'); return el ? el.value.trim() : '' }
+
+    function getConfig () {
+      return {
+        keyMode: val('key-mode'), valMode: val('val-mode'), valLayout: val('val-layout'), hdrType: val('hdr-type'),
+        keySubject: field('key-subject'), keyProto: field('key-proto'),
+        valSubject: field('val-subject'), valProto: field('val-proto'),
+      }
+    }
+
+    // Apply an author-supplied initial config string (data-config) by mapping
+    // its sections back onto the controls. Best-effort; unknown tokens ignored.
+    function applyInitialConfig (str) {
+      if (!str) return
+      if (str.indexOf(':') === -1) {
+        // legacy string
+        if (str === 'key_value') { setRadioValue('val-mode', 'binary') }
+        else if (str === 'value_schema_id_prefix') { setRadioValue('val-mode', 'schema_id_prefix') }
+        else if (str.indexOf('value_schema_latest') === 0) { setRadioValue('val-mode', 'schema_latest') }
+        return
+      }
+      // Map DSL section names to control-group prefixes. The value section is
+      // named "value" in the DSL but its controls use the "val" prefix.
+      var prefixFor = { key: 'key', value: 'val' }
+      str.split(';').forEach(function (section) {
+        var parts = section.split(':')
+        var name = parts[0]
+        var prefix = prefixFor[name]
+        var opts = (parts[1] || '').split(',')
+        opts.forEach(function (opt) {
+          var kv = opt.split('=')
+          if (kv[0] === 'mode' && prefix) setRadioValue(prefix + '-mode', kv[1])
+          if (kv[0] === 'layout') setRadioValue('val-layout', kv[1])
+          if (name === 'headers' && kv[0] === 'value_type') setRadioValue('hdr-type', kv[1])
+        })
+      })
+    }
+
+    function update () {
+      var cfg = getConfig()
+      // Enable subject/protobuf inputs only for schema_latest.
+      ;['key', 'val'].forEach(function (p) {
+        var mode = p === 'key' ? cfg.keyMode : cfg.valMode
+        var on = mode === 'schema_latest'
+        var sub = root.querySelector('[data-field="' + p + '-subject"]')
+        var pr = root.querySelector('[data-field="' + p + '-proto"]')
+        if (sub) sub.disabled = !on
+        if (pr) pr.disabled = !on
+      })
+      // Layout only applies to schema modes.
+      var valIsSchema = isSchemaMode(cfg.valMode)
+      radios('val-layout').forEach(function (r) {
+        r.disabled = !valIsSchema
+        r.closest('.ice-radio').style.opacity = valIsSchema ? '1' : '0.4'
+      })
+      if (!valIsSchema) setRadioValue('val-layout', 'flat')
+
+      // Use the real WASM engine's value fields when loaded and the value is
+      // decoded via a schema; otherwise fall back to the interim JS rules.
+      var useEngine = engineValFields && isSchemaMode(cfg.valMode)
+      var out = translate(cfg, useEngine ? engineValFields : null)
+      var cfgEl = q('[data-out="config"]')
+      cfgEl.innerHTML = esc(out.config.str) +
+        (out.config.legacy ? ' legacy format' : ' new format')
+      q('[data-out="schema"]').innerHTML = out.schema
+      q('[data-out="record"]').innerHTML = out.record
+
+      // Engine provenance badge on the schema card.
+      var engEl = q('[data-out="engine"]')
+      if (engEl) {
+        if (useEngine) {
+          engEl.className = 'ice-tag ice-engine-real'
+          engEl.textContent = 'real engine · wasm'
+        } else if (engineValFields && !isSchemaMode(cfg.valMode)) {
+          engEl.className = 'ice-tag ice-engine-real'
+          engEl.textContent = 'real engine · wasm ready'
+        } else {
+          engEl.className = 'ice-tag ice-engine-preview'
+          engEl.textContent = 'preview engine'
+        }
+      }
+    }
+
+    root.querySelectorAll('input[type="radio"]').forEach(function (r) { r.addEventListener('change', update) })
+    root.querySelectorAll('input[type="text"]').forEach(function (i) { i.addEventListener('input', update) })
+
+    // Load the real WASM engine (shared singleton) for this doc version;
+    // re-render when its value fields are available so the schema panel
+    // reflects production behavior. Falls back to the interim JS engine.
+    var docVersion = root.getAttribute('data-version') ||
+      (document.querySelector('.nav-container') && document.querySelector('.nav-container').getAttribute('data-version')) ||
+      (document.body && document.body.getAttribute('data-version')) || ''
+    loadEngineValueFields(docVersion, root.getAttribute('data-engine-base')).then(function (fields) {
+      if (fields) { engineValFields = fields; update() }
+    })
+
+    var copyBtn = q('[data-copy]')
+    if (copyBtn) {
+      copyBtn.addEventListener('click', function () {
+        var text = q('[data-out="config"]').textContent.replace(/\s*(legacy|new) format$/, '')
+        var done = function () { copyBtn.textContent = 'Copied!'; copyBtn.classList.add('ice-copied'); setTimeout(function () { copyBtn.textContent = 'Copy'; copyBtn.classList.remove('ice-copied') }, 1500) }
+        if (navigator.clipboard && navigator.clipboard.writeText) { navigator.clipboard.writeText(text).then(done, function () {}) } else { done() }
+      })
+    }
+
+    // Author-supplied defaults: data-config (string) or data-defaults (JSON).
+    var initial = root.getAttribute('data-config')
+    if (!initial && root.getAttribute('data-defaults')) {
+      try { initial = (JSON.parse(root.getAttribute('data-defaults')) || {}).config } catch (e) {}
+    }
+    applyInitialConfig(initial)
+    update()
+  }
+
+  function init () {
+    var mounts = document.querySelectorAll('.iceberg-explorer')
+    for (var i = 0; i < mounts.length; i++) {
+      if (!mounts[i].getAttribute('data-hydrated')) {
+        mounts[i].setAttribute('data-hydrated', 'true')
+        hydrate(mounts[i])
+      }
+    }
+  }
+
+  // DOM bootstrap only in a browser. Guarded so the module can be required in
+  // Node for the config-string conformance test (see tests/iceberg-dsl).
+  if (typeof document !== 'undefined') {
+    if (document.readyState === 'loading') {
+      document.addEventListener('DOMContentLoaded', init)
+    } else {
+      init()
+    }
+  }
+
+  // Export the pure DSL logic for conformance testing under Node. No effect in
+  // the browser (module is undefined there).
+  if (typeof module !== 'undefined' && module.exports) {
+    module.exports = { buildConfigString: buildConfigString, isSchemaMode: isSchemaMode }
+  }
+})()
diff --git a/src/layouts/default.hbs b/src/layouts/default.hbs
index 42fc0149..d59b94bf 100644
--- a/src/layouts/default.hbs
+++ b/src/layouts/default.hbs
@@ -4,7 +4,7 @@
 {{> head defaultPageTitle='Untitled'}}
 {{> head-component-color}}
   
-  
+  
 {{> header}}
 {{> body}}
   
diff --git a/src/ui.yml b/src/ui.yml
index 0d606a04..d6382c9a 100644
--- a/src/ui.yml
+++ b/src/ui.yml
@@ -1,6 +1,8 @@
 static_files:
 - blobl.wasm
 - console-config-migrator.wasm
+- iceberg-engine.wasm
+- iceberg-engine.js
 - ask-ai.html
 - redpanda-properties.json
 - assets/**
diff --git a/tests/iceberg-dsl/conformance-test.js b/tests/iceberg-dsl/conformance-test.js
new file mode 100644
index 00000000..001b179f
--- /dev/null
+++ b/tests/iceberg-dsl/conformance-test.js
@@ -0,0 +1,71 @@
+#!/usr/bin/env node
+'use strict'
+
+// Conformance test for the Iceberg Mode Explorer's config-string builder.
+//
+// The DSL string is produced by pure JS in src/js/27-iceberg-explorer.js
+// (buildConfigString). This test pins that JS to Redpanda's OWN format
+// expectations, copied verbatim from the authoritative C++ unit test
+// src/v/model/tests/iceberg_mode_test.cc (the IcebergModeFormat.* cases). If a
+// future Redpanda release changes the DSL serialization, re-syncing these
+// vectors from that test file (and updating the JS to match) is the single
+// maintenance step — that is the "tracks releases" guarantee for the DSL,
+// without needing to compile model.cc to WASM.
+//
+// CI can additionally fetch iceberg_mode_test.cc from a given redpanda ref and
+// diff the expected strings to detect upstream format changes automatically.
+
+const { buildConfigString } = require('../../src/js/27-iceberg-explorer.js')
+
+// cfg shape matches getConfig() in the module.
+function cfg (o) {
+  return Object.assign({
+    keyMode: 'binary', valMode: 'binary', valLayout: 'flat', hdrType: 'binary',
+    keySubject: '', keyProto: '', valSubject: '', valProto: '',
+  }, o)
+}
+
+// Each vector cites the corresponding C++ IcebergModeFormat test case.
+const VECTORS = [
+  // --- legacy strings (backward-compatible serialization) ---
+  { name: 'KeyValue',
+    cfg: cfg({}),
+    expect: 'key_value' },
+  { name: 'ValueSchemaIdPrefix',
+    cfg: cfg({ valMode: 'schema_id_prefix' }),
+    expect: 'value_schema_id_prefix' },
+  { name: 'ValueSchemaLatestBare',
+    cfg: cfg({ valMode: 'schema_latest' }),
+    expect: 'value_schema_latest' },
+  { name: 'ValueSchemaLatestWithSubject',
+    cfg: cfg({ valMode: 'schema_latest', valSubject: 'my-topic-value' }),
+    expect: 'value_schema_latest:subject=my-topic-value' },
+  { name: 'ValueSchemaLatestWithProtobuf',
+    cfg: cfg({ valMode: 'schema_latest', valProto: 'com.example.Msg', valSubject: 'my-topic-value' }),
+    expect: 'value_schema_latest:protobuf_name=com.example.Msg,subject=my-topic-value' },
+  // --- section-based format (new): all sections + options always emitted ---
+  { name: 'NewKeySchema',
+    cfg: cfg({ keyMode: 'schema_id_prefix' }),
+    expect: 'key:mode=schema_id_prefix;value:mode=binary,layout=flat;headers:value_type=binary' },
+  { name: 'NewHeadersString',
+    cfg: cfg({ hdrType: 'string' }),
+    expect: 'key:mode=binary;value:mode=binary,layout=flat;headers:value_type=string' },
+]
+
+let failures = 0
+for (const v of VECTORS) {
+  const got = buildConfigString(v.cfg).str
+  const ok = got === v.expect
+  if (!ok) failures++
+  console.log(`${ok ? 'PASS' : 'FAIL'}  ${v.name}`)
+  if (!ok) {
+    console.log(`      expected: ${v.expect}`)
+    console.log(`      got:      ${got}`)
+  }
+}
+
+console.log(`\n${VECTORS.length - failures}/${VECTORS.length} conformance vectors passed`)
+if (failures) {
+  console.error('\nConfig-string DSL has drifted from iceberg_mode_test.cc vectors.')
+  process.exit(1)
+}