feat: interactive Iceberg Mode Explorer with real C++→WASM engine

.github/workflows/build-iceberg-engine.yml

@@ -0,0 +1,100 @@
+name: Build Iceberg engine (WASM)
+
+# Builds the Iceberg Mode Explorer's engine — Redpanda's real C++ Avro->Iceberg
+# schema mapper compiled to WebAssembly — for a specific Redpanda release, so
+# each docs major.minor loads the engine built from that release's source.
+#
+# This is the analog of the Bloblang playground's update-go-modules workflow,
+# keyed on Redpanda release tags instead of Go module bumps. It produces
+# iceberg-engine-<major.minor>.js/.wasm as build artifacts; publish those to the
+# versioned docs content repo under
+# modules/<module>/assets/attachments/ so the tool loads the version-matched
+# engine at runtime (see src/js/27-iceberg-explorer.js loader).
+
+on:
+  workflow_dispatch:
+    inputs:
+      redpanda_ref:
+        description: 'Redpanda git ref/tag to build the engine from (e.g. v26.2.1 or dev)'
+        required: true
+        default: 'dev'
+  # Trigger from the redpanda repo on a new release via repository_dispatch:
+  #   curl -X POST .../dispatches -d '{"event_type":"redpanda-release","client_payload":{"ref":"v26.2.1"}}'
+  repository_dispatch:
+    types: [redpanda-release]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Resolve Redpanda ref and docs version
+        id: ver
+        run: |
+          REF="${{ github.event.inputs.redpanda_ref || github.event.client_payload.ref }}"
+          echo "ref=$REF" >> "$GITHUB_OUTPUT"
+          # major.minor from a vX.Y.Z tag; 'dev' stays 'dev'.
+          if [[ "$REF" =~ ^v?([0-9]+)\.([0-9]+) ]]; then
+            echo "version=${BASH_REMATCH[1]}.${BASH_REMATCH[2]}" >> "$GITHUB_OUTPUT"
+          else
+            echo "version=$REF" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Set up Emscripten
+        uses: mymindstorm/setup-emsdk@v14
+        with:
+          version: 6.0.3
+
+      - name: Fetch Redpanda source + third-party deps
+        working-directory: iceberg-editor/wasm-spike
+        env:
+          REDPANDA_REF: ${{ steps.ver.outputs.ref }}
+        run: |
+          ./fetch-sources.sh
+          ./third_party/fetch-deps.sh
+
+      - name: Build engine (web target)
+        working-directory: iceberg-editor/wasm
+        run: ./build.sh web
+
+      - name: Stage versioned artifact
+        id: stage
+        run: |
+          V="${{ steps.ver.outputs.version }}"
+          mkdir -p dist
+          cp src/static/iceberg-engine.js  "dist/iceberg-engine-$V.js"
+          cp src/static/iceberg-engine.wasm "dist/iceberg-engine-$V.wasm"
+          # The emscripten glue references the .wasm by its build name; rewrite
+          # it to the versioned name so both files can coexist per version.
+          sed -i "s/iceberg-engine\.wasm/iceberg-engine-$V.wasm/g" "dist/iceberg-engine-$V.js"
+          ls -la dist
+
+      - name: Smoke-test the built engine under Node
+        working-directory: iceberg-editor/wasm
+        run: |
+          ./build.sh node
+          node - <<'EOF'
+          const create = require('./build/iceberg-engine-node.js')
+          create().then(m => {
+            const out = m.avroToIcebergJson(JSON.stringify({
+              type:'record', name:'t', fields:[{name:'a',type:'string'}]
+            }))
+            const p = JSON.parse(out)
+            if (!p.fields || p.fields[0].type !== 'string') { console.error('FAIL', out); process.exit(1) }
+            console.log('engine smoke test OK:', out)
+          })
+          EOF
+
+      - name: Upload versioned engine artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: iceberg-engine-${{ steps.ver.outputs.version }}
+          path: dist/
+          if-no-files-found: error
+
+      # Publishing step (commented): open a PR to the versioned content repo
+      # placing dist/* under modules/<module>/assets/attachments/ for the
+      # matching docs version. Wire this to your content-repo bot token.
+      # - name: Publish to content repo
+      #   run: ./ci/publish-iceberg-engine.sh "${{ steps.ver.outputs.version }}" dist/

.github/workflows/iceberg-dsl-conformance.yml

@@ -0,0 +1,65 @@
+name: Iceberg DSL conformance
+
+# Verifies the Iceberg Mode Explorer's config-string builder (pure JS in
+# src/js/27-iceberg-explorer.js) still matches Redpanda's authoritative DSL
+# serialization, whose expectations live in
+# src/v/model/tests/iceberg_mode_test.cc in the redpanda repo.
+#
+# The unit test (tests/iceberg-dsl/conformance-test.js) always runs. On a
+# schedule / manual dispatch it also fetches the current C++ test file from a
+# chosen redpanda ref and greps for the expected format strings, so upstream
+# DSL changes surface as a failing check even before anyone updates the JS.
+
+on:
+  pull_request:
+    paths:
+      - 'src/js/27-iceberg-explorer.js'
+      - 'tests/iceberg-dsl/**'
+  workflow_dispatch:
+    inputs:
+      redpanda_ref:
+        description: 'redpanda ref to diff the DSL vectors against'
+        required: false
+        default: 'dev'
+  schedule:
+    - cron: '0 6 * * 1' # weekly: catch upstream DSL drift
+
+jobs:
+  conformance:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+
+      - name: Run DSL conformance vectors
+        run: npm run test:iceberg-dsl
+
+      - name: Cross-check vectors against upstream iceberg_mode_test.cc
+        if: github.event_name != 'pull_request'
+        env:
+          REF: ${{ github.event.inputs.redpanda_ref || 'dev' }}
+        run: |
+          set -euo pipefail
+          url="https://raw.githubusercontent.com/redpanda-data/redpanda/$REF/src/v/model/tests/iceberg_mode_test.cc"
+          curl -fsSL "$url" -o upstream_iceberg_mode_test.cc
+          # Merge C++ adjacent string-literal concatenation ("a" "b" -> "ab"),
+          # including across line breaks, so long expected strings that the test
+          # file wraps still match.
+          norm="$(tr -d '\n' < upstream_iceberg_mode_test.cc | sed 's/"[[:space:]]*"//g')"
+          # Every expected string in our conformance vectors must still appear
+          # verbatim in the upstream C++ format tests.
+          missing=0
+          while IFS= read -r expected; do
+            [ -z "$expected" ] && continue
+            if ! printf '%s' "$norm" | grep -qF -- "$expected"; then
+              echo "::warning::Expected DSL string not found upstream ($REF): $expected"
+              missing=$((missing+1))
+            fi
+          done < <(grep -oE "expect: '[^']+'" tests/iceberg-dsl/conformance-test.js | sed "s/expect: '//; s/'$//")
+          if [ "$missing" -gt 0 ]; then
+            echo "$missing expected DSL string(s) no longer present upstream — the DSL may have changed; re-sync tests/iceberg-dsl vectors and update buildConfigString."
+            exit 1
+          fi
+          echo "All DSL vectors still present in upstream iceberg_mode_test.cc @ $REF"

iceberg-editor/README.md

@@ -0,0 +1,28 @@
+# iceberg-editor
+
+WASM engine tooling for the interactive **Iceberg Mode Explorer** docs tool,
+mirroring the layout of `blobl-editor/` (the Bloblang playground's Go→WASM
+engine).
+
+Unlike Bloblang — whose engine is Go and wraps `benthos` directly — Iceberg
+mode translation is implemented in **C++** in the Redpanda core
+(`redpanda/src/v/iceberg/conversion/` + `src/v/datalake/`). To keep the docs
+tool faithful to production behavior (and to stay current automatically, one
+build per `major.minor` release), the engine is the **real C++ translation code
+compiled to WebAssembly with Emscripten**, not a re-implementation.
+
+## Status
+
+- `wasm-spike/` — a **feasibility spike** that proves the hardest dependency
+  (Apache Avro C++ + a thin Seastar shim + the `iceberg/conversion` schema
+  mapper) compiles and links under Emscripten. Run this **before** investing in
+  the full engine module. See `wasm-spike/README.md`.
+
+## Why a spike first
+
+The schema/value mappers are synchronous and free of Seastar's reactor, but the
+shared Iceberg type model (`iceberg/datatypes.h`) transitively includes a few
+Seastar utility types (`ss::sstring`, `chunked_vector`) and Redpanda base utils.
+The spike confirms those can be shimmed and that Avro C++ builds under
+Emscripten. If the spike passes, the remaining work (value mapper, DSL, JSON
+bindings, per-release CI) is mechanical.

iceberg-editor/wasm-spike/.gitignore

@@ -0,0 +1,11 @@
+# Everything fetched or built is reproducible from the scripts — never commit it.
+
+# Fetched Redpanda source (fetch-sources.sh, pinned to $REDPANDA_REF)
+vendor/
+
+# Downloaded third-party deps (fmt, boost, avro fork) — keep only the fetcher
+third_party/*
+!third_party/fetch-deps.sh
+
+# Build outputs (objects, wasm, js glue)
+build/

iceberg-editor/wasm-spike/README.md

@@ -0,0 +1,128 @@
+# Iceberg engine — Emscripten feasibility spike
+
+**Goal:** prove that Redpanda's real C++ Iceberg **schema** mapper
+(`iceberg::type_to_iceberg`) compiles and links to WebAssembly under Emscripten,
+against an Emscripten build of **Avro C++**, using a thin **Seastar shim** in
+place of the reactor. If this passes, the full C++→WASM engine (value mapper,
+DSL parser, JSON bindings, per-release CI) is mechanical.
+
+## ✅ RESULT: PASS (verified 2026-07-22)
+
+The real production mapper runs in wasm. `stage2-schema-avro.sh` prints:
+
+```
+OK: converted Avro record to iceberg::struct_type with 3 field(s)
+```
+
+**Verified toolchain (the coherent combo that works — versions matter):**
+
+| Component | Version / ref | Note |
+| --- | --- | --- |
+| Emscripten | 6.0.3 | `brew install emscripten` |
+| fmt | **12.1.0** | matches redpanda `MODULE.bazel`; fmt 11 fails |
+| Avro C++ | **redpanda-data/avro** @ `6821e2b4…` + `avro-fmt-const.patch` | upstream apache/avro 1.12 does **not** compile under fmt 12 |
+| Boost | 1.85.0 (headers) | for `boost::outcome` |
+| Redpanda source | `dev` | `iceberg/datatypes.cc`, `conversion/{avro_utils,schema_avro}.cc` |
+
+**What the shim had to provide** (all small, in `shim/`): `ss::sstring`→
+`std::string`, `ss::chunked_vector`→`std::vector`, `ss::bool_class` (+ its fmt
+formatter), `ss::visit`/`make_visitor`, `ssx::sformat`, no-op `ss::logger`,
+`ss::defer`, `vassert`, an fmt formatter for `std::exception_ptr`, and a
+schema-only `conversion_outcome.h` that drops the `values.h` (iobuf/absl)
+include the schema path doesn't need.
+
+**Green light** to build the full engine. Remaining work is mechanical and
+additive: the **value** mapper (`values_avro.cc` → pulls in `values.h` →
+`iobuf`/`absl`, the next dependency to size), the **Protobuf** path
+(`schema_protobuf.cc` + libprotobuf-wasm), the tiny **DSL** parser
+(`model/model.cc`, spec'd by `model/tests/iceberg_mode_test.cc`), Emscripten
+**bindings** for `iceberg_translate(config, schema, record) → JSON`, and the
+**per-release CI** that emits `iceberg-engine-<major.minor>.wasm`.
+
+This directory contains **no compiled artifacts** and **no vendored Redpanda
+source** — real source is fetched from a pinned `redpanda` ref at run time
+(`fetch-sources.sh`), which also mirrors the per-release build model. Nothing
+here has been compiled in-repo; it is meant to be run where `emcc` is available.
+
+## Prerequisites
+
+- [emsdk / Emscripten](https://emscripten.org/docs/getting_started/downloads.html)
+  (`emcc`, `emcmake`, `em++`) on `PATH`.
+- `cmake`, `git`, `curl`, and `gh` (for fetching pinned Redpanda source; falls
+  back to raw HTTPS if `gh` is unavailable).
+- `node` (to run the resulting `.wasm`).
+
+## What gets proven, in two stages
+
+The spike is deliberately staged so a cheap, high-confidence check runs before
+the expensive external-library step.
+
+### Stage 1 — `stage1-datatypes.sh` (no external libraries)
+
+Compiles `iceberg/datatypes.cc` to a wasm object with `emcc -c`, using only:
+`fmt` (header-only), the Seastar shim (`shim/`), `named_type.h`, and a `vassert`
+shim. **Proves** the Iceberg type model + Redpanda base utils compile under
+Emscripten with the shim. If this fails, the shim is wrong — fix it here before
+touching Avro.
+
+### Stage 2 — `stage2-schema-avro.sh` (the real unknown)
+
+Builds Apache Avro C++ for wasm (`build-avro-cpp.sh`), then compiles
+`datatypes.cc` + `schema_avro.cc` + `poc_main.cc` and links them against the
+Avro C++ wasm library into `build/iceberg-schema-poc.wasm`. `poc_main.cc`
+parses a small Avro schema and calls `iceberg::type_to_iceberg`, so a successful
+`node build/iceberg-schema-poc.js` run proves the whole path works end to end.
+
+To sever the schema mapper from `iobuf`/`absl` (which are only pulled in
+transitively via `conversion_outcome.h` → `values.h`, and are needed only by the
+*value* mapper), the spike puts a **schema-only** `conversion_outcome.h` on the
+include path (`shim/iceberg/conversion/conversion_outcome.h`) that keeps
+`result<>` and `conversion_exception` but drops the `values.h` include. This is
+a legitimate simplification and documents that the schema and value paths have
+different dependency footprints.
+
+## Run it
+
+```bash
+export REDPANDA_REF=dev          # or a release tag, e.g. v26.2.1
+./fetch-sources.sh               # pulls real source into vendor/ at $REDPANDA_REF
+./stage1-datatypes.sh            # cheap proof (no Avro)
+./build-avro-cpp.sh              # builds Avro C++ for wasm (slow, one-time)
+./stage2-schema-avro.sh          # the real proof; runs the wasm via node
+```
+
+## Interpreting results (this is the decision the spike exists to inform)
+
+- **Stage 1 fails to compile** → the Seastar/base shim is incomplete. Extend
+  `shim/` until `datatypes.cc` compiles. Low risk; expected to be quick.
+- **`build-avro-cpp.sh` fails** → Avro C++ (or its deps: Boost, Snappy) doesn't
+  build cleanly under Emscripten. This is the biggest risk. If it's only
+  optional features (codecs), disable them (`-DAVRO_*`), since the docs tool only
+  needs schema parsing, not container files/compression.
+- **Stage 2 compiles + `node` prints `OK`** → **green light.** The real
+  translation code runs in wasm. Proceed to the full engine module: add the
+  value mapper (`values_avro.cc`, which pulls in `values.h` → `iobuf`/`absl`; the
+  next dependency to size), the Protobuf path, the DSL parser (tiny; specified by
+  `model/tests/iceberg_mode_test.cc`), Emscripten bindings for
+  `iceberg_translate(config, schema, record) → JSON`, and the release-tag CI.
+
+## Files
+
+| Path | Purpose |
+| --- | --- |
+| `fetch-sources.sh` | Fetch real Redpanda source at `$REDPANDA_REF` into `vendor/` |
+| `shim/base/seastarx.h` | `ss` = `seastar` namespace alias (matches real) |
+| `shim/base/vassert.h` | `vassert(...)` → `assert`/`abort` |
+| `shim/seastar/core/sstring.hh` | `ss::sstring` → `std::string` |
+| `shim/seastar/core/chunked_vector.hh` | `ss::chunked_vector` → `std::vector` |
+| `shim/seastar/util/bool_class.hh` | `ss::bool_class<Tag>` strong-typedef bool |
+| `shim/seastar/util/defer.hh` | Minimal `ss::defer` |
+| `shim/seastar/util/log.hh` | No-op `ss::logger` |
+| `shim/iceberg/conversion/conversion_outcome.h` | Schema-only outcome (no `values.h`) |
+| `poc_main.cc` | Parses an Avro schema, calls `type_to_iceberg` |
+| `stage1-datatypes.sh` | Compile-only proof of the shim closure |
+| `build-avro-cpp.sh` | Build Apache Avro C++ for wasm |
+| `stage2-schema-avro.sh` | Compile + link + run the schema-mapper wasm |
+| `third_party/` | Downloaded deps (fmt, boost.outcome, avro-cpp); git-ignored |
+| `vendor/` | Fetched Redpanda source; git-ignored |
+| `build/` | Build outputs; git-ignored |

iceberg-editor/wasm-spike/build-avro-cpp.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+# Compile Apache Avro C++ (Redpanda's fork) to WebAssembly objects with emcc.
+#
+# We compile the sources directly rather than via CMake: the docs tool only
+# needs schema PARSING (compileJsonSchemaFromString + the Node/Schema API), so
+# we skip the codegen CLI (avrogencpp.cc) and container-file I/O (DataFile.cc,
+# which needs Snappy/zlib codecs). This proved simpler and more portable under
+# Emscripten than driving Avro's CMake.
+#
+# Objects land in $BUILD_DIR/wasm-obj/ and are linked by stage2-schema-avro.sh.
+set -euo pipefail
+source "$(dirname "${BASH_SOURCE[0]}")/env.sh"
+
+require emcc
+[ -d "$AVRO_SRC/include/avro" ] || {
+    echo "Avro fork missing. Run ./third_party/fetch-deps.sh first." >&2; exit 1; }
+[ -d "$FMT_INCLUDE" ] || {
+    echo "fmt missing. Run ./third_party/fetch-deps.sh first." >&2; exit 1; }
+[ -d "$BOOST_INCLUDE/boost" ] || {
+    echo "boost missing. Run ./third_party/fetch-deps.sh first." >&2; exit 1; }
+
+OBJ="$BUILD_DIR/wasm-obj"
+mkdir -p "$OBJ"
+
+# Avro's own .cc files include their headers unqualified (e.g. "Decoder.hh"),
+# so both include/ and include/avro/ must be on the path.
+AVRO_INCS=(-I "$FMT_INCLUDE" -I "$BOOST_INCLUDE"
+           -I "$AVRO_SRC/include" -I "$AVRO_SRC/include/avro")
+
+echo "Compiling Avro C++ (fork) sources to wasm objects ..."
+n=0
+for f in "$AVRO_SRC"/impl/*.cc "$AVRO_SRC"/impl/parsing/*.cc \
+         "$AVRO_SRC"/impl/json/*.cc; do
+    base="$(basename "$f")"
+    # Skip the codegen CLI (own main) and container-file I/O (codec deps).
+    case "$base" in avrogencpp.cc | DataFile.cc) continue ;; esac
+    emcc -std=c++20 -c -DFMT_HEADER_ONLY=1 "${AVRO_INCS[@]}" \
+        "$f" -o "$OBJ/${base%.cc}.o"
+    n=$((n + 1))
+done
+echo "PASS: compiled $n Avro C++ sources to $OBJ/"