From ed36ac46fc7e3ca15bd9ce4205f9dd7a0985f1d0 Mon Sep 17 00:00:00 2001 From: Rafael Richards Date: Sat, 30 May 2026 21:19:09 -0400 Subject: [PATCH 1/2] callouts: build + bake m-stdlib YDB call-out .so's into the engine image MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit m-stdlib's three optional modules (STDCOMPRESS / STDCRYPTO / STDHTTP) reach native code on YottaDB via $&pkg.fn -> a .so registered through a .xc descriptor. The running container had no such libraries, so the optional suites couldn't run — and with a .xc descriptor present but its .so missing, STDCOMPRESSTST hung the engine. This bakes the libraries in and wires them. How: - New multi-stage Dockerfile `callout-builder` stage: installs gcc + the -dev headers (zlib1g-dev, libzstd-dev, libssl-dev, libcurl4-openssl-dev), compiles m-stdlib's src/callouts/*.c via tools/build-callouts.sh. The toolchain lives ONLY in this stage; the final image carries just the .so's and the already-present runtime libs (libz 1.3 / libzstd 1.5.5 / libcrypto 3 / libcurl 4.8.0) — staying minimal. - Final stage: COPY the compiled .so's -> /opt/stdlib/lib, the .xc descriptors -> /opt/stdlib/xc, and export STDLIB_LIB + ydb_xc_stdcompress / _stdcrypto / _stdhttp via /etc/profile.d/stdlib-callouts.sh (sourced by `bash -lc`, the m-cli DockerEngine transport). A build-time assert fails the image if any of the three .so's is absent — enforcing the hang-guard invariant (never a descriptor without its library). - Makefile: `callouts-stage` copies the C sources + .xc from m-stdlib into the build context (gitignored docker/_callouts/ — single-sourced in m-stdlib, never vendored here); `up` depends on it. New `test-optional` runs the four callout-backed suites in byte mode. - dist/: lifecycle.json gains the callout env vars + a stdlib_callouts block + the test-optional target; verified_on bumped to 2026-05-30 (re-verified with the new image). Verification (via `m test`, no hand docker exec): - make smoke: healthy (mumps ok, OCI labels, healthcheck, mte status). - make test-optional (YDB byte mode, --chset m): 4 suites, 151 assertions, 0 failed, no hang — STDCRYPTOTST 23, STDCRYPTODOCTST 1, STDCOMPRESSTST 59, STDHTTPTST 68. The deployed .so's resolve the Stage-A STDCOMPRESS hang (which was the missing-.so case). Co-Authored-By: Claude Opus 4.8 (1M context) --- .gitignore | 4 ++++ Makefile | 36 +++++++++++++++++++++++++++++++++-- dist/lifecycle.json | 18 +++++++++++++++--- dist/m-test-engine.json | 2 +- dist/repo.meta.json | 2 +- docker/Dockerfile | 42 +++++++++++++++++++++++++++++++++++++++++ 6 files changed, 97 insertions(+), 7 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3cc24d5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +# Callout sources staged from m-stdlib at build time (single-sourced there, +# never vendored here). Populated by `make callouts-stage`, consumed by the +# Dockerfile's callout-builder stage. +docker/_callouts/ diff --git a/Makefile b/Makefile index 09dfb73..4aba009 100644 --- a/Makefile +++ b/Makefile @@ -17,15 +17,47 @@ # But run-from-here works too: cd into m-test-engine and `make smoke` # for a quick check. -.PHONY: up down logs shell smoke clean manifest check-manifest check-docs-prose skill-install +.PHONY: up down logs shell smoke clean manifest check-manifest check-docs-prose skill-install callouts-stage test-optional COMPOSE := docker compose -f docker/compose.yml -up: +# m-stdlib checkout that owns the callout C sources + .xc descriptors. They are +# single-sourced there and staged into the build context at build time (never +# vendored into this repo — see docker/_callouts/ in .gitignore). Override if +# your m-stdlib lives elsewhere. +M_STDLIB ?= $(HOME)/vista-cloud-dev/m-stdlib +CALLOUT_DIR := docker/_callouts + +# callouts-stage — copy m-stdlib's callout build inputs into the Docker build +# context so the callout-builder image stage can compile them. Idempotent; +# fails loudly if m-stdlib (or its callout sources) can't be found. +callouts-stage: + @if [ ! -d "$(M_STDLIB)/src/callouts" ]; then \ + echo "ERROR: m-stdlib callout sources not found at $(M_STDLIB)/src/callouts" >&2; \ + echo " set M_STDLIB=/path/to/m-stdlib" >&2; exit 1; \ + fi + @rm -rf $(CALLOUT_DIR) + @mkdir -p $(CALLOUT_DIR)/src/callouts $(CALLOUT_DIR)/tools $(CALLOUT_DIR)/xc + @cp $(M_STDLIB)/src/callouts/*.c $(CALLOUT_DIR)/src/callouts/ + @cp $(M_STDLIB)/tools/build-callouts.sh $(CALLOUT_DIR)/tools/ + @cp $(M_STDLIB)/tools/std_compress.xc $(M_STDLIB)/tools/std_crypto.xc $(M_STDLIB)/tools/std_http.xc $(CALLOUT_DIR)/xc/ + @echo "callouts-stage: staged $$(ls $(CALLOUT_DIR)/src/callouts/*.c | wc -l) C source(s) + 3 .xc from $(M_STDLIB)" + +up: callouts-stage $(COMPOSE) up -d --build @echo @echo "m-test-engine is up. Verify with: make smoke" +# test-optional — run m-stdlib's callout-backed suites against this engine in +# byte (M) mode (the contract for the byte-oriented modules). Proves the baked +# callouts resolve with no hang. Drives the Go `m` binary directly because the +# byte suites need --chset m, which m-stdlib's own `make test-optional` does +# not yet pass. +M ?= $(HOME)/vista-cloud-dev/m-cli/dist/m +test-optional: + cd $(M_STDLIB) && $(M) test tests/STDCRYPTOTST.m tests/STDCRYPTODOCTST.m tests/STDCOMPRESSTST.m tests/STDHTTPTST.m \ + --engine ydb --docker m-test-engine --routines=src --chset m + down: $(COMPOSE) down diff --git a/dist/lifecycle.json b/dist/lifecycle.json index c5e0a55..2afccd5 100644 --- a/dist/lifecycle.json +++ b/dist/lifecycle.json @@ -16,14 +16,26 @@ "logs": { "make": "make logs", "compose": "docker compose -f docker/compose.yml logs -f" }, "shell": { "make": "make shell", "compose": "docker exec -it m-test-engine bash" }, "smoke": { "make": "make smoke", "compose": "docker exec m-test-engine bash -lc '$ydb_dist/mumps -run %XCMD '\\''write \"smoke ok\",!'\\'''" }, - "clean": { "make": "make clean", "compose": "docker compose -f docker/compose.yml down -v" } + "clean": { "make": "make clean", "compose": "docker compose -f docker/compose.yml down -v" }, + "test-optional": { "make": "make test-optional", "compose": "n/a — drives the Go `m` binary against this container in byte mode" } }, "exec_convention": { "shape": "docker exec m-test-engine bash -lc ''", - "ydb_env_loaded_via": "/etc/profile.d/ydb-env.sh (sources /opt/yottadb/current/ydb_env_set)", - "available_env_vars": ["ydb_dist", "ydb_routines"], + "ydb_env_loaded_via": "/etc/profile.d/ydb-env.sh (sources /opt/yottadb/current/ydb_env_set); /etc/profile.d/stdlib-callouts.sh (m-stdlib callout vars)", + "available_env_vars": ["ydb_dist", "ydb_routines", "STDLIB_LIB", "ydb_xc_stdcompress", "ydb_xc_stdcrypto", "ydb_xc_stdhttp"], "notes": "Use bash -lc so the YDB env loads. Single-quote M commands so the outer bash does not expand $ZVERSION etc. (those are YDB special variables, not shell vars)." }, + "stdlib_callouts": { + "provides": "m-stdlib optional-module YDB call-out libraries, baked at image build from m-stdlib's src/callouts/*.c via tools/build-callouts.sh (single-sourced in m-stdlib; staged into the build context by `make callouts-stage`, never vendored here).", + "lib_dir": "/opt/stdlib/lib", + "xc_dir": "/opt/stdlib/xc", + "libraries": { + "stdcompress.so": { "package": "stdcompress", "xc": "std_compress.xc", "links": ["libz.so.1 (1.3)", "libzstd.so.1 (1.5.5)"] }, + "std_crypto.so": { "package": "stdcrypto", "xc": "std_crypto.xc", "links": ["libcrypto.so.3 (OpenSSL 3)"] }, + "http.so": { "package": "stdhttp", "xc": "std_http.xc", "links": ["libcurl.so.4 (4.8.0)"] } + }, + "verify": "make test-optional — runs m-stdlib's 4 callout-backed suites (STDCRYPTOTST/STDCRYPTODOCTST/STDCOMPRESSTST/STDHTTPTST) in byte mode; last run 4 suites / 151 assertions / 0 failed, no hang." + }, "consumers": [ { "repo": "m-cli", "transport": "DockerEngine (src/m_cli/engine.py)" }, { "repo": "m-stdlib", "transport": "test runner; reuses m-cli's DockerEngine when m-cli is installed" } diff --git a/dist/m-test-engine.json b/dist/m-test-engine.json index af426c0..2a9ffd8 100644 --- a/dist/m-test-engine.json +++ b/dist/m-test-engine.json @@ -44,5 +44,5 @@ "watch": { "destructive": false, "read_only": true }, "capabilities": { "destructive": false, "read_only": true } }, - "verified_on": "2026-05-11" + "verified_on": "2026-05-30" } \ No newline at end of file diff --git a/dist/repo.meta.json b/dist/repo.meta.json index 6c9488d..02c685e 100644 --- a/dist/repo.meta.json +++ b/dist/repo.meta.json @@ -6,7 +6,7 @@ "language": ["dockerfile"], "license": "AGPL-3.0", "agent_instructions": "AGENTS.md", - "verified_on": "2026-05-11", + "verified_on": "2026-05-30", "exposes": { "lifecycle": "dist/lifecycle.json", "engine_contract": "dist/m-test-engine.json", diff --git a/docker/Dockerfile b/docker/Dockerfile index e75f784..64399b3 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -14,6 +14,24 @@ # # Pinned to yottadb-base:latest-master to match what m-stdlib's CI uses. +# ── callout-builder stage ───────────────────────────────────────────── +# Compiles m-stdlib's optional-module YDB call-out .so's (libz/libzstd -> +# stdcompress.so, libcrypto -> std_crypto.so, libcurl -> http.so) from the C +# sources staged into the build context by `make callouts-stage`. The build +# toolchain (gcc + -dev headers) lives ONLY in this stage so the final image +# stays minimal — it carries the compiled .so's and the already-present runtime +# libs, not the compilers. The .c sources are single-sourced in m-stdlib and +# never committed here (docker/_callouts/ is gitignored). +FROM yottadb/yottadb-base:latest-master AS callout-builder +RUN apt-get update && apt-get install -y --no-install-recommends \ + gcc libc6-dev zlib1g-dev libzstd-dev libssl-dev libcurl4-openssl-dev \ + && rm -rf /var/lib/apt/lists/* +COPY _callouts/ /build/ +# build-callouts.sh reads $ydb_dist for libyottadb.h and each .c's `// link:` +# directive for its -l flags; outputs so//*.so under /build. +RUN cd /build && ydb_dist=/opt/yottadb/current bash tools/build-callouts.sh + +# ── final image ─────────────────────────────────────────────────────── FROM yottadb/yottadb-base:latest-master # Build-time identity. Defaults are sentinels so a local `docker build` @@ -52,6 +70,30 @@ RUN echo '. /opt/yottadb/current/ydb_env_set 2>/dev/null || true' \ > /etc/profile.d/ydb-env.sh \ && chmod +x /etc/profile.d/ydb-env.sh +# m-stdlib optional-module call-outs. The compiled .so's (from the +# callout-builder stage) land in $STDLIB_LIB; the .xc descriptors (staged from +# m-stdlib) land beside them. The descriptors' first line is "$STDLIB_LIB/.so", +# resolved at load time from the process env. Wiring ydb_xc_ lets YottaDB +# resolve $&stdcompress.* / $&stdcrypto.* / $&stdhttp.* — so m-stdlib's optional +# suites run on this engine. Exported via /etc/profile.d so `bash -lc` (m-cli's +# DockerEngine transport, the smoke target, healthcheck) picks them up. +# +# HANG-GUARD invariant: a .xc descriptor whose .so is missing makes YottaDB hang +# on first $& resolution. The .so's are COPYed first and below the build asserts +# all three exist, so a descriptor is never wired without its library present. +COPY --from=callout-builder /build/so/linux-x86_64/*.so /opt/stdlib/lib/ +COPY _callouts/xc/*.xc /opt/stdlib/xc/ +RUN set -e; \ + for so in stdcompress std_crypto http; do \ + test -f /opt/stdlib/lib/$so.so || { echo "MISSING callout .so: $so.so" >&2; exit 1; }; \ + done; \ + { echo 'export STDLIB_LIB=/opt/stdlib/lib'; \ + echo 'export ydb_xc_stdcompress=/opt/stdlib/xc/std_compress.xc'; \ + echo 'export ydb_xc_stdcrypto=/opt/stdlib/xc/std_crypto.xc'; \ + echo 'export ydb_xc_stdhttp=/opt/stdlib/xc/std_http.xc'; \ + } > /etc/profile.d/stdlib-callouts.sh; \ + chmod +x /etc/profile.d/stdlib-callouts.sh + # Container-side introspection script (Phase 4 of the m-engine plan). # `mte status --json` returns a structured snapshot consumed by # m-cli's `m engine status --verbose`. The script uses a login-shell From 45053e2f43b91f77aaade8267e77e111e9e3420c Mon Sep 17 00:00:00 2001 From: Rafael Richards Date: Sat, 20 Jun 2026 08:34:19 -0400 Subject: [PATCH 2/2] image 0.2.0: rebake the stdhttp callout so HTTP egress works on YDB r2.07 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The stdhttp $ZF call-out was dark on 0.1.0 — every $&stdhttp.* ref failed to compile/resolve on YottaDB r2.07, so STDS3 egress could not leave YDB. Root-caused live (see m-stdlib discoveries.md G-HTTP-YDB): two latent bugs in the YDB http_perform path (never exercised before — the package never loaded and IRIS uses %Net.HttpRequest): 1. underscore in the M-side call-out labels in std_http.xc (_=concat → %YDB-E-EXPR/ZCRTENOTF) — fixed in m-stdlib (perform:/available:). 2. http_perform missing the leading `int argc` ABI param → SIGSEGV — fixed in m-stdlib. The image fix is purely a rebake: `make callouts-stage` single-sources the corrected http.c + std_http.xc from m-stdlib; nothing in the bake logic changes. This commit: - Dockerfile: ydb-version LABEL r2.02 -> r2.07 (matches the baked release). - dist/m-test-engine.json: default_tag 0.1.0 -> 0.2.0; ydb_version r2.02 -> r2.07; verified_on -> 2026-06-20. - tools/gen-skill.py: refresh the drift example; skill regenerated (r2.07). - docs/m-engine-tracker.md: 0.2.0 rebake entry. Verified on the rebaked image (both engines): make smoke ✓; in-image $&stdhttp.available()=1; STDHTTPTST 67/67 + STDS3MINIOTST 11/11 on YDB (unchanged on IRIS); v-stdlib VSLS3E2ETST 6/6 YDB; crypto/compress/sigv4 unregressed. Co-Authored-By: Claude Opus 4.8 (1M context) --- dist/m-test-engine.json | 6 +++--- docker/Dockerfile | 2 +- docs/m-engine-tracker.md | 45 ++++++++++++++++++++++++++++++++++++++++ tools/gen-skill.py | 4 +++- 4 files changed, 52 insertions(+), 5 deletions(-) diff --git a/dist/m-test-engine.json b/dist/m-test-engine.json index 2a9ffd8..3c51103 100644 --- a/dist/m-test-engine.json +++ b/dist/m-test-engine.json @@ -2,7 +2,7 @@ "$schema": "https://raw.githubusercontent.com/m-dev-tools/m-test-engine/main/dist/m-test-engine.schema.json", "protocol": 1, "image": "ghcr.io/m-dev-tools/m-test-engine", - "default_tag": "0.1.0", + "default_tag": "0.2.0", "container": "m-test-engine", "bind_mount": { "host": "$HOME/m-work", @@ -12,7 +12,7 @@ "compose_file": "docker/compose.yml", "repo_url": "https://github.com/m-dev-tools/m-test-engine", "min_docker": "20.10", - "ydb_version": "r2.02", + "ydb_version": "r2.07", "run_args": { "hostname": "m-test-engine", "working_dir": "/m-work", @@ -44,5 +44,5 @@ "watch": { "destructive": false, "read_only": true }, "capabilities": { "destructive": false, "read_only": true } }, - "verified_on": "2026-05-30" + "verified_on": "2026-06-20" } \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile index 64399b3..7dd1cc0 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -49,7 +49,7 @@ ARG GIT_SHA=local-dev # protocol_mismatch / image_outdated WARN in m-cli's status output. LABEL org.m-dev-tools.m-test-engine.protocol="1" LABEL org.m-dev-tools.m-test-engine.bind-mount="/m-work" -LABEL org.m-dev-tools.m-test-engine.ydb-version="r2.02" +LABEL org.m-dev-tools.m-test-engine.ydb-version="r2.07" LABEL org.m-dev-tools.m-test-engine.image-rev="${GIT_SHA}" # Standard OCI annotation labels — surfaced by `docker inspect` and diff --git a/docs/m-engine-tracker.md b/docs/m-engine-tracker.md index 655392c..71ea6d1 100644 --- a/docs/m-engine-tracker.md +++ b/docs/m-engine-tracker.md @@ -653,3 +653,48 @@ across phases 1–5; the only operational follow-ups are: - Foreign-author work on m-cli `engine-phase3` (parallel session added `_parse_ydb_routines` + WARN-on-missing tests) is still uncommitted, for that session to land. + +--- + +## Image `0.2.0` — `stdhttp` callout rebake (2026-06-20) + +Rebaked the image so the `stdhttp` `$ZF` call-out loads and runs on +YottaDB **r2.07** (it was dark on `0.1.0` — see m-stdlib `discoveries.md` +G-HTTP-YDB). Three root causes, all in the **YDB `http_perform` path** +that had never executed before (the package never loaded; IRIS uses +`%Net.HttpRequest`, not the `.so`): + +1. **Underscore in the M-side call-out labels** (`tools/std_http.xc` + `http_available:` / `http_perform:`) — in M `_` is the concatenation + operator, so `$&stdhttp.http_available()` parsed as + `$&stdhttp.http`_`available()` → `%YDB-E-EXPR` / `ZCRTENOTF`. Fixed in + m-stdlib: labels renamed `perform:` / `available:` (C symbols keep + their underscores), matching every working call-out (`sha256:`, + `availableLibz:`). +2. **`http_perform` C signature missing the leading `int argc`** that the + YDB call-out ABI passes as arg #0 → every argument shifted one slot → + `SIGSEGV`. Added `int argc` to `http_perform` + `http_available`. +3. Capture-capacity read after `->length` was zeroed (empty responses) — + reordered. + +**Bake mechanism unchanged** — `make callouts-stage` single-sources the +fixed `http.c` + `std_http.xc` from m-stdlib; no Dockerfile bake-logic +change beyond bumping the `ydb-version` LABEL `r2.02`→`r2.07`. + +**Changes:** `docker/Dockerfile` `ydb-version` LABEL → `r2.07`; +`dist/m-test-engine.json` `default_tag` `0.1.0`→`0.2.0`, `ydb_version` +`r2.02`→`r2.07`, `verified_on` → 2026-06-20; `tools/gen-skill.py` drift +example refreshed + skill regenerated (declares r2.07). + +**Verified (rebaked image, both engines):** `make smoke` ✓ (r2.07 LABEL, +healthy); in-image `$&stdhttp.available()` = 1; m-stdlib `STDHTTPTST` +67/67 + `STDS3MINIOTST` 11/11 on **YDB** (and unchanged on IRIS); +v-stdlib `VSLS3E2ETST` 6/6 on YDB; `STDCRYPTO`/`STDCOMPRESS`/`STDSIGV4` +unregressed. + +**Follow-ups (deferred, not blocking egress):** +- `stdfs` call-out is still not wired in `/etc/profile.d/stdlib-callouts.sh` + (`stdfs.so` is present). `std_fs.xc` has the same zero-arg + `stdfs_available()` *and* would need the same `int argc` / underscore + audit in m-stdlib before wiring — left for a focused STDFS pass. +- GHCR publish of `0.2.0` is a user action (push auth). diff --git a/tools/gen-skill.py b/tools/gen-skill.py index da30d35..f8f9076 100644 --- a/tools/gen-skill.py +++ b/tools/gen-skill.py @@ -160,7 +160,9 @@ def render(manifest: dict) -> str: - `runtime_ydb_version_drift` — *live* `mte status --release` diverges from `manifest.ydb_version`. Catches upstream-base drift that the static-label check can't see (e.g. when `yottadb-base:latest-master` - ships V7.1-002 but the LABEL says r2.02). + rolls its YottaDB release forward but the LABEL still pins the old one + — what happened pre-`0.2.0`, when the image had moved to r2.07 while the + LABEL/manifest still declared r2.02). ## Failure modes