callouts: build + bake m-stdlib YDB call-out .so's into the engine image

.gitignore

@@ -0,0 +1,4 @@
+# Callout sources staged from m-stdlib at build time (single-sourced there,
+# never vendored here). Populated by `make callouts-stage`, consumed by the
+# Dockerfile's callout-builder stage.
+docker/_callouts/

Makefile

@@ -17,15 +17,47 @@
 # But run-from-here works too: cd into m-test-engine and `make smoke`
 # for a quick check.
 
-.PHONY: up down logs shell smoke clean manifest check-manifest check-docs-prose skill-install
+.PHONY: up down logs shell smoke clean manifest check-manifest check-docs-prose skill-install callouts-stage test-optional
 
 COMPOSE := docker compose -f docker/compose.yml
 
-up:
+# m-stdlib checkout that owns the callout C sources + .xc descriptors. They are
+# single-sourced there and staged into the build context at build time (never
+# vendored into this repo — see docker/_callouts/ in .gitignore). Override if
+# your m-stdlib lives elsewhere.
+M_STDLIB    ?= $(HOME)/vista-cloud-dev/m-stdlib
+CALLOUT_DIR := docker/_callouts
+
+# callouts-stage — copy m-stdlib's callout build inputs into the Docker build
+# context so the callout-builder image stage can compile them. Idempotent;
+# fails loudly if m-stdlib (or its callout sources) can't be found.
+callouts-stage:
+	@if [ ! -d "$(M_STDLIB)/src/callouts" ]; then \
+	  echo "ERROR: m-stdlib callout sources not found at $(M_STDLIB)/src/callouts" >&2; \
+	  echo "       set M_STDLIB=/path/to/m-stdlib" >&2; exit 1; \
+	fi
+	@rm -rf $(CALLOUT_DIR)
+	@mkdir -p $(CALLOUT_DIR)/src/callouts $(CALLOUT_DIR)/tools $(CALLOUT_DIR)/xc
+	@cp $(M_STDLIB)/src/callouts/*.c            $(CALLOUT_DIR)/src/callouts/
+	@cp $(M_STDLIB)/tools/build-callouts.sh     $(CALLOUT_DIR)/tools/
+	@cp $(M_STDLIB)/tools/std_compress.xc $(M_STDLIB)/tools/std_crypto.xc $(M_STDLIB)/tools/std_http.xc $(CALLOUT_DIR)/xc/
+	@echo "callouts-stage: staged $$(ls $(CALLOUT_DIR)/src/callouts/*.c | wc -l) C source(s) + 3 .xc from $(M_STDLIB)"
+
+up: callouts-stage
 	$(COMPOSE) up -d --build
 	@echo
 	@echo "m-test-engine is up. Verify with: make smoke"
 
+# test-optional — run m-stdlib's callout-backed suites against this engine in
+# byte (M) mode (the contract for the byte-oriented modules). Proves the baked
+# callouts resolve with no hang. Drives the Go `m` binary directly because the
+# byte suites need --chset m, which m-stdlib's own `make test-optional` does
+# not yet pass.
+M       ?= $(HOME)/vista-cloud-dev/m-cli/dist/m
+test-optional:
+	cd $(M_STDLIB) && $(M) test tests/STDCRYPTOTST.m tests/STDCRYPTODOCTST.m tests/STDCOMPRESSTST.m tests/STDHTTPTST.m \
+	    --engine ydb --docker m-test-engine --routines=src --chset m
+
 down:
 	$(COMPOSE) down
 

dist/lifecycle.json

@@ -16,14 +16,26 @@
     "logs":  { "make": "make logs",  "compose": "docker compose -f docker/compose.yml logs -f" },
     "shell": { "make": "make shell", "compose": "docker exec -it m-test-engine bash" },
     "smoke": { "make": "make smoke", "compose": "docker exec m-test-engine bash -lc '$ydb_dist/mumps -run %XCMD '\\''write \"smoke ok\",!'\\'''" },
-    "clean": { "make": "make clean", "compose": "docker compose -f docker/compose.yml down -v" }
+    "clean": { "make": "make clean", "compose": "docker compose -f docker/compose.yml down -v" },
+    "test-optional": { "make": "make test-optional", "compose": "n/a — drives the Go `m` binary against this container in byte mode" }
   },
   "exec_convention": {
     "shape": "docker exec m-test-engine bash -lc '<command>'",
-    "ydb_env_loaded_via": "/etc/profile.d/ydb-env.sh (sources /opt/yottadb/current/ydb_env_set)",
-    "available_env_vars": ["ydb_dist", "ydb_routines"],
+    "ydb_env_loaded_via": "/etc/profile.d/ydb-env.sh (sources /opt/yottadb/current/ydb_env_set); /etc/profile.d/stdlib-callouts.sh (m-stdlib callout vars)",
+    "available_env_vars": ["ydb_dist", "ydb_routines", "STDLIB_LIB", "ydb_xc_stdcompress", "ydb_xc_stdcrypto", "ydb_xc_stdhttp"],
     "notes": "Use bash -lc so the YDB env loads. Single-quote M commands so the outer bash does not expand $ZVERSION etc. (those are YDB special variables, not shell vars)."
   },
+  "stdlib_callouts": {
+    "provides": "m-stdlib optional-module YDB call-out libraries, baked at image build from m-stdlib's src/callouts/*.c via tools/build-callouts.sh (single-sourced in m-stdlib; staged into the build context by `make callouts-stage`, never vendored here).",
+    "lib_dir": "/opt/stdlib/lib",
+    "xc_dir": "/opt/stdlib/xc",
+    "libraries": {
+      "stdcompress.so": { "package": "stdcompress", "xc": "std_compress.xc", "links": ["libz.so.1 (1.3)", "libzstd.so.1 (1.5.5)"] },
+      "std_crypto.so":  { "package": "stdcrypto",   "xc": "std_crypto.xc",   "links": ["libcrypto.so.3 (OpenSSL 3)"] },
+      "http.so":        { "package": "stdhttp",     "xc": "std_http.xc",     "links": ["libcurl.so.4 (4.8.0)"] }
+    },
+    "verify": "make test-optional — runs m-stdlib's 4 callout-backed suites (STDCRYPTOTST/STDCRYPTODOCTST/STDCOMPRESSTST/STDHTTPTST) in byte mode; last run 4 suites / 151 assertions / 0 failed, no hang."
+  },
   "consumers": [
     { "repo": "m-cli",    "transport": "DockerEngine (src/m_cli/engine.py)" },
     { "repo": "m-stdlib", "transport": "test runner; reuses m-cli's DockerEngine when m-cli is installed" }

dist/m-test-engine.json

@@ -2,7 +2,7 @@
   "$schema": "https://raw.githubusercontent.com/m-dev-tools/m-test-engine/main/dist/m-test-engine.schema.json",
   "protocol": 1,
   "image": "ghcr.io/m-dev-tools/m-test-engine",
-  "default_tag": "0.1.0",
+  "default_tag": "0.2.0",
   "container": "m-test-engine",
   "bind_mount": {
     "host": "$HOME/m-work",
@@ -12,7 +12,7 @@
   "compose_file": "docker/compose.yml",
   "repo_url": "https://github.com/m-dev-tools/m-test-engine",
   "min_docker": "20.10",
-  "ydb_version": "r2.02",
+  "ydb_version": "r2.07",
   "run_args": {
     "hostname": "m-test-engine",
     "working_dir": "/m-work",
@@ -44,5 +44,5 @@
     "watch":        { "destructive": false, "read_only": true  },
     "capabilities": { "destructive": false, "read_only": true  }
   },
-  "verified_on": "2026-05-11"
+  "verified_on": "2026-06-20"
 }

dist/repo.meta.json

@@ -6,7 +6,7 @@
   "language": ["dockerfile"],
   "license": "AGPL-3.0",
   "agent_instructions": "AGENTS.md",
-  "verified_on": "2026-05-11",
+  "verified_on": "2026-05-30",
   "exposes": {
     "lifecycle":       "dist/lifecycle.json",
     "engine_contract": "dist/m-test-engine.json",

docker/Dockerfile

@@ -14,6 +14,24 @@
 #
 # Pinned to yottadb-base:latest-master to match what m-stdlib's CI uses.
 
+# ── callout-builder stage ─────────────────────────────────────────────
+# Compiles m-stdlib's optional-module YDB call-out .so's (libz/libzstd ->
+# stdcompress.so, libcrypto -> std_crypto.so, libcurl -> http.so) from the C
+# sources staged into the build context by `make callouts-stage`. The build
+# toolchain (gcc + -dev headers) lives ONLY in this stage so the final image
+# stays minimal — it carries the compiled .so's and the already-present runtime
+# libs, not the compilers. The .c sources are single-sourced in m-stdlib and
+# never committed here (docker/_callouts/ is gitignored).
+FROM yottadb/yottadb-base:latest-master AS callout-builder
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        gcc libc6-dev zlib1g-dev libzstd-dev libssl-dev libcurl4-openssl-dev \
+    && rm -rf /var/lib/apt/lists/*
+COPY _callouts/ /build/
+# build-callouts.sh reads $ydb_dist for libyottadb.h and each .c's `// link:`
+# directive for its -l flags; outputs so/<platform>/*.so under /build.
+RUN cd /build && ydb_dist=/opt/yottadb/current bash tools/build-callouts.sh
+
+# ── final image ───────────────────────────────────────────────────────
 FROM yottadb/yottadb-base:latest-master
 
 # Build-time identity. Defaults are sentinels so a local `docker build`
@@ -31,7 +49,7 @@ ARG GIT_SHA=local-dev
 # protocol_mismatch / image_outdated WARN in m-cli's status output.
 LABEL org.m-dev-tools.m-test-engine.protocol="1"
 LABEL org.m-dev-tools.m-test-engine.bind-mount="/m-work"
-LABEL org.m-dev-tools.m-test-engine.ydb-version="r2.02"
+LABEL org.m-dev-tools.m-test-engine.ydb-version="r2.07"
 LABEL org.m-dev-tools.m-test-engine.image-rev="${GIT_SHA}"
 
 # Standard OCI annotation labels — surfaced by `docker inspect` and
@@ -52,6 +70,30 @@ RUN echo '. /opt/yottadb/current/ydb_env_set 2>/dev/null || true' \
     > /etc/profile.d/ydb-env.sh \
     && chmod +x /etc/profile.d/ydb-env.sh
 
+# m-stdlib optional-module call-outs. The compiled .so's (from the
+# callout-builder stage) land in $STDLIB_LIB; the .xc descriptors (staged from
+# m-stdlib) land beside them. The descriptors' first line is "$STDLIB_LIB/<lib>.so",
+# resolved at load time from the process env. Wiring ydb_xc_<pkg> lets YottaDB
+# resolve $&stdcompress.* / $&stdcrypto.* / $&stdhttp.* — so m-stdlib's optional
+# suites run on this engine. Exported via /etc/profile.d so `bash -lc` (m-cli's
+# DockerEngine transport, the smoke target, healthcheck) picks them up.
+#
+# HANG-GUARD invariant: a .xc descriptor whose .so is missing makes YottaDB hang
+# on first $& resolution. The .so's are COPYed first and below the build asserts
+# all three exist, so a descriptor is never wired without its library present.
+COPY --from=callout-builder /build/so/linux-x86_64/*.so /opt/stdlib/lib/
+COPY _callouts/xc/*.xc /opt/stdlib/xc/
+RUN set -e; \
+    for so in stdcompress std_crypto http; do \
+      test -f /opt/stdlib/lib/$so.so || { echo "MISSING callout .so: $so.so" >&2; exit 1; }; \
+    done; \
+    { echo 'export STDLIB_LIB=/opt/stdlib/lib'; \
+      echo 'export ydb_xc_stdcompress=/opt/stdlib/xc/std_compress.xc'; \
+      echo 'export ydb_xc_stdcrypto=/opt/stdlib/xc/std_crypto.xc'; \
+      echo 'export ydb_xc_stdhttp=/opt/stdlib/xc/std_http.xc'; \
+    } > /etc/profile.d/stdlib-callouts.sh; \
+    chmod +x /etc/profile.d/stdlib-callouts.sh
+
 # Container-side introspection script (Phase 4 of the m-engine plan).
 # `mte status --json` returns a structured snapshot consumed by
 # m-cli's `m engine status --verbose`. The script uses a login-shell

docs/m-engine-tracker.md

@@ -653,3 +653,48 @@ across phases 1–5; the only operational follow-ups are:
 - Foreign-author work on m-cli `engine-phase3` (parallel session
   added `_parse_ydb_routines` + WARN-on-missing tests) is still
   uncommitted, for that session to land.
+
+---
+
+## Image `0.2.0` — `stdhttp` callout rebake (2026-06-20)
+
+Rebaked the image so the `stdhttp` `$ZF` call-out loads and runs on
+YottaDB **r2.07** (it was dark on `0.1.0` — see m-stdlib `discoveries.md`
+G-HTTP-YDB). Three root causes, all in the **YDB `http_perform` path**
+that had never executed before (the package never loaded; IRIS uses
+`%Net.HttpRequest`, not the `.so`):
+
+1. **Underscore in the M-side call-out labels** (`tools/std_http.xc`
+   `http_available:` / `http_perform:`) — in M `_` is the concatenation
+   operator, so `$&stdhttp.http_available()` parsed as
+   `$&stdhttp.http`_`available()` → `%YDB-E-EXPR` / `ZCRTENOTF`. Fixed in
+   m-stdlib: labels renamed `perform:` / `available:` (C symbols keep
+   their underscores), matching every working call-out (`sha256:`,
+   `availableLibz:`).
+2. **`http_perform` C signature missing the leading `int argc`** that the
+   YDB call-out ABI passes as arg #0 → every argument shifted one slot →
+   `SIGSEGV`. Added `int argc` to `http_perform` + `http_available`.
+3. Capture-capacity read after `->length` was zeroed (empty responses) —
+   reordered.
+
+**Bake mechanism unchanged** — `make callouts-stage` single-sources the
+fixed `http.c` + `std_http.xc` from m-stdlib; no Dockerfile bake-logic
+change beyond bumping the `ydb-version` LABEL `r2.02`→`r2.07`.
+
+**Changes:** `docker/Dockerfile` `ydb-version` LABEL → `r2.07`;
+`dist/m-test-engine.json` `default_tag` `0.1.0`→`0.2.0`, `ydb_version`
+`r2.02`→`r2.07`, `verified_on` → 2026-06-20; `tools/gen-skill.py` drift
+example refreshed + skill regenerated (declares r2.07).
+
+**Verified (rebaked image, both engines):** `make smoke` ✓ (r2.07 LABEL,
+healthy); in-image `$&stdhttp.available()` = 1; m-stdlib `STDHTTPTST`
+67/67 + `STDS3MINIOTST` 11/11 on **YDB** (and unchanged on IRIS);
+v-stdlib `VSLS3E2ETST` 6/6 on YDB; `STDCRYPTO`/`STDCOMPRESS`/`STDSIGV4`
+unregressed.
+
+**Follow-ups (deferred, not blocking egress):**
+- `stdfs` call-out is still not wired in `/etc/profile.d/stdlib-callouts.sh`
+  (`stdfs.so` is present). `std_fs.xc` has the same zero-arg
+  `stdfs_available()` *and* would need the same `int argc` / underscore
+  audit in m-stdlib before wiring — left for a focused STDFS pass.
+- GHCR publish of `0.2.0` is a user action (push auth).

tools/gen-skill.py

@@ -160,7 +160,9 @@ def render(manifest: dict) -> str:
 - `runtime_ydb_version_drift` — *live* `mte status --release` diverges
   from `manifest.ydb_version`. Catches upstream-base drift that the
   static-label check can't see (e.g. when `yottadb-base:latest-master`
-  ships V7.1-002 but the LABEL says r2.02).
+  rolls its YottaDB release forward but the LABEL still pins the old one
+  — what happened pre-`0.2.0`, when the image had moved to r2.07 while the
+  LABEL/manifest still declared r2.02).
 
 ## Failure modes