diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 7b146d61f7..27739c32db 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -29,7 +29,7 @@ jobs: - run: cargo build --release -p agentos-native-sidecar - run: cargo build --release -p agentos-native-baseline - run: cargo build --release --target wasm32-wasip1 -p agentos-native-baseline - - run: make -C registry/native commands + - run: make -C toolchain commands - run: node packages/runtime-core/scripts/copy-wasm-commands.mjs --require - run: pnpm --dir packages/runtime-core build - run: pnpm --dir packages/runtime-benchmarks bench:check diff --git a/.github/workflows/ci-nightly.yml b/.github/workflows/ci-nightly.yml index 79bb6cdb3f..e0108161dc 100644 --- a/.github/workflows/ci-nightly.yml +++ b/.github/workflows/ci-nightly.yml @@ -23,7 +23,7 @@ jobs: with: targets: wasm32-wasip1 - run: pnpm install --frozen-lockfile - - run: make -C registry/native commands + - run: make -C toolchain commands - run: node packages/runtime-core/scripts/copy-wasm-commands.mjs --require - run: cargo check --workspace - run: cargo clippy --workspace --all-targets -- -D warnings diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 69e0136cb9..c41b2f1a76 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,6 +42,7 @@ jobs: - run: node scripts/check-agentos-client-protocol-compat.mjs - run: node --test scripts/verify-fixed-versions.test.mjs - run: node scripts/verify-fixed-versions.mjs + - run: pnpm check-layout - run: cargo fmt --check - run: pnpm check-types - run: pnpm lint @@ -63,11 +64,11 @@ jobs: targets: wasm32-wasip1 - uses: Swatinem/rust-cache@v2 with: - workspaces: registry/native -> target + workspaces: toolchain -> toolchain/target cache-workspace-crates: true - key: wasm-commands-${{ hashFiles('registry/native/Cargo.lock') }} + key: wasm-commands-${{ hashFiles('toolchain/Cargo.lock') }} - run: pnpm install --frozen-lockfile --filter '@rivet-dev/agentos-runtime-core...' - - run: make -C registry/native commands + - run: make -C toolchain commands - run: node packages/runtime-core/scripts/copy-wasm-commands.mjs --require - uses: actions/upload-artifact@v4 with: @@ -104,9 +105,7 @@ jobs: - uses: actions/download-artifact@v4 with: name: wasm-commands - # Restore the canonical build output so registry packages can stage - # from the same path used by a local `just registry-native` build. - path: registry/native/target/wasm32-wasip1/release/commands + path: packages/runtime-core/commands - run: node packages/runtime-core/scripts/copy-wasm-commands.mjs --require - run: | if grep -qE '^[[:space:]]*-[[:space:]]*website[[:space:]]*$' pnpm-workspace.yaml; then diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index dd7cb0a5f7..512f308c13 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -66,11 +66,11 @@ jobs: targets: wasm32-wasip1 - uses: Swatinem/rust-cache@v2 with: - workspaces: registry/native -> target + workspaces: toolchain -> toolchain/target cache-workspace-crates: true - key: wasm-commands-${{ hashFiles('registry/native/Cargo.lock') }} + key: wasm-commands-${{ hashFiles('toolchain/Cargo.lock') }} - run: pnpm install --frozen-lockfile --filter '@rivet-dev/agentos-runtime-core...' - - run: make -C registry/native commands + - run: make -C toolchain commands - run: node packages/runtime-core/scripts/copy-wasm-commands.mjs --require - uses: actions/upload-artifact@v4 with: diff --git a/.gitignore b/.gitignore index ddccd47e77..e7d40a3268 100644 --- a/.gitignore +++ b/.gitignore @@ -37,7 +37,19 @@ core.[0-9]* # Secrets secrets/**/* -# Registry build artifacts +# Generated WASM toolchain and software artifacts. These are rebuilt for tests +# and releases and must never be committed. +packages/runtime-core/commands/ +toolchain/vendor/ +toolchain/c/build/ +toolchain/c/vendor/ +toolchain/c/libs/ +toolchain/c/sysroot/ +toolchain/c/.cache/ +toolchain/std-patches/wasi-libc-overrides/*.o +software/*/bin/ + +# Legacy registry build paths retained for old worktrees. registry/native/target/ registry/native/vendor/ registry/native/c/build/ @@ -47,6 +59,7 @@ registry/native/c/.cache/ registry/native/c/sysroot/ registry/software/coreutils/bin/ registry/software/*/wasm/ +software/*/wasm/ registry/software/*/agentos-package.meta.json registry/.last-publish-hash registry/software/*/.last-publish-hash diff --git a/CLAUDE.md b/CLAUDE.md index b9172199b2..c5c872ea13 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -74,26 +74,55 @@ the guest — over inventing a softer fallback that hides the failure. only when the caller explicitly supplied them. - Agent adapters must use real upstream SDKs. Do not replace SDK adapters with direct API-call stubs. -- Native registry command binaries are generated artifacts. Do not commit - `registry/software/coreutils/bin/`; a fresh checkout intentionally has no - staged coreutils commands. Any VM, browser demo, or test that needs `sh` or - coreutils must build and stage the complete package from the repository root: +- WASM command binaries and every toolchain build output are generated + artifacts. Never commit `packages/runtime-core/commands/`, `software/*/bin/`, + `toolchain/vendor/`, `toolchain/c/{build,vendor,libs,sysroot,.cache}/`, or + `toolchain/std-patches/wasi-libc-overrides/*.o`. A fresh checkout intentionally + contains source and patches only. Rebuild and stage the complete default tool + set from the repository root with: ```bash pnpm install --frozen-lockfile - just registry-native - pnpm --filter @agentos-software/coreutils build:runtime + just tools-rebuild ``` - `just registry-native` compiles the patched Rust and C WASI sources into - `registry/native/target/wasm32-wasip1/release/commands/`; the pnpm command - then stages `bin/` and assembles `dist/package/`. `just registry-native-cmd - sh` builds only one development artifact and is not sufficient to assemble - the complete coreutils package. The ordinary `build` script may create an - empty placeholder for source-only repository checks; that placeholder is not - runnable and must never be treated as proof that coreutils was built. - Publishing coreutils must run the strict `build:runtime` lifecycle and fail - when the generated command set is absent or incomplete. + `just tools-rebuild` runs `just toolchain-build`, copies the canonical output + from `toolchain/target/wasm32-wasip1/release/commands/` into runtime staging, + and builds the `@agentos-software/*` packages. For focused development, + `just toolchain-cmd ` rebuilds one command, but it is not sufficient + for a release or complete package validation. Publish workflows must rebuild + and stage the complete command set and fail when it is absent or incomplete. + +## Software Build (WASM Toolchain) + +Registry software is **real upstream Linux software** (GNU coreutils, grep, sed, +gawk, real curl/sqlite/duckdb/vim, …) compiled to `wasm32-wasip1` against a +**sysroot we fully own** — a patched Rust std + libc whose gaps are filled by +custom host-syscall imports. Treat that target as **native POSIX**; +`wasm32-wasip1` is an implementation detail, not a feature ceiling. + +- **We do not depend on stock WASI / wasi-libc.** The sysroot is ours. A missing + libc/POSIX API (`getrlimit`/`RLIMIT_NOFILE`, `getgroups`, spawn, fd dup, …) is + never a blocker — implement it (real, or a sane stub) in the patched + std/libc/host-import layer. "WASI doesn't have X" is not a reason to stop; X is + ours to add. +- **Fix portability one layer down, in the sysroot** — a new std/libc patch or a + new host import — not with `cfg(target_*)` branches or shims in the tool's own + source. A WASM-specific branch in application code usually means the fix + belongs in the libc layer. +- **Patch the real upstream tool only as a fallback**, when the fix genuinely + cannot live in the sysroot. Patching the real tool is allowed; reimplementing + it is not. +- **"NOT POSSIBLE" is reserved for genuine impossibility** after exhausting both + sysroot patches and tool patches — never for a missing syscall we could + implement. Document the specific wall if you claim it. +- **Working in `software/`, you may (and should) fix the layer underneath.** When + a package behaves differently from real Linux, the root cause is usually not the + package — it's the runtime. It is in-scope and expected to fix the underlying + implementation: the Node-compat / bridge layer, the WASM execution runtime, the + kernel/VFS syscalls, or the patched sysroot/libc. Do **not** paper over a + Linux-deviating behavior in the package, its wrapper, or its test — chase it + down into whichever runtime layer owns it and make that layer match Linux. ## Publishing diff --git a/Cargo.lock b/Cargo.lock index 6c46128bd8..b27fa02fde 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -208,6 +208,7 @@ dependencies = [ "url", "v8", "wat", + "webpki-root-certs", ] [[package]] @@ -4273,6 +4274,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-root-certs" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d46a5a140e6f7afeccd8eae97eff335163939eac8b929834875168b29b3d267" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "webpki-roots" version = "0.26.11" diff --git a/Cargo.toml b/Cargo.toml index 8cc1efcefb..8bbe980838 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,6 @@ [workspace] resolver = "2" +exclude = ["software"] members = [ "crates/agentos-protocol", "crates/agentos-sidecar-core", diff --git a/README.md b/README.md index e9357918ca..1cf5b93e0f 100644 --- a/README.md +++ b/README.md @@ -157,7 +157,6 @@ Browse pre-built agents, tools, filesystems, and software packages at the [agent | `@agentos-software/git` | git | git version control | rust | - | - | | `@agentos-software/grep` | grep | GNU grep pattern matching (grep, egrep, fgrep) | rust | - | - | | `@agentos-software/gzip` | gzip | GNU gzip compression (gzip, gunzip, zcat) | rust | - | - | -| `@agentos-software/http-get` | http-get | Minimal HTTP GET fetch helper | c | - | - | | `@agentos-software/jq` | jq | jq JSON processor | rust | - | - | | `@agentos-software/ripgrep` | ripgrep | ripgrep fast recursive search | rust | - | - | | `@agentos-software/sed` | sed | GNU sed stream editor | rust | - | - | @@ -165,7 +164,7 @@ Browse pre-built agents, tools, filesystems, and software packages at the [agent | `@agentos-software/tar` | tar | GNU tar archiver | rust | - | - | | `@agentos-software/tree` | tree | tree directory listing | rust | - | - | | `@agentos-software/unzip` | unzip | unzip archive extraction | c | - | - | -| `@agentos-software/wget` | wget | GNU wget HTTP client | c | - | - | +| `@agentos-software/wget` | wget | GNU Wget file downloader | c | - | - | | `@agentos-software/yq` | yq | yq YAML/JSON processor | rust | - | - | | `@agentos-software/zip` | zip | zip archive creation | c | - | - | @@ -175,7 +174,7 @@ Browse pre-built agents, tools, filesystems, and software packages at the [agent |---------|-------------|----------| | `@agentos-software/build-essential` | Build-essential WASM command set (standard + make + git + curl) | standard, make, git, curl | | `@agentos-software/common` | Common WASM command set (coreutils + sed + grep + gawk + findutils + diffutils + tar + gzip) | coreutils, sed, grep, gawk, findutils, diffutils, tar, gzip | -| `@agentos-software/everything` | All available WASM command packages in a single bundle | coreutils, sed, grep, gawk, findutils, diffutils, tar, gzip, curl, zip, unzip, jq, ripgrep, fd, tree, file, yq, codex-cli | +| `@agentos-software/everything` | All available WASM command packages in a single bundle | coreutils, sed, grep, gawk, findutils, diffutils, tar, gzip, curl, wget, zip, unzip, jq, ripgrep, fd, tree, file, yq, codex-cli | ## License diff --git a/crates/bridge/bridge-contract.json b/crates/bridge/bridge-contract.json index 12c3e68767..97e0c20eff 100644 --- a/crates/bridge/bridge-contract.json +++ b/crates/bridge/bridge-contract.json @@ -189,12 +189,13 @@ "_childProcessSpawnStart", "_childProcessPoll", "_childProcessStdinWrite", - "_childProcessStdinClose", - "_childProcessKill", - "_processKill", - "_processSignalState" - ] - }, + "_childProcessStdinClose", + "_childProcessKill", + "_processKill", + "_processSignalState", + "_processTakeSignal" + ] + }, { "convention": "syncPromise", "argumentTypes": [ @@ -945,13 +946,17 @@ "method": "process.kill", "translateArgs": false }, - "_processSignalState": { - "method": "process.signal_state", - "translateArgs": false - }, - "_ptySetRawMode": { - "method": "__pty_set_raw_mode", - "translateArgs": false + "_processSignalState": { + "method": "process.signal_state", + "translateArgs": false + }, + "_processTakeSignal": { + "method": "process.take_signal", + "translateArgs": false + }, + "_ptySetRawMode": { + "method": "__pty_set_raw_mode", + "translateArgs": false }, "_resolveModule": { "method": "__resolve_module", diff --git a/crates/client/tests/common/mod.rs b/crates/client/tests/common/mod.rs index 1013b35300..ca01e86829 100644 --- a/crates/client/tests/common/mod.rs +++ b/crates/client/tests/common/mod.rs @@ -149,8 +149,8 @@ fn wasm_command_mounts() -> Vec { /// Locate the materialized coreutils package under the in-repo registry build. pub fn coreutils_package_dir() -> Option { - let registry_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("../../registry/software/coreutils/dist/package"); + let registry_dir = + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../software/coreutils/dist/package"); if registry_dir.join("agentos-package.json").is_file() { return std::fs::canonicalize(registry_dir).ok(); } @@ -175,7 +175,7 @@ pub fn coreutils_wasm_dir() -> Option { } let legacy_registry_dir = - PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../registry/software/coreutils/wasm"); + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../software/coreutils/wasm"); if legacy_registry_dir.is_dir() { return std::fs::canonicalize(legacy_registry_dir).ok(); } @@ -244,6 +244,6 @@ pub async fn require_wasm_commands(os: &AgentOs, test_name: &str) -> bool { eprintln!("skipping {message}"); false } else { - panic!("{message}; run the registry/native command build or set AGENT_OS_CLIENT_ALLOW_E2E_SKIPS=1 for local skip-only runs"); + panic!("{message}; run the toolchain command build or set AGENT_OS_CLIENT_ALLOW_E2E_SKIPS=1 for local skip-only runs"); } } diff --git a/crates/client/tests/packages_aospkg_e2e.rs b/crates/client/tests/packages_aospkg_e2e.rs index 027f3a6824..fd9f104bcb 100644 --- a/crates/client/tests/packages_aospkg_e2e.rs +++ b/crates/client/tests/packages_aospkg_e2e.rs @@ -16,7 +16,7 @@ mod common; fn coreutils_aospkg() -> Option { for path in [ PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("../../registry/software/coreutils/dist/package.aospkg"), + .join("../../software/coreutils/dist/package.aospkg"), PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("../../node_modules/@agentos-software/coreutils/dist/package.aospkg"), ] { diff --git a/crates/client/tests/pi_session_e2e.rs b/crates/client/tests/pi_session_e2e.rs index 9b15fd7fc9..2210301eba 100644 --- a/crates/client/tests/pi_session_e2e.rs +++ b/crates/client/tests/pi_session_e2e.rs @@ -9,7 +9,7 @@ //! - `AGENT_OS_PI_MODULE_CWD` env (a workspace with a built/installed `@agentos-software/pi`), else //! - the repo root, but only when the in-repo adapter is built //! (`node_modules/@agentos-software/pi/dist/adapter.js`). Build it with `pnpm --dir packages/core -//! build && pnpm --dir registry/agent/pi build` (core first for types). +//! build && pnpm --dir software/pi build` (core first for types). //! //! Background: a real agent SDK exercises module-loading patterns (tsc `__exportStar` CJS barrels, //! deep pnpm symlink graphs, `__dirname` package self-location) that mock ACP adapters never touch. @@ -55,11 +55,11 @@ fn pi_module_cwd() -> Option { fn pi_package_path() -> Option { // Prefer the packed .aospkg — the artifact the registry actually ships. - let aospkg = repo_root().join("registry/agent/pi/dist/package.aospkg"); + let aospkg = repo_root().join("software/pi/dist/package.aospkg"); if aospkg.is_file() { return std::fs::canonicalize(aospkg).ok(); } - let dir = repo_root().join("registry/agent/pi/dist/package"); + let dir = repo_root().join("software/pi/dist/package"); if dir.join("agentos-package.json").is_file() { std::fs::canonicalize(dir).ok() } else { diff --git a/crates/client/tests/shell_pty_packages_e2e.rs b/crates/client/tests/shell_pty_packages_e2e.rs index 746998a613..b057fdd5c6 100644 --- a/crates/client/tests/shell_pty_packages_e2e.rs +++ b/crates/client/tests/shell_pty_packages_e2e.rs @@ -17,7 +17,7 @@ fn coreutils_package_path() -> Option { // Prefer the packed .aospkg — the artifact registry packages actually ship. for aospkg in [ PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("../../registry/software/coreutils/dist/package.aospkg"), + .join("../../software/coreutils/dist/package.aospkg"), PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("../../node_modules/@agentos-software/coreutils/dist/package.aospkg"), ] { @@ -26,8 +26,7 @@ fn coreutils_package_path() -> Option { } } for dir in [ - PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("../../registry/software/coreutils/dist/package"), + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../software/coreutils/dist/package"), PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("../../node_modules/@agentos-software/coreutils/dist/package"), ] { diff --git a/crates/execution/CLAUDE.md b/crates/execution/CLAUDE.md index 4bf08bd151..5aff12c315 100644 --- a/crates/execution/CLAUDE.md +++ b/crates/execution/CLAUDE.md @@ -91,7 +91,7 @@ ESM loader hooks (`loader.mjs`) and CJS `Module._load` patches (`runner.mjs`) ar - Node-facing `readdir` results must filter `.`/`..`. - Async methods should dispatch under `fs.promises.*`. - `fs.promises` methods that need real concurrency must use dedicated async bridge globals in `packages/build-tools/bridge-src/`; wrapping `fs.*Sync` inside `async` functions still serializes `Promise.all(...)` behind the first sidecar response. -- When adding WASI guest imports in `registry/native/crates/wasi-ext`, mirror the required module/object in `crates/execution/src/node_import_cache.rs`'s inline `NODE_WASM_RUNNER_SOURCE`; missing modules fail at `WebAssembly.instantiate()` before guest `main()` runs. +- When adding WASI guest imports in `toolchain/crates/wasi-ext`, mirror the required module/object in `crates/execution/src/node_import_cache.rs`'s inline `NODE_WASM_RUNNER_SOURCE`; missing modules fail at `WebAssembly.instantiate()` before guest `main()` runs. - Keep the embedded WASI shim in `crates/execution/src/wasm.rs` aligned with the patched wasi-libc surface used by the C command suite; overrides like `fcntl(F_SETFL)` now depend on `fd_fdstat_set_flags`, and missing imports fail at instantiation time before the guest command can do real work. - The shared-V8 WASM runner now resolves its own module loads plus internal guest `fs.openSync` / `fs.readSync` / `fs.writeSync` / `fs.closeSync` traffic inside `crates/execution/src/wasm.rs`; if the embedded runner gains more internal file syscalls, extend that internal sync-RPC handling there instead of surfacing those requests to callers or reintroducing a host-Node runtime path. - fd-based APIs (`open`, `read`, `write`, `close`, `fstat`) plus `createReadStream`/`createWriteStream` should ride the same bridge. diff --git a/crates/execution/assets/runners/wasi-module.js b/crates/execution/assets/runners/wasi-module.js index 7dfe4d0384..b861b77ba9 100644 --- a/crates/execution/assets/runners/wasi-module.js +++ b/crates/execution/assets/runners/wasi-module.js @@ -54,6 +54,7 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = const __agentOSWasiOpenDirectory = 2; const __agentOSWasiOpenExclusive = 4; const __agentOSWasiOpenTruncate = 8; + const __agentOSWasiFdflagsAppend = 1; const __agentOSWasiRightFdRead = 1n << 1n; const __agentOSWasiRightFdWrite = 1n << 6n; const __agentOSWasiDefaultRightsBase = 0xffffffffffffffffn; @@ -396,6 +397,14 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = } } + _hasReadRights(rights) { + try { + return (BigInt(rights) & __agentOSWasiRightFdRead) !== 0n; + } catch { + return true; + } + } + _writeUint32(ptr, value) { try { this._memoryView().setUint32(Number(ptr) >>> 0, Number(value) >>> 0, true); @@ -779,6 +788,9 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = } _mapFsError(error) { + __agentOSWasiDebug( + `fs error code=${String(error?.code ?? "")} message=${String(error?.message ?? error)}`, + ); switch (error?.code) { case "EACCES": case "EPERM": @@ -827,6 +839,7 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = refCount: 1, open: true, readOnly: entry.readOnly === true, + append: entry.append === true, }; } @@ -1380,19 +1393,44 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = return this._measureWasiPhase("writeResultPtr", () => this._writeUint32(nwrittenPtr, written)); } } + const entry = this._descriptorEntry(descriptor); + const localHostPassthrough = + handle.kind === "host-passthrough" && + entry?.kind === "file" && + entry.realFd === handle.targetFd; + const position = handle.append + ? this._measureWasiPhase("appendFstat", () => + Number(__agentOSFs().fstatSync(handle.targetFd).size ?? 0) + ) + : localHostPassthrough + ? (entry.offset ?? 0) + : null; const written = this._measureWasiPhase("writeSync", () => __agentOSFs().writeSync( handle.targetFd, bytes, 0, bytes.length, - null, + position, ) ); + if (localHostPassthrough) { + if (handle.append) { + entry.offset = this._measureWasiPhase("appendFstat", () => + Number(__agentOSFs().fstatSync(handle.targetFd).size ?? 0) + ); + } else { + entry.offset = (entry.offset ?? 0) + written; + } + } return this._measureWasiPhase("writeResultPtr", () => this._writeUint32(nwrittenPtr, written)); } if (handle?.kind === "guest-file" && typeof handle.targetFd === "number") { - const position = handle.append ? null : (handle.position ?? 0); + const position = handle.append + ? this._measureWasiPhase("appendFstat", () => + Number(__agentOSFs().fstatSync(handle.targetFd).size ?? 0) + ) + : (handle.position ?? 0); const written = this._measureWasiPhase("writeSync", () => __agentOSFs().writeSync( handle.targetFd, @@ -1469,7 +1507,11 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = } if (entry.kind === "file") { this._clearStatCache(); - const position = typeof entry.offset === "number" ? entry.offset : null; + const position = entry.append + ? this._measureWasiPhase("appendFstat", () => + Number(__agentOSFs().fstatSync(entry.realFd).size ?? 0) + ) + : (typeof entry.offset === "number" ? entry.offset : null); const written = this._measureWasiPhase("writeSync", () => __agentOSFs().writeSync( entry.realFd, @@ -1479,7 +1521,11 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = position, ) ); - if (typeof entry.offset === "number") { + if (entry.append) { + entry.offset = this._measureWasiPhase("appendFstat", () => + Number(__agentOSFs().fstatSync(entry.realFd).size ?? 0) + ); + } else if (typeof entry.offset === "number") { entry.offset += written; } return this._measureWasiPhase("writeResultPtr", () => this._writeUint32(nwrittenPtr, written)); @@ -1715,6 +1761,11 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = (handle?.kind === "passthrough" || handle?.kind === "host-passthrough") && typeof handle.targetFd === "number" ) { + const localEntry = this._descriptorEntry(descriptor); + const localHostPassthrough = + handle.kind === "host-passthrough" && + localEntry?.kind === "file" && + localEntry.realFd === handle.targetFd; const totalLength = this._boundedReadLength(iovs, iovsLen); const buffer = Buffer.alloc(totalLength); const bytesRead = __agentOSFs().readSync( @@ -1722,8 +1773,11 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = buffer, 0, totalLength, - null, + localHostPassthrough ? (localEntry.offset ?? 0) : null, ); + if (localHostPassthrough) { + localEntry.offset = (localEntry.offset ?? 0) + bytesRead; + } const written = this._writeToIovs(iovs, iovsLen, buffer.subarray(0, bytesRead)); return this._writeUint32(nreadPtr, written); } @@ -2206,8 +2260,12 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = this._clearStatCache(); } const fsConstants = __agentOSFs().constants ?? {}; + const requestedFdFlags = Number(_fdflags) >>> 0; + const append = (requestedFdFlags & __agentOSWasiFdflagsAppend) !== 0; let openFlags = requestedWriteAccess - ? fsConstants.O_RDWR ?? 2 + ? (this._hasReadRights(requestedRightsBase) + ? fsConstants.O_RDWR ?? 2 + : fsConstants.O_WRONLY ?? 1) : fsConstants.O_RDONLY ?? 0; if ((requestedFlags & __agentOSWasiOpenCreate) !== 0) { openFlags |= fsConstants.O_CREAT ?? 64; @@ -2218,6 +2276,9 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = if ((requestedFlags & __agentOSWasiOpenTruncate) !== 0) { openFlags |= fsConstants.O_TRUNC ?? 512; } + if (append) { + openFlags |= fsConstants.O_APPEND ?? 1024; + } if (openDirectory) { openFlags |= fsConstants.O_DIRECTORY ?? 0; } @@ -2235,10 +2296,13 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = hostPath: fsPath, readOnly: resolved.readOnly === true, realFd, - offset: 0, + offset: append + ? Number(__agentOSFs().fstatSync(realFd).size ?? 0) + : 0, + append, rightsBase: requestedRightsBase & allowedRightsInheriting, rightsInheriting: requestedRightsInheriting & allowedRightsInheriting, - fdFlags: (Number(_fdflags) >>> 0) & 0xffff, + fdFlags: requestedFdFlags & 0xffff, }); }); return this._measureWasiPhase("writeOpenedFd", () => this._writeUint32(openedFdPtr, openedFd)); diff --git a/crates/execution/assets/runners/wasm-runner.mjs b/crates/execution/assets/runners/wasm-runner.mjs index fb7e839855..6fac39d26f 100644 --- a/crates/execution/assets/runners/wasm-runner.mjs +++ b/crates/execution/assets/runners/wasm-runner.mjs @@ -21,7 +21,10 @@ const WASI_ERRNO_AGAIN = 6; const WASI_ERRNO_BADF = 8; const WASI_ERRNO_CHILD = 10; const WASI_ERRNO_INVAL = 28; +const WASI_ERRNO_IO = 29; +const WASI_ERRNO_MFILE = 33; const WASI_ERRNO_NOENT = 44; +const WASI_ERRNO_PERM = 63; const WASI_ERRNO_PIPE = 64; const WASI_ERRNO_ROFS = 69; const WASI_ERRNO_SPIPE = 70; @@ -30,6 +33,7 @@ const WASI_ERRNO_FAULT = 21; const WASI_RIGHT_FD_WRITE = 64n; const WASI_FILETYPE_UNKNOWN = 0; const WASI_FILETYPE_CHARACTER_DEVICE = 2; +const WASI_FILETYPE_DIRECTORY = 3; const WASI_FILETYPE_REGULAR_FILE = 4; const WASI_OFLAGS_CREAT = 1; const WASI_OFLAGS_DIRECTORY = 2; @@ -428,7 +432,8 @@ const passthroughHandles = new Map([ ]); const retainedSyntheticHandlesByDisplayFd = new Map(); const retainedSpawnOutputHandlesByFd = new Map(); -let nextSyntheticFd = 64; +const FIRST_SYNTHETIC_FD = 1 << 20; +let nextSyntheticFd = FIRST_SYNTHETIC_FD; let nextSyntheticPipeId = 1; const syntheticWaitArray = new Int32Array(new SharedArrayBuffer(4)); let delegateWriteScratch = { base: 0, capacity: 0 }; @@ -1160,13 +1165,19 @@ function fsOpenFlagForPathOpen(oflags, rightsBase, fdflags) { return 'r+'; } -function allocateSyntheticFd() { - let fd = nextSyntheticFd; - while ( +function syntheticFdInUse(fd) { + return ( syntheticFdEntries.has(fd) || passthroughHandles.has(fd) || + retainedSpawnOutputHandlesByFd.has(fd) || + retainedSyntheticHandlesByDisplayFd.has(fd) || delegateManagedFdRefCounts.has(fd) - ) { + ); +} + +function allocateSyntheticFd(minFd = nextSyntheticFd) { + let fd = Math.max(FIRST_SYNTHETIC_FD, Number(minFd) >>> 0); + while (syntheticFdInUse(fd)) { fd += 1; } nextSyntheticFd = fd + 1; @@ -1226,8 +1237,9 @@ function openManagedPathIoFd(guestPath, rightsBase, fdflags) { return null; } try { + const hostPath = resolveHostFsPath(guestPath) ?? guestPath; return fsModule.openSync( - guestPath, + hostPath, fsOpenNumericFlagsForManagedPath(rightsBase, fdflags), 0o666, ); @@ -1243,16 +1255,35 @@ function retainPathOpenDelegateFd(openedFdPtr, guestPath, fdflags, rightsBase) { try { const openedFd = new DataView(instanceMemory.buffer).getUint32(Number(openedFdPtr), true); + let retainedFd = openedFd; + if (openedFd > 2 && syntheticFdInUse(openedFd)) { + if (typeof delegateManagedFdRenumber !== 'function') { + return WASI_ERRNO_FAULT; + } + retainedFd = allocateSyntheticFd(openedFd + 1); + const renumberResult = delegateManagedFdRenumber(openedFd, retainedFd); + if (renumberResult !== WASI_ERRNO_SUCCESS) { + return renumberResult; + } + const writeResult = writeGuestUint32(openedFdPtr, retainedFd); + if (writeResult !== WASI_ERRNO_SUCCESS) { + return writeResult; + } + traceHostProcess('path-open-delegate-renumber', { + openedFd, + retainedFd, + }); + } const append = (Number(fdflags) & WASI_FDFLAGS_APPEND) !== 0; - retainDelegateFd(openedFd); - if (openedFd > 2 && !passthroughHandles.has(openedFd)) { + retainDelegateFd(retainedFd); + if (retainedFd > 2 && !passthroughHandles.has(retainedFd)) { const ioFd = openManagedPathIoFd(guestPath, rightsBase, fdflags); - closedPassthroughFds.delete(openedFd); - passthroughHandles.set(openedFd, { + closedPassthroughFds.delete(retainedFd); + passthroughHandles.set(retainedFd, { kind: 'passthrough', - targetFd: openedFd, + targetFd: retainedFd, ioFd, - displayFd: openedFd, + displayFd: retainedFd, refCount: 0, open: true, readOnly: @@ -1331,6 +1362,19 @@ function writeGuestFilestat(ptr, stats, filetype = WASI_FILETYPE_REGULAR_FILE) { } } +function wasiFiletypeFromStats(stats) { + if (typeof stats?.isDirectory === 'function' && stats.isDirectory()) { + return WASI_FILETYPE_DIRECTORY; + } + if (typeof stats?.isCharacterDevice === 'function' && stats.isCharacterDevice()) { + return WASI_FILETYPE_CHARACTER_DEVICE; + } + if (typeof stats?.isFile === 'function' && stats.isFile()) { + return WASI_FILETYPE_REGULAR_FILE; + } + return WASI_FILETYPE_UNKNOWN; +} + function writeGuestFdstat(ptr, filetype, flags, rightsBase, rightsInheriting) { if (!(instanceMemory instanceof WebAssembly.Memory)) { return WASI_ERRNO_FAULT; @@ -1354,14 +1398,17 @@ function mapSyntheticFsError(error) { case 'EBADF': return WASI_ERRNO_BADF; case 'EACCES': - case 'EPERM': return WASI_ERRNO_ACCES; + case 'EPERM': + return WASI_ERRNO_PERM; + case 'ENOENT': + return WASI_ERRNO_NOENT; case 'EINVAL': return WASI_ERRNO_INVAL; case 'EROFS': return WASI_ERRNO_ROFS; default: - return WASI_ERRNO_FAULT; + return WASI_ERRNO_IO; } } @@ -1373,7 +1420,12 @@ function seekGuestFileHandle(handle, offset, whence) { } else if (numericWhence === WASI_WHENCE_CUR) { base = BigInt(handle.position ?? 0); } else if (numericWhence === WASI_WHENCE_END) { - base = BigInt(Number(fsModule.fstatSync(handle.targetFd).size ?? 0)); + // Passthrough (read-only delegate) handles keep the real host fd in ioFd; + // targetFd is only a synthetic guest fd number and fstat'ing it reports + // size 0. Prefer ioFd so SEEK_END returns the true file size (e.g. mbedTLS + // sizing a CA bundle via fseek(SEEK_END)+ftell before reading it). + const sizeFd = typeof handle.ioFd === 'number' ? handle.ioFd : handle.targetFd; + base = BigInt(Number(fsModule.fstatSync(sizeFd).size ?? 0)); } else { return null; } @@ -1776,6 +1828,91 @@ function writeBytesToGuestIovs(iovs, iovsLen, bytes) { return written >>> 0; } +function guestIovByteLength(iovs, iovsLen) { + if (!(instanceMemory instanceof WebAssembly.Memory)) { + throw new Error('WebAssembly memory is not available'); + } + + const view = new DataView(instanceMemory.buffer); + let total = 0; + for (let index = 0; index < (Number(iovsLen) >>> 0); index += 1) { + const entryOffset = (Number(iovs) >>> 0) + index * 8; + total += view.getUint32(entryOffset + 4, true); + } + return total >>> 0; +} + +function readHostNetSocketToGuestIovs(socket, iovs, iovsLen, nreadPtr) { + try { + const requestedLength = guestIovByteLength(iovs, iovsLen); + if (requestedLength === 0) { + return writeGuestUint32(nreadPtr, 0); + } + + if (socket.nonblock) { + let queued = dequeueHostNetBytes(socket, requestedLength); + if (queued.length > 0) { + return writeGuestUint32(nreadPtr, writeBytesToGuestIovs(iovs, iovsLen, queued)); + } + if (socket.lastError) return WASI_ERRNO_FAULT; + if (socket.readableEnded || socket.closed || !socket.socketId) { + return writeGuestUint32(nreadPtr, 0); + } + pollHostNetSocket(socket, 0); + queued = dequeueHostNetBytes(socket, requestedLength); + if (queued.length > 0) { + return writeGuestUint32(nreadPtr, writeBytesToGuestIovs(iovs, iovsLen, queued)); + } + if (socket.readableEnded || socket.closed || !socket.socketId) { + return writeGuestUint32(nreadPtr, 0); + } + return WASI_ERRNO_AGAIN; + } + + const deadline = + socket.recvTimeoutMs == null ? null : Date.now() + Math.max(0, socket.recvTimeoutMs); + while (true) { + const queued = dequeueHostNetBytes(socket, requestedLength); + if (queued.length > 0) { + return writeGuestUint32(nreadPtr, writeBytesToGuestIovs(iovs, iovsLen, queued)); + } + if (socket.lastError) return WASI_ERRNO_FAULT; + if (socket.readableEnded || socket.closed || !socket.socketId) { + return writeGuestUint32(nreadPtr, 0); + } + + const pollWaitMs = + deadline == null ? 50 : Math.max(0, Math.min(50, deadline - Date.now())); + if (deadline != null && pollWaitMs === 0) { + return WASI_ERRNO_AGAIN; + } + pollHostNetSocket(socket, pollWaitMs); + if (deadline != null && Date.now() >= deadline) { + return WASI_ERRNO_AGAIN; + } + } + } catch { + return WASI_ERRNO_FAULT; + } +} + +function writeHostNetSocketFromGuestIovs(socket, iovs, iovsLen, nwrittenPtr) { + if (!socket?.socketId || socket.closed) { + return WASI_ERRNO_BADF; + } + + try { + const bytes = collectGuestIovBytes(iovs, iovsLen); + if (bytes.length === 0) { + return writeGuestUint32(nwrittenPtr, 0); + } + const written = Number(callSyncRpc('net.write', [socket.socketId, bytes])) >>> 0; + return writeGuestUint32(nwrittenPtr, written); + } catch { + return WASI_ERRNO_FAULT; + } +} + function dequeuePipeBytes(pipe, maxBytes) { const requested = Math.max(0, Number(maxBytes) >>> 0); if (requested === 0 || pipe.chunks.length === 0) { @@ -2149,10 +2286,11 @@ function routeChunkToDelegateFd(fd, bytes) { } function finalizeChildExit(record, exitCode, signal) { - const status = - signal == null - ? (Number(exitCode ?? 1) & 0xff) - : 128 + (signalNumberFromName(signal) & 0x7f); + const signalNumber = signal == null ? 0 : signalNumberFromName(signal) & 0x7f; + const rawExitCode = signalNumber === 0 ? Number(exitCode ?? 1) & 0xff : 0; + const status = signalNumber === 0 ? rawExitCode : 128 + signalNumber; + record.exitCode = rawExitCode; + record.exitSignal = signalNumber; record.exitStatus = status; for (const fd of record.delegateRetainedFds ?? []) { if (releaseDelegateFd(fd) && typeof delegateManagedFdClose === 'function') { @@ -2312,6 +2450,8 @@ function createSyntheticChildRecord(result, stdinTarget, stdoutTarget, stderrTar stdoutPipe: null, stderrPipe: null, delegateRetainedFds: [], + exitCode: null, + exitSignal: null, exitStatus: null, pendingEvents, synthetic: true, @@ -2652,6 +2792,8 @@ function readSyncRpcLine() { } } +const pendingWasmSignals = []; + function callSyncRpc(method, args = []) { if ( globalThis.__agentOSSyncRpc && @@ -2698,13 +2840,23 @@ function callSyncRpc(method, args = []) { } const hostNetSockets = new Map(); -let nextHostNetSocketFd = 0x40000000; +let nextHostNetSocketFd = 4096; const HOST_NET_TIMEOUT_SENTINEL = '__agentos_net_timeout__'; +const HOST_NET_MSG_PEEK = 0x0001; function getHostNetSocket(fd) { return hostNetSockets.get(Number(fd) >>> 0) ?? null; } +function allocateHostNetSocketFd() { + for (let fd = HOST_NET_SOCKET_FD_MIN; fd <= HOST_NET_SOCKET_FD_MAX; fd += 1) { + if (!hostNetSockets.has(fd)) { + return fd; + } + } + return null; +} + function dequeueHostNetBytes(socket, maxBytes) { const requested = Math.max(0, Number(maxBytes) >>> 0); if (requested === 0 || socket.readChunks.length === 0) { @@ -2730,6 +2882,76 @@ function dequeueHostNetBytes(socket, maxBytes) { return Buffer.concat(parts); } +function peekHostNetBytes(socket, maxBytes) { + const requested = Math.max(0, Number(maxBytes) >>> 0); + if (requested === 0 || socket.readChunks.length === 0) { + return Buffer.alloc(0); + } + + const parts = []; + let remaining = requested; + for (const chunk of socket.readChunks) { + if (remaining === 0) break; + const chunkLength = Math.min(chunk.length, remaining); + parts.push(chunk.subarray(0, chunkLength)); + remaining -= chunkLength; + } + + return Buffer.concat(parts); +} + +function decodeHostNetSocketReadResult(result) { + if (result == null) { + return { kind: 'end' }; + } + + if (result === HOST_NET_TIMEOUT_SENTINEL) { + return { kind: 'timeout' }; + } + + if (typeof result === 'string') { + if (result === HOST_NET_TIMEOUT_SENTINEL) { + return { kind: 'timeout' }; + } + return { kind: 'data', bytes: Buffer.from(result, 'base64') }; + } + + const decoded = decodeSyncRpcValue(result); + if (Buffer.isBuffer(decoded)) { + return { kind: 'data', bytes: decoded }; + } + if (decoded == null) { + return { kind: 'end' }; + } + if (decoded === HOST_NET_TIMEOUT_SENTINEL) { + return { kind: 'timeout' }; + } + return { kind: 'timeout' }; +} + +function readReadyHostNetSocket(socket) { + if (!socket?.socketId || socket.closed) { + socket.readableEnded = true; + return null; + } + + const result = decodeHostNetSocketReadResult( + callSyncRpc('net.socket_read', [socket.socketId]), + ); + if (result.kind === 'data') { + if (result.bytes.length > 0) { + socket.readChunks.push(Buffer.from(result.bytes)); + } + return result; + } + if (result.kind === 'end') { + socket.readableEnded = true; + socket.closed = true; + socket.socketId = null; + } + return result; +} + function pollHostNetSocket(socket, waitMs) { if (!socket?.socketId || socket.closed) { return null; @@ -2764,6 +2986,20 @@ function pollHostNetSocket(socket, waitMs) { return event; } + if (event.readable === true || (Number(event.revents) & 0x001) !== 0) { + return readReadyHostNetSocket(socket); + } + + if (event.hangup === true) { + socket.readableEnded = true; + return event; + } + + if (event.error === true) { + socket.lastError = 'socket error'; + return event; + } + return event; } @@ -2795,13 +3031,19 @@ function parseHostNetAddress(raw) { }; } +function parseHostNetUnixAddress(raw) { + const value = String(raw ?? ''); + return value.startsWith('unix:') ? value.slice(5) : null; +} + function parseHostNetListenAddress(raw) { const value = String(raw ?? '').trim(); if (!value) { throw new Error('host_net listen address is required'); } - if (value.startsWith('/')) { - return { path: value }; + const unixPath = parseHostNetUnixAddress(value); + if (unixPath != null) { + return { path: unixPath }; } const address = parseHostNetAddress(value); return { host: address.host, port: address.port }; @@ -2831,6 +3073,7 @@ const HOST_NET_SOCK_DGRAM = 5; const HOST_NET_SOCKET_TYPE_MASK = 0xf; const HOST_NET_SOL_SOCKET = 1; const HOST_NET_WASI_SOL_SOCKET = 0x7fffffff; +const HOST_NET_SO_ERROR = 4; const HOST_NET_SO_RCVTIMEO_64 = 20; const HOST_NET_SO_RCVTIMEO_32 = 66; const HOST_NET_TIMEVAL_BYTES = 16; @@ -2969,7 +3212,8 @@ function writeGuestBytes(ptr, maxLen, bytes, actualLenPtr) { // Perform a single NON-BLOCKING accept on a listening host_net socket. On success it // registers the accepted connection as a new host_net socket and returns // { acceptedFd, address } (address is a Buffer: "host:port" for TCP, the peer path for -// AF_UNIX). Returns null when no connection is currently pending. Used by both net_poll +// AF_UNIX), or { error } when accepting the pending connection failed. Returns null when +// no connection is currently pending. Used by both net_poll // (to report accurate listener readiness) and net_accept (non-blocking semantics) so the // server never blocks inside accept() and starves already-connected clients. function tryHostNetAcceptOnce(socket) { @@ -2984,7 +3228,11 @@ function tryHostNetAcceptOnce(socket) { return null; } - const acceptedFd = nextHostNetSocketFd++; + const acceptedFd = allocateHostNetSocketFd(); + if (acceptedFd == null) { + callSyncRpc('net.destroy', [result.socketId]); + return { error: WASI_ERRNO_MFILE }; + } hostNetSockets.set(acceptedFd, { domain: socket.domain, sockType: socket.sockType, @@ -3010,7 +3258,7 @@ function tryHostNetAcceptOnce(socket) { port: result.info.remotePort, }), 'utf8'); } else { - address = Buffer.from(String(result.info?.remotePath ?? ''), 'utf8'); + address = Buffer.from(`unix:${String(result.info?.remotePath ?? '')}`, 'utf8'); } return { acceptedFd, address }; } @@ -3040,6 +3288,7 @@ const hostNetImport = { process.env.AGENTOS_SANDBOX_ROOT.length > 0); try { while (true) { + dispatchPendingWasmSignals(); const view = new DataView(instanceMemory.buffer); let ready = 0; // fds the kernel owns (PTY/pipe stdio in sidecar-managed mode): their readiness @@ -3124,8 +3373,11 @@ const hostNetImport = { let response = null; try { response = callSyncRpc('__kernel_poll', [kernelTargets, sliceMs]); - } catch { - response = null; + } catch (error) { + traceHostProcess('kernel-poll-error', { + message: error instanceof Error ? error.message : String(error), + }); + return WASI_ERRNO_FAULT; } const responseEntries = Array.isArray(response?.fds) ? response.fds : []; for (const entry of kernelEntries) { @@ -3144,6 +3396,7 @@ const hostNetImport = { } if (ready > 0 || t === 0 || (deadline != null && Date.now() >= deadline)) { + dispatchPendingWasmSignals(); new DataView(instanceMemory.buffer).setUint32(Number(retReadyPtr) >>> 0, ready >>> 0, true); return 0; } @@ -3173,7 +3426,10 @@ const hostNetImport = { const numericType = Number(sockType) >>> 0; const numericProtocol = Number(protocol) >>> 0; - const fd = nextHostNetSocketFd++; + const fd = allocateHostNetSocketFd(); + if (fd == null) { + return WASI_ERRNO_MFILE; + } hostNetSockets.set(fd, { domain: numericDomain, sockType: numericType, @@ -3217,20 +3473,19 @@ const hostNetImport = { const nulAt = rawAddr.indexOf(String.fromCharCode(0)); if (nulAt >= 0) rawAddr = rawAddr.slice(0, nulAt); rawAddr = rawAddr.trim(); - // AF_UNIX path connect (e.g. X11 /tmp/.X11-unix/X0): the C library passes the - // raw sun_path, the same '/'-prefixed string net_bind/net_listen receive. Route it - // to the sidecar's path-based net.connect, which dials the host-backed unix socket - // the listener bound under the VM sandbox root, so two guests in one VM can talk. - if (rawAddr.startsWith('/')) { + // AF_UNIX addresses use an explicit wire prefix so relative paths and paths containing ':' + // cannot be mistaken for TCP host:port strings. + const unixPath = parseHostNetUnixAddress(rawAddr); + if (unixPath != null) { let result; try { - result = callSyncRpc('net.connect', [{ path: rawAddr }]); + result = callSyncRpc('net.connect', [{ path: unixPath }]); } catch (e) { - try { process.stderr.write('[host_net] connect ' + rawAddr + ' failed: ' + (e && e.message ? e.message : String(e)) + '\n'); } catch (_) {} + try { process.stderr.write('[host_net] connect ' + unixPath + ' failed: ' + (e && e.message ? e.message : String(e)) + '\n'); } catch (_) {} return WASI_ERRNO_FAULT; } if (!result || typeof result.socketId !== 'string') { - try { process.stderr.write('[host_net] ' + rawAddr + ' returned no socketId\n'); } catch (_) {} + try { process.stderr.write('[host_net] ' + unixPath + ' returned no socketId\n'); } catch (_) {} return WASI_ERRNO_FAULT; } socket.socketId = result.socketId; @@ -3424,6 +3679,9 @@ const hostNetImport = { pumpSpawnedChildren(10); } } + if (accepted.error != null) { + return accepted.error; + } if (writeGuestUint32(retFdPtr, accepted.acceptedFd) !== WASI_ERRNO_SUCCESS) { return WASI_ERRNO_FAULT; } @@ -3488,15 +3746,14 @@ const hostNetImport = { } try { - if ((Number(flags) >>> 0) !== 0) { - // Non-zero recv flags are currently ignored in the WASM host_net shim. - } + const recvFlags = Number(flags) >>> 0; + const peek = (recvFlags & HOST_NET_MSG_PEEK) !== 0; // Non-blocking sockets (O_NONBLOCK via net_set_nonblock, used by libxcb's poll_for_*): // pull whatever is queued, do ONE short readiness probe, and return EAGAIN if still empty // instead of blocking. libxcb assumes its "poll" reads never block on an empty socket. if (socket.nonblock) { - let queued = dequeueHostNetBytes(socket, bufLen); + let queued = peek ? peekHostNetBytes(socket, bufLen) : dequeueHostNetBytes(socket, bufLen); if (queued.length > 0) { return writeGuestBytes(bufPtr, bufLen, queued, retReceivedPtr); } @@ -3505,7 +3762,7 @@ const hostNetImport = { return writeGuestUint32(retReceivedPtr, 0); } pollHostNetSocket(socket, 0); - queued = dequeueHostNetBytes(socket, bufLen); + queued = peek ? peekHostNetBytes(socket, bufLen) : dequeueHostNetBytes(socket, bufLen); if (queued.length > 0) { return writeGuestBytes(bufPtr, bufLen, queued, retReceivedPtr); } @@ -3518,7 +3775,7 @@ const hostNetImport = { const deadline = socket.recvTimeoutMs == null ? null : Date.now() + Math.max(0, socket.recvTimeoutMs); while (true) { - const queued = dequeueHostNetBytes(socket, bufLen); + const queued = peek ? peekHostNetBytes(socket, bufLen) : dequeueHostNetBytes(socket, bufLen); if (queued.length > 0) { return writeGuestBytes(bufPtr, bufLen, queued, retReceivedPtr); } @@ -3672,6 +3929,32 @@ const hostNetImport = { } return WASI_ERRNO_SUCCESS; }, + net_getsockopt(fd, level, optname, optvalPtr, optvalLenPtr) { + const socket = getHostNetSocket(fd); + if (!socket || socket.closed) { + return WASI_ERRNO_BADF; + } + + try { + const optvalLen = readGuestUint32(optvalLenPtr); + const normalizedLevel = Number(level) >>> 0; + const normalizedOptname = Number(optname) >>> 0; + if ( + (normalizedLevel === HOST_NET_SOL_SOCKET || + normalizedLevel === HOST_NET_WASI_SOL_SOCKET) && + normalizedOptname === HOST_NET_SO_ERROR + ) { + if (optvalLen < 4) { + return WASI_ERRNO_INVAL; + } + new DataView(instanceMemory.buffer).setInt32(Number(optvalPtr) >>> 0, 0, true); + return writeGuestUint32(optvalLenPtr, 4); + } + return WASI_ERRNO_INVAL; + } catch { + return WASI_ERRNO_FAULT; + } + }, net_close(fd) { const numericFd = Number(fd) >>> 0; const socket = hostNetSockets.get(numericFd); @@ -3695,7 +3978,7 @@ const hostNetImport = { return WASI_ERRNO_FAULT; } }, - net_tls_connect(fd, hostnamePtr, hostnameLen) { + net_tls_connect(fd, hostnamePtr, hostnameLen, flags = 0) { const socket = getHostNetSocket(fd); if (!socket?.socketId || socket.closed) { return WASI_ERRNO_BADF; @@ -3704,7 +3987,7 @@ const hostNetImport = { try { const servername = readGuestString(hostnamePtr, hostnameLen); const tlsOptions = { servername }; - if (guestEnv.NODE_TLS_REJECT_UNAUTHORIZED === '0') { + if ((Number(flags) & 1) === 1 || guestEnv.NODE_TLS_REJECT_UNAUTHORIZED === '0') { tlsOptions.rejectUnauthorized = false; } callSyncRpc('net.socket_upgrade_tls', [ @@ -3720,6 +4003,8 @@ const hostNetImport = { const hostProcessImport = { proc_spawn( + execPathPtr, + execPathLen, argvPtr, argvLen, envpPtr, @@ -3734,13 +4019,28 @@ const hostProcessImport = { if (permissionTier !== 'full') { return WASI_ERRNO_FAULT; } + traceHostProcess('proc-spawn-call', { + execPathPtr: Number(execPathPtr) >>> 0, + execPathLen: Number(execPathLen) >>> 0, + argvPtr: Number(argvPtr) >>> 0, + argvLen: Number(argvLen) >>> 0, + envpPtr: Number(envpPtr) >>> 0, + envpLen: Number(envpLen) >>> 0, + stdinFd: Number(stdinFd) >>> 0, + stdoutFd: Number(stdoutFd) >>> 0, + stderrFd: Number(stderrFd) >>> 0, + cwdPtr: Number(cwdPtr) >>> 0, + cwdLen: Number(cwdLen) >>> 0, + retPidPtr: Number(retPidPtr) >>> 0, + }); try { - const argv = decodeNullSeparatedStrings(readGuestBytes(argvPtr, argvLen)); - if (argv.length === 0) { + const command = readGuestString(execPathPtr, execPathLen); + if (command.length === 0) { return WASI_ERRNO_FAULT; } - - const [command, ...args] = argv; + const argv = decodeNullSeparatedStrings(readGuestBytes(argvPtr, argvLen)); + const argv0 = argv[0] ?? command; + const args = argv.slice(1); const env = parseSerializedEnv(readGuestBytes(envpPtr, envpLen)); const cwd = Number(cwdLen) > 0 ? readGuestString(cwdPtr, cwdLen) : undefined; @@ -3768,6 +4068,7 @@ const hostProcessImport = { } traceHostProcess('proc-spawn-begin', { command, + argv0, args, cwd: cwd ?? null, stdinFd: Number(stdinFd) >>> 0, @@ -3797,6 +4098,7 @@ const hostProcessImport = { command, args, options: { + argv0, cwd, env, internalBootstrapEnv: {}, @@ -3854,6 +4156,8 @@ const hostProcessImport = { stdinReadyAtMs: Date.now() + 100, delegateRetainedFds, retainedSpawnOutputHandles, + exitCode: null, + exitSignal: null, exitStatus: null, }; spawnedChildren.set(pid, record); @@ -3862,6 +4166,7 @@ const hostProcessImport = { command, childId: result.childId, pid, + resolvedArgs: result.args ?? null, }); if (stdinRedirectBytes != null) { if (stdinRedirectBytes.length > 0) { @@ -3882,7 +4187,7 @@ const hostProcessImport = { return WASI_ERRNO_FAULT; } }, - proc_waitpid(pid, options, retStatusPtr, retPidPtr) { + proc_waitpid(pid, options, retExitCodePtr, retSignalPtr, retPidPtr) { const requestedPid = Number(pid) >>> 0; if (permissionTier !== 'full') { return requestedPid === 0xffffffff ? WASI_ERRNO_CHILD : WASI_ERRNO_SRCH; @@ -3903,7 +4208,10 @@ const hostProcessImport = { pid: record.pid, }); if (typeof record.exitStatus === 'number') { - if (writeGuestUint32(retStatusPtr, record.exitStatus) !== WASI_ERRNO_SUCCESS) { + if (writeGuestUint32(retExitCodePtr, record.exitCode ?? 0) !== WASI_ERRNO_SUCCESS) { + return WASI_ERRNO_FAULT; + } + if (writeGuestUint32(retSignalPtr, record.exitSignal ?? 0) !== WASI_ERRNO_SUCCESS) { return WASI_ERRNO_FAULT; } const writePidResult = writeGuestUint32(retPidPtr, record.pid); @@ -3956,7 +4264,10 @@ const hostProcessImport = { if (event.type === 'exit') { processChildEvent(record, event); - if (writeGuestUint32(retStatusPtr, record.exitStatus ?? 1) !== WASI_ERRNO_SUCCESS) { + if (writeGuestUint32(retExitCodePtr, record.exitCode ?? 0) !== WASI_ERRNO_SUCCESS) { + return WASI_ERRNO_FAULT; + } + if (writeGuestUint32(retSignalPtr, record.exitSignal ?? 0) !== WASI_ERRNO_SUCCESS) { return WASI_ERRNO_FAULT; } const writePidResult = writeGuestUint32(retPidPtr, record.pid); @@ -4026,8 +4337,8 @@ const hostProcessImport = { readHandleCount: 0, writeHandleCount: 0, }; - const readFd = nextSyntheticFd++; - const writeFd = nextSyntheticFd++; + const readFd = allocateSyntheticFd(); + const writeFd = allocateSyntheticFd(); syntheticFdEntries.set(readFd, createPipeHandle('pipe-read', pipe, readFd)); syntheticFdEntries.set(writeFd, createPipeHandle('pipe-write', pipe, writeFd)); if (writeGuestUint32(retReadFdPtr, readFd) !== WASI_ERRNO_SUCCESS) { @@ -4044,20 +4355,7 @@ const hostProcessImport = { if (!handle) { return WASI_ERRNO_BADF; } - let duplicatedFd = 0; - while ( - duplicatedFd <= 2 && - ( - syntheticFdEntries.has(duplicatedFd) || - passthroughHandles.has(duplicatedFd) || - delegateManagedFdRefCounts.has(duplicatedFd) - ) - ) { - duplicatedFd += 1; - } - if (duplicatedFd > 2) { - duplicatedFd = nextSyntheticFd++; - } + const duplicatedFd = allocateSyntheticFd(0); syntheticFdEntries.set(duplicatedFd, handle); traceHostProcess('fd-dup', { fd: Number(fd) >>> 0, @@ -4129,15 +4427,7 @@ const hostProcessImport = { return WASI_ERRNO_BADF; } - let duplicatedFd = minimumFdNumber >>> 0; - while ( - syntheticFdEntries.has(duplicatedFd) || - passthroughHandles.has(duplicatedFd) || - delegateManagedFdRefCounts.has(duplicatedFd) - ) { - duplicatedFd += 1; - } - nextSyntheticFd = Math.max(nextSyntheticFd, duplicatedFd + 1); + const duplicatedFd = allocateSyntheticFd(minimumFdNumber); syntheticFdEntries.set(duplicatedFd, handle); traceHostProcess('fd-dup-min', { @@ -4153,9 +4443,42 @@ const hostProcessImport = { return WASI_ERRNO_FAULT; } }, + proc_closefrom(lowFd) { + const minimumFd = Number(lowFd) >>> 0; + const openVirtualFds = new Set([ + ...syntheticFdEntries.keys(), + ...passthroughHandles.keys(), + ...hostNetSockets.keys(), + ...delegateManagedFdRefCounts.keys(), + ]); + let firstError = WASI_ERRNO_SUCCESS; + for (const fd of [...openVirtualFds].sort((left, right) => left - right)) { + if (fd < minimumFd) { + continue; + } + const result = wasiImport.fd_close(fd); + if ( + result !== WASI_ERRNO_SUCCESS && + result !== WASI_ERRNO_BADF && + firstError === WASI_ERRNO_SUCCESS + ) { + firstError = result; + } + } + return firstError; + }, sleep_ms(milliseconds) { try { - Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, Number(milliseconds) >>> 0); + const waitArray = new Int32Array(new SharedArrayBuffer(4)); + const deadline = Date.now() + (Number(milliseconds) >>> 0); + while (Date.now() < deadline) { + // Keep guest sleeps interruptible by V8 termination during SIGTERM, + // SIGKILL, and VM disposal. Also drain handled Wasm signals at + // syscall boundaries so cooperative handlers run during sleeps. + dispatchPendingWasmSignals(); + Atomics.wait(waitArray, 0, 0, Math.max(1, Math.min(10, deadline - Date.now()))); + } + dispatchPendingWasmSignals(); return WASI_ERRNO_SUCCESS; } catch { return WASI_ERRNO_FAULT; @@ -4174,11 +4497,12 @@ const hostProcessImport = { mask: decodeSignalMask(maskLo, maskHi), flags: Number(flags) >>> 0, }; - emitControlMessage({ - type: 'signal_state', - signal: Number(signal) >>> 0, - registration, - }); + callSyncRpc('process.signal_state', [ + Number(signal) >>> 0, + registration.action, + JSON.stringify(registration.mask), + registration.flags, + ]); return WASI_ERRNO_SUCCESS; } catch { return WASI_ERRNO_FAULT; @@ -4418,14 +4742,14 @@ const hostFsImport = { try { const target = resolvePathOpenGuestPath(fd, pathPtr, pathLen); if (typeof target !== 'string') { - return 1; + return WASI_ERRNO_NOENT; } const mapping = resolveHostFsMapping(target); if (!mapping || typeof mapping.hostPath !== 'string') { - return 1; + return WASI_ERRNO_NOENT; } if (mapping.readOnly) { - return 1; + return WASI_ERRNO_ROFS; } traceHostProcess('host-fs-chmod', { target, @@ -4434,9 +4758,11 @@ const hostFsImport = { }); chmodMappedGuestPath(target, mapping.hostPath, Number(mode) >>> 0); return 0; - } catch { - traceHostProcess('host-fs-chmod-fault', {}); - return 1; + } catch (error) { + traceHostProcess('host-fs-chmod-fault', { + message: error instanceof Error ? error.message : String(error), + }); + return mapSyntheticFsError(error); } }, fchmod(fd, mode) { @@ -4444,12 +4770,15 @@ const hostFsImport = { const descriptor = Number(fd) >>> 0; const handle = lookupFdHandle(descriptor); if (handle?.readOnly === true) { - return 1; + return WASI_ERRNO_ROFS; } if (typeof handle?.guestPath === 'string') { const mapping = resolveHostFsMapping(handle.guestPath); - if (!mapping || typeof mapping.hostPath !== 'string' || mapping.readOnly) { - return 1; + if (!mapping || typeof mapping.hostPath !== 'string') { + return WASI_ERRNO_NOENT; + } + if (mapping.readOnly) { + return WASI_ERRNO_ROFS; } chmodMappedGuestPath(handle.guestPath, mapping.hostPath, Number(mode) >>> 0); return 0; @@ -4458,9 +4787,11 @@ const hostFsImport = { typeof handle?.targetFd === 'number' ? Number(handle.targetFd) >>> 0 : descriptor; fsModule.fchmodSync(targetFd, Number(mode) >>> 0); return 0; - } catch { - traceHostProcess('host-fs-fchmod-fault', {}); - return 1; + } catch (error) { + traceHostProcess('host-fs-fchmod-fault', { + message: error instanceof Error ? error.message : String(error), + }); + return mapSyntheticFsError(error); } }, ftruncate(fd, length) { @@ -4804,6 +5135,10 @@ const delegateManagedFdClose = typeof wasiImport.fd_close === 'function' ? wasiImport.fd_close.bind(wasiImport) : null; +const delegateManagedFdRenumber = + typeof wasiImport.fd_renumber === 'function' + ? wasiImport.fd_renumber.bind(wasiImport) + : null; const delegateManagedFdPrestatGet = typeof wasiImport.fd_prestat_get === 'function' ? wasiImport.fd_prestat_get.bind(wasiImport) @@ -4823,6 +5158,11 @@ const KERNEL_POLLHUP = 0x0010; wasiImport.fd_read = (fd, iovs, iovsLen, nreadPtr) => { const numericFd = Number(fd) >>> 0; + const hostNetSocket = getHostNetSocket(numericFd); + if (hostNetSocket) { + return readHostNetSocketToGuestIovs(hostNetSocket, iovs, iovsLen, nreadPtr); + } + const handle = __agentOSWasiMeasurePhase('fd_read', 'lookup_handle', () => lookupFdHandle(numericFd) ); @@ -4994,11 +5334,13 @@ wasiImport.fd_read = (fd, iovs, iovsLen, nreadPtr) => { } if (handle?.kind === 'passthrough') { - return delegateManagedFdRead - ? __agentOSWasiMeasurePhase('fd_read', 'delegate_call', () => - delegateManagedFdRead(handle.targetFd, iovs, iovsLen, nreadPtr) - ) - : WASI_ERRNO_BADF; + if (!delegateManagedFdRead) { + return WASI_ERRNO_BADF; + } + const result = __agentOSWasiMeasurePhase('fd_read', 'delegate_call', () => + delegateManagedFdRead(handle.targetFd, iovs, iovsLen, nreadPtr) + ); + return result; } if (rejectClosedPassthroughFd(numericFd)) { @@ -5072,9 +5414,10 @@ wasiImport.fd_pread = (fd, iovs, iovsLen, offset, nreadPtr) => { return mapSyntheticFsError(error); } } - return delegateFdPread - ? delegateFdPread(handle.targetFd, iovs, iovsLen, offset, nreadPtr) - : WASI_ERRNO_BADF; + if (!delegateFdPread) { + return WASI_ERRNO_BADF; + } + return delegateFdPread(handle.targetFd, iovs, iovsLen, offset, nreadPtr); } if (rejectClosedPassthroughFd(fd)) { @@ -5089,6 +5432,9 @@ wasiImport.fd_pread = (fd, iovs, iovsLen, offset, nreadPtr) => { wasiImport.fd_pwrite = (fd, iovs, iovsLen, offset, nwrittenPtr) => { const handle = lookupFdHandle(fd); if (handle?.kind === 'guest-file') { + if (handle.readOnly === true) { + return WASI_ERRNO_ROFS; + } try { const bytes = collectGuestIovBytes(iovs, iovsLen); const written = fsModule.writeSync( @@ -5293,11 +5639,50 @@ wasiImport.fd_fdstat_get = (fd, statPtr) => { ); } + if (handle?.kind === 'guest-file') { + try { + const stat = fsModule.fstatSync(handle.targetFd); + return writeGuestFdstat( + statPtr, + wasiFiletypeFromStats(stat), + 0, + WASI_RIGHT_FD_READ | + WASI_RIGHT_FD_SEEK | + WASI_RIGHT_FD_TELL | + WASI_RIGHT_FD_FILESTAT_GET | + WASI_RIGHT_FD_WRITE | + WASI_RIGHT_FD_SYNC, + 0n, + ); + } catch (error) { + return mapSyntheticFsError(error); + } + } + if (handle && handle.kind !== 'passthrough') { return WASI_ERRNO_BADF; } if (handle?.kind === 'passthrough') { + if (typeof handle.ioFd === 'number') { + try { + const stat = fsModule.fstatSync(handle.ioFd); + return writeGuestFdstat( + statPtr, + wasiFiletypeFromStats(stat), + 0, + WASI_RIGHT_FD_READ | + WASI_RIGHT_FD_SEEK | + WASI_RIGHT_FD_TELL | + WASI_RIGHT_FD_FILESTAT_GET | + WASI_RIGHT_FD_WRITE | + WASI_RIGHT_FD_SYNC, + 0n, + ); + } catch (error) { + return mapSyntheticFsError(error); + } + } return delegateManagedFdFdstatGet ? __agentOSWasiMeasurePhase('fd_fdstat_get', 'delegate_call', () => delegateManagedFdFdstatGet(handle.targetFd, statPtr) @@ -5355,9 +5740,10 @@ wasiImport.fd_filestat_get = (fd, statPtr) => { return mapSyntheticFsError(error); } } - return delegateManagedFdFilestatGet - ? delegateManagedFdFilestatGet(handle.targetFd, statPtr) - : WASI_ERRNO_BADF; + if (!delegateManagedFdFilestatGet) { + return WASI_ERRNO_BADF; + } + return delegateManagedFdFilestatGet(handle.targetFd, statPtr); } if (rejectClosedPassthroughFd(fd)) { @@ -5477,10 +5863,15 @@ wasiImport.fd_prestat_dir_name = (fd, pathPtr, pathLen) => { }; wasiImport.fd_write = (fd, iovs, iovsLen, nwrittenPtr) => { + const numericFd = Number(fd) >>> 0; + const hostNetSocket = getHostNetSocket(numericFd); + if (hostNetSocket) { + return writeHostNetSocketFromGuestIovs(hostNetSocket, iovs, iovsLen, nwrittenPtr); + } + const handle = __agentOSWasiMeasurePhase('fd_write', 'lookup_handle', () => lookupFdHandle(fd) ); - const numericFd = Number(fd) >>> 0; if (handle?.kind === 'pipe-write') { try { const bytes = __agentOSWasiMeasurePhase('fd_write', 'guest_iov_collect', () => @@ -5502,6 +5893,9 @@ wasiImport.fd_write = (fd, iovs, iovsLen, nwrittenPtr) => { } if (handle?.kind === 'guest-file') { + if (handle.readOnly === true) { + return WASI_ERRNO_ROFS; + } try { const bytes = __agentOSWasiMeasurePhase('fd_write', 'guest_iov_collect', () => collectGuestIovBytes(iovs, iovsLen) @@ -5595,11 +5989,17 @@ wasiImport.fd_write = (fd, iovs, iovsLen, nwrittenPtr) => { }; wasiImport.fd_close = (fd) => { + const numericFd = Number(fd) >>> 0; traceHostProcess('fd-close-begin', { - fd: Number(fd) >>> 0, - syntheticKind: syntheticFdEntries.get(Number(fd) >>> 0)?.kind ?? null, - passthroughKind: passthroughHandles.get(Number(fd) >>> 0)?.kind ?? null, + fd: numericFd, + syntheticKind: syntheticFdEntries.get(numericFd)?.kind ?? null, + passthroughKind: passthroughHandles.get(numericFd)?.kind ?? null, }); + if (hostNetSockets.has(numericFd)) { + return __agentOSWasiMeasurePhase('fd_close', 'host_socket_close', () => + hostNetImport.net_close(numericFd) + ); + } if (__agentOSWasiMeasurePhase('fd_close', 'synthetic_close', () => closeSyntheticFd(fd))) { traceHostProcess('fd-close-synthetic', { fd: Number(fd) >>> 0 }); return WASI_ERRNO_SUCCESS; @@ -5648,6 +6048,66 @@ wasiImport.fd_close = (fd) => { : WASI_ERRNO_BADF; }; +wasiImport.fd_renumber = (from, to) => { + try { + const sourceFd = Number(from) >>> 0; + const targetFd = Number(to) >>> 0; + if (sourceFd === targetFd) { + return lookupFdHandle(sourceFd) || delegateManagedFdRefCounts.has(sourceFd) + ? WASI_ERRNO_SUCCESS + : WASI_ERRNO_BADF; + } + + const syntheticHandle = syntheticFdEntries.get(sourceFd); + const passthroughHandle = passthroughHandles.get(sourceFd); + const retainedSpawnOutputHandle = retainedSpawnOutputHandlesByFd.get(sourceFd); + if (!syntheticHandle && !passthroughHandle && !retainedSpawnOutputHandle) { + if (rejectClosedPassthroughFd(sourceFd)) { + return WASI_ERRNO_BADF; + } + return delegateManagedFdRenumber + ? delegateManagedFdRenumber(sourceFd, targetFd) + : WASI_ERRNO_BADF; + } + + if ( + syntheticFdEntries.has(targetFd) || + passthroughHandles.has(targetFd) || + retainedSpawnOutputHandlesByFd.has(targetFd) || + delegateManagedFdRefCounts.has(targetFd) + ) { + const closeResult = wasiImport.fd_close(targetFd); + if (closeResult !== WASI_ERRNO_SUCCESS) { + return closeResult; + } + } + + if (syntheticHandle) { + syntheticFdEntries.delete(sourceFd); + syntheticFdEntries.set(targetFd, syntheticHandle); + } else if (passthroughHandle) { + passthroughHandles.delete(sourceFd); + passthroughHandles.set(targetFd, passthroughHandle); + closedPassthroughFds.add(sourceFd); + closedPassthroughFds.delete(targetFd); + } else { + retainedSpawnOutputHandlesByFd.delete(sourceFd); + retainedSpawnOutputHandlesByFd.set(targetFd, retainedSpawnOutputHandle); + } + + nextSyntheticFd = Math.max(nextSyntheticFd, targetFd + 1); + traceHostProcess('fd-renumber', { + from: sourceFd, + to: targetFd, + syntheticKind: syntheticHandle?.kind ?? null, + passthroughKind: passthroughHandle?.kind ?? null, + }); + return WASI_ERRNO_SUCCESS; + } catch { + return WASI_ERRNO_FAULT; + } +}; + wasiImport.poll_oneoff = (inPtr, outPtr, nsubscriptions, neventsPtr) => { if (!(instanceMemory instanceof WebAssembly.Memory)) { return delegateManagedPollOneoff @@ -5728,7 +6188,9 @@ wasiImport.poll_oneoff = (inPtr, outPtr, nsubscriptions, neventsPtr) => { }); } - if (!hasSyntheticSubscription && !hasRemappedPassthroughSubscription) { + const hasClockSubscription = subscriptions.some((subscription) => subscription.kind === 'clock'); + + if (!hasSyntheticSubscription && !hasRemappedPassthroughSubscription && !hasClockSubscription) { return delegateManagedPollOneoff ? delegateManagedPollOneoff(inPtr, outPtr, nsubscriptions, neventsPtr) : WASI_ERRNO_BADF; @@ -5775,8 +6237,17 @@ wasiImport.poll_oneoff = (inPtr, outPtr, nsubscriptions, neventsPtr) => { pollTargets, Math.max(0, Number(waitMs) >>> 0), ]); - } catch { - return []; + } catch (error) { + traceHostProcess('kernel-poll-error', { + message: error instanceof Error ? error.message : String(error), + }); + return subscriptions.map((subscription) => ({ + userdata: subscription.userdata, + error: WASI_ERRNO_FAULT, + type: subscription.kind === 'fd_read' ? 1 : 2, + nbytes: 0, + flags: 0, + })); } const responseEntries = Array.isArray(response?.fds) ? response.fds : []; @@ -5812,6 +6283,7 @@ wasiImport.poll_oneoff = (inPtr, outPtr, nsubscriptions, neventsPtr) => { } while (readyEvents.length === 0) { + dispatchPendingWasmSignals(); for (const subscription of subscriptions) { if (subscription.error != null) { readyEvents.push({ @@ -5895,7 +6367,7 @@ wasiImport.poll_oneoff = (inPtr, outPtr, nsubscriptions, neventsPtr) => { ); } - if (readyEvents.length === 0 && subscriptions.some((subscription) => subscription.kind === 'clock')) { + if (readyEvents.length === 0 && hasClockSubscription) { const clockSubscription = subscriptions.find((subscription) => subscription.kind === 'clock'); readyEvents.push({ userdata: clockSubscription.userdata, @@ -6036,6 +6508,27 @@ function dispatchWasmSignal(signal) { } } +function dispatchPendingWasmSignals() { + while (pendingWasmSignals.length > 0) { + dispatchWasmSignal(pendingWasmSignals.shift()); + } + while (true) { + let signal; + try { + signal = callSyncRpc('process.take_signal', []); + } catch (error) { + if (error?.code === 'ERR_AGENTOS_WASM_SYNC_RPC_UNAVAILABLE') { + return; + } + throw error; + } + if (typeof signal !== 'number') { + return; + } + dispatchWasmSignal(signal); + } +} + Object.defineProperty(globalThis, '__secureExecWasmSignalDispatch', { configurable: true, writable: true, @@ -6044,7 +6537,9 @@ Object.defineProperty(globalThis, '__secureExecWasmSignalDispatch', { typeof payload?.number === 'number' ? payload.number : signalNumberFromName(payload?.signal); - dispatchWasmSignal(signal); + if (signal > 0) { + pendingWasmSignals.push(signal); + } }, }); diff --git a/crates/execution/src/node_import_cache.rs b/crates/execution/src/node_import_cache.rs index 4703e281ed..fbe47a1c4e 100644 --- a/crates/execution/src/node_import_cache.rs +++ b/crates/execution/src/node_import_cache.rs @@ -15,7 +15,7 @@ const NODE_IMPORT_CACHE_PATH_ENV: &str = "AGENTOS_NODE_IMPORT_CACHE_PATH"; const NODE_IMPORT_CACHE_LOADER_PATH_ENV: &str = "AGENTOS_NODE_IMPORT_CACHE_LOADER_PATH"; const NODE_IMPORT_CACHE_SCHEMA_VERSION: &str = "1"; const NODE_IMPORT_CACHE_LOADER_VERSION: &str = "8"; -const NODE_IMPORT_CACHE_ASSET_VERSION: &str = "85"; +const NODE_IMPORT_CACHE_ASSET_VERSION: &str = "96"; const NODE_IMPORT_CACHE_DIR_PREFIX: &str = "agentos-node-import-cache"; const DEFAULT_NODE_IMPORT_CACHE_MATERIALIZE_TIMEOUT: Duration = Duration::from_secs(30); const PYODIDE_DIST_DIR: &str = "pyodide-dist"; diff --git a/crates/execution/src/wasm.rs b/crates/execution/src/wasm.rs index 8a7a806dc2..3658ae7a0e 100644 --- a/crates/execution/src/wasm.rs +++ b/crates/execution/src/wasm.rs @@ -2094,7 +2094,11 @@ fn translate_wasm_signal_state_sync_rpc_request( let flags = request .args .get(3) - .and_then(Value::as_u64) + .and_then(|value| { + value + .as_u64() + .or_else(|| value.as_i64().map(|signed| signed as u64)) + }) .unwrap_or_default() as u32; execution @@ -2852,6 +2856,11 @@ if (typeof globalThis !== "undefined") {{ flags, ]); }} + case "process.take_signal": + if (typeof _processTakeSignal === "undefined") {{ + throw new Error("secure-exec WASM signal-drain bridge is unavailable"); + }} + return _processTakeSignal.applySync(void 0, args); default: throw new Error(`secure-exec WASM sync RPC method not implemented in V8 runtime: ${{method}}`); }} diff --git a/crates/kernel/src/fd_table.rs b/crates/kernel/src/fd_table.rs index 3e57be8ec9..f42eb2b109 100644 --- a/crates/kernel/src/fd_table.rs +++ b/crates/kernel/src/fd_table.rs @@ -583,6 +583,15 @@ impl ProcessFdTable { child.next_fd = self.next_fd; for (fd, entry) in &self.entries { + // Kernel process creation is spawn (fork + exec combined), so + // close-on-exec descriptors must not leak into the child. This + // matters for pipe write ends: an inherited writer keeps the + // pipe's writer refcount above zero forever, so a blocked reader + // (for example a grandchild sharing the parent's stdin pipe) + // would never observe EOF. + if entry.fd_flags & FD_CLOEXEC != 0 { + continue; + } entry.description.increment_ref_count(); child.entries.insert( *fd, diff --git a/crates/native-sidecar/Cargo.toml b/crates/native-sidecar/Cargo.toml index c2f7826540..00afcd1b39 100644 --- a/crates/native-sidecar/Cargo.toml +++ b/crates/native-sidecar/Cargo.toml @@ -74,3 +74,7 @@ serde_bare = "0.5" tar = "0.4" wat = "1.0" v8 = "130" + +[build-dependencies] +base64 = "0.22" +webpki-root-certs = "1.0.8" diff --git a/crates/native-sidecar/src/execution.rs b/crates/native-sidecar/src/execution.rs index 049d7565ed..0aa6737213 100644 --- a/crates/native-sidecar/src/execution.rs +++ b/crates/native-sidecar/src/execution.rs @@ -41,10 +41,10 @@ use crate::state::{ JavascriptTlsDataValue, JavascriptTlsMaterial, JavascriptUdpFamily, JavascriptUdpSocketEvent, JavascriptUnixListenerEvent, KernelSocketReadinessEvent, KernelSocketReadinessRegistry, KernelSocketReadinessTarget, LoopbackTlsPendingWriteHandle, LoopbackTlsPendingWriteState, - NetworkResourceCounts, PendingTcpSocket, PendingUnixSocket, ProcNetEntry, ProcessEventEnvelope, - PythonHostSocket, ResolvedChildProcessExecution, ResolvedTcpConnectAddr, SharedBridge, - SharedSidecarRequestClient, SidecarKernel, SocketQueryKind, ToolExecution, VmDnsConfig, - VmListenPolicy, VmState, DEFAULT_JAVASCRIPT_NET_BACKLOG, EXECUTION_DRIVER_NAME, + NetworkResourceCounts, PendingKernelStdin, PendingTcpSocket, PendingUnixSocket, ProcNetEntry, + ProcessEventEnvelope, PythonHostSocket, ResolvedChildProcessExecution, ResolvedTcpConnectAddr, + SharedBridge, SharedSidecarRequestClient, SidecarKernel, SocketQueryKind, ToolExecution, + VmDnsConfig, VmListenPolicy, VmState, DEFAULT_JAVASCRIPT_NET_BACKLOG, EXECUTION_DRIVER_NAME, EXECUTION_SANDBOX_ROOT_ENV, JAVASCRIPT_COMMAND, LOOPBACK_EXEMPT_PORTS_ENV, MAPPED_HOST_FD_START, PYTHON_COMMAND, TOOL_DRIVER_NAME, VM_LISTEN_ALLOW_PRIVILEGED_METADATA_KEY, WASM_COMMAND, WASM_STDIO_SYNC_RPC_ENV, @@ -471,6 +471,7 @@ impl ActiveProcess { kernel_pid, kernel_handle, kernel_stdin_writer_fd: None, + pending_kernel_stdin: PendingKernelStdin::default(), tty_master_fd: None, runtime, detached: false, @@ -483,6 +484,7 @@ impl ActiveProcess { next_mapped_host_fd: MAPPED_HOST_FD_START, pending_execution_events: VecDeque::new(), pending_self_signal_exit: None, + pending_wasm_signals: VecDeque::new(), child_processes: BTreeMap::new(), next_child_process_id: 0, http_servers: BTreeMap::new(), @@ -528,6 +530,14 @@ impl ActiveProcess { Ok(()) } + pub(crate) fn queue_pending_wasm_signal(&mut self, signal: i32) -> Result<(), SidecarError> { + if self.pending_wasm_signals.len() >= MAX_PROCESS_EVENT_QUEUE { + return Err(process_event_queue_overflow_error()); + } + self.pending_wasm_signals.push_back(signal); + Ok(()) + } + pub(crate) fn with_host_cwd(mut self, host_cwd: PathBuf) -> Self { self.host_cwd = host_cwd; self @@ -761,6 +771,10 @@ fn closed_wasm_event_channel(message: &str) -> bool { message == WasmExecutionError::EventChannelClosed.to_string() } +fn is_expected_v8_termination_diagnostic(chunk: &[u8]) -> bool { + chunk == b"Error: Execution terminated\n" +} + fn missing_vm_error(vm_id: &str) -> SidecarError { SidecarError::InvalidState(format!("VM {vm_id} is no longer active")) } @@ -4542,6 +4556,8 @@ where let (connection_id, session_id, vm_id) = self.vm_scope_for(&request.ownership)?; self.require_owned_vm(&connection_id, &session_id, &vm_id)?; + self.drain_root_signal_state_events(&vm_id, &payload.process_id)?; + let handlers = self .vms .get(&vm_id) @@ -4555,6 +4571,77 @@ where }) } + fn drain_root_signal_state_events( + &mut self, + vm_id: &str, + process_id: &str, + ) -> Result<(), SidecarError> { + let mut deferred = VecDeque::new(); + + loop { + let event = { + let Some(vm) = self.vms.get_mut(vm_id) else { + break; + }; + let Some(process) = vm.active_processes.get_mut(process_id) else { + break; + }; + if let Some(event) = process.pending_execution_events.pop_front() { + Some(event) + } else { + match process.execution.poll_event_blocking(Duration::ZERO) { + Ok(event) => event, + Err(SidecarError::Execution(message)) + if (process.runtime == GuestRuntimeKind::JavaScript + && closed_javascript_event_channel(&message)) + || (process.runtime == GuestRuntimeKind::Python + && closed_python_event_channel(&message)) + || (process.runtime == GuestRuntimeKind::WebAssembly + && closed_wasm_event_channel(&message)) => + { + None + } + Err(error) => return Err(error), + } + } + }; + + let Some(event) = event else { + break; + }; + + match event { + ActiveExecutionEvent::SignalState { + signal, + registration, + } => { + if let Some(vm) = self.vms.get_mut(vm_id) { + apply_process_signal_state_update( + &mut vm.signal_states, + process_id, + signal, + registration, + ); + } + } + other => deferred.push_back(other), + } + } + + if let Some(vm) = self.vms.get_mut(vm_id) { + if let Some(process) = vm.active_processes.get_mut(process_id) { + for event in deferred.into_iter().rev() { + if process.pending_execution_events.len() >= MAX_PROCESS_EVENT_QUEUE { + return Err(process_event_queue_overflow_error()); + } + process.pending_execution_events.push_front(event); + } + } + } + + Ok(()) + } + pub(crate) async fn get_zombie_timer_count( &mut self, request: &RequestFrame, @@ -4646,7 +4733,7 @@ where KillBehavior::SharedV8DispatchOrTerminate } ActiveExecution::Wasm(execution) if execution.uses_shared_v8_runtime() => { - KillBehavior::SharedV8Terminate + KillBehavior::SharedV8DispatchOrTerminate } ActiveExecution::Python(execution) if execution.uses_shared_v8_runtime() => { KillBehavior::SharedV8Terminate @@ -4699,8 +4786,37 @@ where } } KillBehavior::SharedV8DispatchOrTerminate => { - if signal != 0 && !dispatch_v8_process_signal(process, signal)? { - process.execution.terminate()?; + if signal != 0 { + let is_shared_wasm = matches!( + &process.execution, + ActiveExecution::Wasm(execution) if execution.uses_shared_v8_runtime() + ); + if is_shared_wasm { + let action = vm + .signal_states + .get(process_id) + .and_then(|handlers| handlers.get(&(signal as u32))) + .map(|registration| ®istration.action); + match action { + Some(SignalDispositionAction::Ignore) => {} + Some(SignalDispositionAction::User) => { + process.queue_pending_wasm_signal(signal)?; + } + Some(SignalDispositionAction::Default) | None => { + if !matches!( + canonical_signal_name(signal), + Some("SIGWINCH" | "SIGCHLD" | "SIGURG") + ) { + process.execution.terminate()?; + process.queue_pending_execution_event( + ActiveExecutionEvent::Exited(128 + signal), + )?; + } + } + } + } else if !dispatch_v8_process_signal(process, signal)? { + process.execution.terminate()?; + } } } KillBehavior::Noop => {} @@ -5895,6 +6011,7 @@ where command, args: request.args.clone(), options: JavascriptChildProcessSpawnOptions { + argv0: None, cwd, env: request.env.clone(), input: None, @@ -6677,6 +6794,12 @@ where &parent_host_cwd, &mut request, )?; + if let (Some(argv0), Some(current_argv0)) = ( + request.options.argv0.as_ref(), + resolved.process_args.first_mut(), + ) { + current_argv0.clone_from(argv0); + } { let vm = self .vms @@ -6686,6 +6809,23 @@ where } let resolved = resolved; record_execute_phase("child_process_resolve_execution", phase_start.elapsed()); + let parent_sync_roots = { + let vm = self.vms.get(vm_id).ok_or_else(|| missing_vm_error(vm_id))?; + let parent = vm + .active_processes + .get(process_id) + .ok_or_else(|| missing_process_error(vm_id, process_id))?; + (parent.host_write_dirty_recursive() + || !parent.clean_host_writes_are_observable_recursive()) + .then(|| (parent.host_cwd.clone(), parent.guest_cwd.clone())) + }; + if let Some((host_cwd, guest_cwd)) = parent_sync_roots { + let vm = self + .vms + .get_mut(vm_id) + .ok_or_else(|| missing_vm_error(vm_id))?; + sync_process_host_roots_to_kernel(vm, &host_cwd, &guest_cwd)?; + } let (parent_kernel_pid, child_process_id) = { let vm = self .vms @@ -7188,6 +7328,12 @@ where &parent_host_cwd, &mut request, )?; + if let (Some(argv0), Some(current_argv0)) = ( + request.options.argv0.as_ref(), + resolved.process_args.first_mut(), + ) { + current_argv0.clone_from(argv0); + } { let vm = self .vms @@ -7197,6 +7343,29 @@ where } let resolved = resolved; record_execute_phase("child_process_resolve_execution", phase_start.elapsed()); + let parent_sync_roots = { + let vm = self.vms.get(vm_id).ok_or_else(|| missing_vm_error(vm_id))?; + let root = vm + .active_processes + .get(process_id) + .ok_or_else(|| missing_process_error(vm_id, process_id))?; + let parent = + Self::active_process_by_path(root, current_process_path).ok_or_else(|| { + SidecarError::InvalidState(format!( + "unknown child process path {current_process_label} during nested spawn" + )) + })?; + (parent.host_write_dirty_recursive() + || !parent.clean_host_writes_are_observable_recursive()) + .then(|| (parent.host_cwd.clone(), parent.guest_cwd.clone())) + }; + if let Some((host_cwd, guest_cwd)) = parent_sync_roots { + let vm = self + .vms + .get_mut(vm_id) + .ok_or_else(|| missing_vm_error(vm_id))?; + sync_process_host_roots_to_kernel(vm, &host_cwd, &guest_cwd)?; + } let sidecar_requests = self.sidecar_requests.clone(); let vm = self @@ -7837,6 +8006,11 @@ where { return Ok(false); } + // Top off the child's stdin pipe from any host-side backlog before + // probing readiness: the child draining the pipe is exactly what frees + // capacity for the next queued stdin bytes (and, once the backlog is + // empty, executes a deferred stdin close so EOF arrives in order). + flush_pending_kernel_stdin(kernel, child)?; let now = Instant::now(); let requested_timeout_ms = match request.method.as_str() { "__kernel_stdin_read" => parse_kernel_stdin_read_args(request)?.1, @@ -8085,6 +8259,26 @@ where })); } ActiveExecutionEvent::Stderr(chunk) => { + let suppress_termination_diagnostic = { + let Some(vm) = self.vms.get_mut(vm_id) else { + return Ok(Value::Null); + }; + let Some(parent) = Self::descendant_parent_process_mut( + vm, + process_id, + current_process_path, + ) else { + return Ok(Value::Null); + }; + let Some(child) = parent.child_processes.get(child_process_id) else { + return Ok(Value::Null); + }; + child.pending_self_signal_exit.is_some() + && is_expected_v8_termination_diagnostic(&chunk) + }; + if suppress_termination_diagnostic { + continue; + } return Ok(json!({ "type": "stderr", "data": javascript_sync_rpc_bytes_value(&chunk), @@ -8137,7 +8331,7 @@ where let Some(vm) = self.vms.get_mut(vm_id) else { return Ok(Value::Null); }; - let signal_name = { + let termination_signal = { let Some(parent) = Self::descendant_parent_process_mut( vm, process_id, @@ -8148,14 +8342,14 @@ where let Some(child) = parent.child_processes.get_mut(child_process_id) else { return Ok(Value::Null); }; - child.pending_self_signal_exit.take().and_then(|signal| { - if exit_code == 128 + signal { - canonical_signal_name(signal).map(str::to_owned) - } else { - None - } - }) + child.pending_self_signal_exit.take() }; + let signal_name = termination_signal + .and_then(canonical_signal_name) + .map(str::to_owned); + let process_exit_code = termination_signal + .map(|signal| 128 + signal) + .unwrap_or(exit_code); let (parent_runtime_pid, parent_v8_signal_session, should_signal_parent) = { let Some(parent) = Self::descendant_parent_process(vm, process_id, current_process_path) @@ -8195,7 +8389,7 @@ where release_inherited_child_raw_mode(&mut vm.kernel, &child)?; let kernel_readiness = Arc::clone(&vm.kernel_socket_readiness); terminate_child_process_tree(&mut vm.kernel, &mut child, &kernel_readiness); - child.kernel_handle.finish(exit_code); + child.kernel_handle.finish(process_exit_code); let _ = vm.kernel.wait_and_reap(child.kernel_pid); vm.signal_states.remove(child_process_id); for (detached_process_id, detached_child) in detached_children { @@ -8213,7 +8407,10 @@ where } let mut payload = Map::new(); payload.insert(String::from("type"), Value::String(String::from("exit"))); - payload.insert(String::from("exitCode"), Value::from(exit_code)); + payload.insert( + String::from("exitCode"), + termination_signal.map_or_else(|| Value::from(exit_code), |_| Value::Null), + ); if let Some(signal_name) = signal_name { payload.insert(String::from("signal"), Value::String(signal_name)); } @@ -8620,6 +8817,11 @@ where let Some(vm) = self.vms.get_mut(vm_id) else { return Ok(()); }; + let registration = vm + .signal_states + .get(child_process_id) + .and_then(|handlers| handlers.get(&(signal as u32))) + .map(|registration| registration.action.clone()); let Some(root) = vm.active_processes.get_mut(process_id) else { return Ok(()); }; @@ -8630,7 +8832,7 @@ where let Some(child) = parent.child_processes.get_mut(child_process_id) else { return Ok(()); }; - terminate_tracked_child_process_for_signal(&mut vm.kernel, child, signal)?; + signal_tracked_child_process(&mut vm.kernel, child, signal, registration.as_ref())?; let child_process_label = if current_process_path.is_empty() { child_process_id.to_owned() } else { @@ -8804,9 +9006,10 @@ where "ESRCH: unknown process pid {target_pid}" ))); }; - if let Some(session) = target.execution.javascript_v8_session_handle().filter( - |_| matches!(&target.execution, ActiveExecution::Javascript(execution) if execution.uses_shared_v8_runtime()) - || matches!(&target.execution, ActiveExecution::Wasm(execution) if execution.uses_shared_v8_runtime()), + if matches!(&target.execution, ActiveExecution::Wasm(execution) if execution.uses_shared_v8_runtime()) { + target.queue_pending_wasm_signal(signal)?; + } else if let Some(session) = target.execution.javascript_v8_session_handle().filter( + |_| matches!(&target.execution, ActiveExecution::Javascript(execution) if execution.uses_shared_v8_runtime()), ) { dispatch_v8_session_signal_async(session, signal); } else if !dispatch_v8_process_signal(target, signal)? { @@ -8959,6 +9162,11 @@ where let Some(vm) = self.vms.get_mut(vm_id) else { return Ok(()); }; + let registration = vm + .signal_states + .get(child_process_id) + .and_then(|handlers| handlers.get(&(signal as u32))) + .map(|registration| registration.action.clone()); let process = vm .active_processes .get_mut(process_id) @@ -8972,7 +9180,7 @@ where "unknown child process {child_process_id} during kill" )) })?; - terminate_tracked_child_process_for_signal(&mut vm.kernel, child, signal)?; + signal_tracked_child_process(&mut vm.kernel, child, signal, registration.as_ref())?; emit_security_audit_event( &self.bridge, vm_id, @@ -9034,6 +9242,12 @@ where let Some(vm) = self.vms.get_mut(vm_id) else { return Ok(()); }; + let signal_key = path.last().map(String::as_str).unwrap_or(&process_id); + let registration = vm + .signal_states + .get(signal_key) + .and_then(|handlers| handlers.get(&(signal as u32))) + .map(|registration| registration.action.clone()); let Some(root) = vm.active_processes.get_mut(&process_id) else { return Ok(()); }; @@ -9042,7 +9256,12 @@ where "ESRCH: no such process {target_kernel_pid}" ))); }; - terminate_tracked_child_process_for_signal(&mut vm.kernel, target, signal)?; + signal_tracked_child_process( + &mut vm.kernel, + target, + signal, + registration.as_ref(), + )?; emit_security_audit_event( &self.bridge, vm_id, @@ -9132,37 +9351,38 @@ where } } -/// Applies a kill signal to a tracked child execution. Shared-runtime -/// executions for lethal signals are terminated directly with a synthetic -/// signal exit so child polls observe a prompt close; everything else routes -/// through the kernel process table. -fn terminate_tracked_child_process_for_signal( +/// Applies a signal to a tracked child using the same default/ignore/user +/// disposition rules as Linux. Default-terminating signals preserve their +/// signal identity separately from the shell-style backend exit code. +fn signal_tracked_child_process( kernel: &mut SidecarKernel, child: &mut ActiveProcess, signal: i32, + registration: Option<&SignalDispositionAction>, ) -> Result<(), SidecarError> { - let should_terminate_shared_runtime = child.execution.uses_shared_v8_runtime() - && signal != 0 - && !matches!( - signal, - libc::SIGHUP - | libc::SIGINT - | libc::SIGTERM - | libc::SIGCHLD - | libc::SIGWINCH - | libc::SIGSTOP - | libc::SIGCONT - ); - if should_terminate_shared_runtime { - child.execution.terminate()?; - child.pending_self_signal_exit = Some(signal); - child.queue_pending_execution_event(ActiveExecutionEvent::Exited(128 + signal))?; - } else { - kernel + if signal == 0 || matches!(signal, libc::SIGSTOP | libc::SIGCONT) { + return kernel .kill_process(EXECUTION_DRIVER_NAME, child.kernel_pid, signal) - .map_err(kernel_error)?; + .map_err(kernel_error); + } + + match registration { + Some(SignalDispositionAction::Ignore) => Ok(()), + Some(SignalDispositionAction::User) => kernel + .kill_process(EXECUTION_DRIVER_NAME, child.kernel_pid, signal) + .map_err(kernel_error), + Some(SignalDispositionAction::Default) | None + if matches!( + canonical_signal_name(signal), + Some("SIGWINCH" | "SIGCHLD" | "SIGURG") + ) => + { + Ok(()) + } + Some(SignalDispositionAction::Default) | None => { + apply_active_process_default_signal(kernel, child, signal) + } } - Ok(()) } fn sidecar_error_is_esrch(error: &SidecarError) -> bool { @@ -9186,15 +9406,23 @@ fn apply_active_process_default_signal( if process.execution.uses_shared_v8_runtime() { process.execution.terminate()?; - if signal != 0 && matches!(process.execution, ActiveExecution::Wasm(_)) { - process.queue_pending_execution_event(ActiveExecutionEvent::Exited(128 + signal))?; + if signal != 0 { + process.pending_self_signal_exit = Some(signal); + if matches!(process.execution, ActiveExecution::Wasm(_)) { + process + .queue_pending_execution_event(ActiveExecutionEvent::Exited(128 + signal))?; + } } return Ok(()); } kernel .kill_process(EXECUTION_DRIVER_NAME, process.kernel_pid, signal) - .map_err(kernel_error) + .map_err(kernel_error)?; + if signal != 0 { + process.pending_self_signal_exit = Some(signal); + } + Ok(()) } fn map_wasm_signal_registration( @@ -9885,8 +10113,7 @@ pub(crate) fn sync_active_process_host_writes_to_kernel( vm: &mut VmState, ) -> Result<(), SidecarError> { if vm.root_filesystem_mode != RootFilesystemMode::ReadOnly { - let shadow_root = vm.cwd.clone(); - sync_host_directory_tree_to_kernel(vm, &shadow_root, "/")?; + sync_vm_shadow_root_to_kernel(vm)?; } let normalized_vm_root = normalize_host_path(&vm.cwd); @@ -9898,6 +10125,88 @@ pub(crate) fn sync_active_process_host_writes_to_kernel( Ok(()) } +/// Syncs the VM shadow root into the kernel VFS and reconciles deletions. +/// +/// The walk itself is additive (it copies shadow entries into the kernel), so +/// on its own it can never express a guest deletion performed directly on the +/// shadow tree — a removed file would be resurrected forever. To get real +/// Linux semantics the walk records every guest path it sees and diffs that +/// set against the previous walk's inventory: paths that were present in the +/// shadow before and are gone now are removed from the kernel VFS as well. +/// +/// Deletion propagation is scoped to the guest-writable shadow region: it only +/// runs for the VM shadow root walk (never for external host roots), re-checks +/// the protected/mount skip list, never touches the POSIX bootstrap skeleton, +/// and removes directories non-recursively so a kernel directory that still +/// has kernel-only content (for example a mount point or a path that was never +/// materialized into the shadow) is left in place with a warning. +fn sync_vm_shadow_root_to_kernel(vm: &mut VmState) -> Result<(), SidecarError> { + let shadow_root = normalize_host_path(&vm.cwd); + if host_sync_root_is_filesystem_root(&shadow_root) { + tracing::warn!("skipping host shadow sync rooted at the host filesystem root"); + return Ok(()); + } + let mut synced_file_times = BTreeMap::new(); + let mut inventory = BTreeSet::new(); + sync_host_directory_tree_to_kernel_inner( + vm, + &shadow_root, + &shadow_root, + "/", + &mut synced_file_times, + Some(&mut inventory), + )?; + propagate_shadow_deletions_to_kernel(vm, &inventory); + vm.shadow_sync_inventory = inventory; + Ok(()) +} + +/// Removes kernel paths whose shadow copy disappeared since the last walk. +/// +/// Best-effort by design: a failure to reconcile one stale path must not +/// poison the guest filesystem operation that triggered the sync, so failures +/// are surfaced as host-visible warnings instead of errors. Children sort +/// after their parents in the `BTreeSet`, so the reverse iteration removes +/// leaves before the directories that contain them. +fn propagate_shadow_deletions_to_kernel(vm: &mut VmState, current: &BTreeSet) { + if vm.shadow_sync_inventory.is_empty() { + return; + } + let stale: Vec = vm + .shadow_sync_inventory + .iter() + .rev() + .filter(|path| !current.contains(*path)) + .cloned() + .collect(); + for path in stale { + if path == "/" || should_skip_shadow_sync_path(vm, &path) || is_shadow_bootstrap_dir(&path) + { + continue; + } + let stat = match vm.kernel.lstat(&path) { + Ok(stat) => stat, + // Already gone (for example removed through the kernel-direct + // guest fs path, which mirrors deletions into the shadow itself). + Err(_) => continue, + }; + let result = if stat.is_directory && !stat.is_symbolic_link { + vm.kernel.remove_dir(&path) + } else { + vm.kernel.remove_file(&path) + }; + if let Err(error) = result { + if error.code() != "ENOENT" { + tracing::warn!( + path = %path, + error = %error, + "failed to propagate guest shadow deletion into the kernel VFS" + ); + } + } + } +} + fn collect_active_process_host_sync_roots( vm: &VmState, normalized_vm_root: &Path, @@ -9934,17 +10243,24 @@ fn collect_process_host_sync_roots( fn sync_process_host_writes_to_kernel( vm: &mut VmState, process: &ActiveProcess, +) -> Result<(), SidecarError> { + sync_process_host_roots_to_kernel(vm, &process.host_cwd, &process.guest_cwd) +} + +fn sync_process_host_roots_to_kernel( + vm: &mut VmState, + process_host_cwd: &Path, + process_guest_cwd: &str, ) -> Result<(), SidecarError> { if vm.root_filesystem_mode != RootFilesystemMode::ReadOnly { - let shadow_root = vm.cwd.clone(); - sync_host_directory_tree_to_kernel(vm, &shadow_root, "/")?; + sync_vm_shadow_root_to_kernel(vm)?; } if !path_is_within_root( - &normalize_host_path(&process.host_cwd), + &normalize_host_path(process_host_cwd), &normalize_host_path(&vm.cwd), ) { - sync_host_directory_tree_to_kernel(vm, &process.host_cwd, &process.guest_cwd)?; + sync_host_directory_tree_to_kernel(vm, process_host_cwd, process_guest_cwd)?; } Ok(()) @@ -9976,6 +10292,7 @@ fn sync_host_directory_tree_to_kernel( &normalized_host_root, &normalized_guest_root, &mut synced_file_times, + None, ) } @@ -9985,6 +10302,7 @@ fn sync_host_directory_tree_to_kernel_inner( current_host_dir: &Path, guest_root: &str, synced_file_times: &mut BTreeMap<(u64, u64), (u64, u64)>, + mut inventory: Option<&mut BTreeSet>, ) -> Result<(), SidecarError> { let entries = match fs::read_dir(current_host_dir) { Ok(entries) => entries, @@ -10046,6 +10364,15 @@ fn sync_host_directory_tree_to_kernel_inner( continue; } + // Record every shadow entry we can represent in the kernel (dirs, + // files, symlinks) so the deletion reconcile can tell "was in the + // shadow, now gone" apart from "never came from the shadow". + if let Some(inventory) = inventory.as_deref_mut() { + if file_type.is_dir() || file_type.is_file() || file_type.is_symlink() { + inventory.insert(guest_path.clone()); + } + } + if file_type.is_dir() { let metadata = entry.metadata().map_err(|error| { SidecarError::Io(format!( @@ -10081,6 +10408,7 @@ fn sync_host_directory_tree_to_kernel_inner( &host_path, guest_root, synced_file_times, + inventory.as_deref_mut(), )?; continue; } @@ -11645,9 +11973,10 @@ mod runtime_guest_path_mapping_tests { #[cfg(test)] mod kernel_poll_sync_rpc_tests { use super::{ - service_javascript_kernel_poll_sync_rpc, ActiveExecution, ActiveProcess, - JavascriptSyncRpcRequest, KernelPollFdResponse, SidecarKernel, ToolExecution, - EXECUTION_DRIVER_NAME, JAVASCRIPT_COMMAND, + close_kernel_process_stdin, service_javascript_kernel_poll_sync_rpc, + service_javascript_kernel_stdin_sync_rpc, write_kernel_process_stdin, ActiveExecution, + ActiveProcess, JavascriptSyncRpcRequest, KernelPollFdResponse, SidecarKernel, + ToolExecution, EXECUTION_DRIVER_NAME, JAVASCRIPT_COMMAND, }; use agentos_kernel::command_registry::CommandDriver; use agentos_kernel::kernel::{KernelVmConfig, SpawnOptions}; @@ -11655,6 +11984,7 @@ mod kernel_poll_sync_rpc_tests { use agentos_kernel::permissions::Permissions; use agentos_kernel::poll::{POLLHUP, POLLIN}; use agentos_kernel::vfs::MemoryFileSystem; + use base64::Engine as _; use serde_json::{json, Value}; use std::collections::HashMap; #[test] @@ -11691,7 +12021,7 @@ mod kernel_poll_sync_rpc_tests { .fd_close(EXECUTION_DRIVER_NAME, pid, stdin_read_fd) .expect("close original stdin read fd"); - let process = ActiveProcess::new( + let mut process = ActiveProcess::new( pid, kernel_handle, super::GuestRuntimeKind::JavaScript, @@ -11707,7 +12037,7 @@ mod kernel_poll_sync_rpc_tests { let response = service_javascript_kernel_poll_sync_rpc( &mut kernel, - &process, + &mut process, &JavascriptSyncRpcRequest { id: 1, method: String::from("__kernel_poll"), @@ -11745,6 +12075,106 @@ mod kernel_poll_sync_rpc_tests { process.kernel_handle.finish(0); kernel.waitpid(pid).expect("wait javascript kernel process"); } + + #[test] + fn top_level_kernel_stdin_redrives_backlog_and_deferred_eof() { + let mut config = KernelVmConfig::new("vm-top-level-stdin-backlog"); + config.permissions = Permissions::allow_all(); + let mut kernel = SidecarKernel::new(MountTable::new(MemoryFileSystem::new()), config); + kernel + .register_driver(CommandDriver::new( + EXECUTION_DRIVER_NAME, + [JAVASCRIPT_COMMAND], + )) + .expect("register execution driver"); + let kernel_handle = kernel + .spawn_process( + JAVASCRIPT_COMMAND, + Vec::new(), + SpawnOptions { + requester_driver: Some(String::from(EXECUTION_DRIVER_NAME)), + ..SpawnOptions::default() + }, + ) + .expect("spawn kernel process"); + let pid = kernel_handle.pid(); + let (read_fd, writer_fd) = kernel + .open_pipe(EXECUTION_DRIVER_NAME, pid) + .expect("open stdin pipe"); + kernel + .fd_dup2(EXECUTION_DRIVER_NAME, pid, read_fd, 0) + .expect("dup stdin read end onto fd 0"); + kernel + .fd_close(EXECUTION_DRIVER_NAME, pid, read_fd) + .expect("close original stdin read fd"); + kernel + .fd_fcntl( + EXECUTION_DRIVER_NAME, + pid, + writer_fd, + agentos_kernel::fd_table::F_SETFL, + agentos_kernel::fd_table::O_NONBLOCK, + ) + .expect("make stdin writer nonblocking"); + + let mut process = ActiveProcess::new( + pid, + kernel_handle, + super::GuestRuntimeKind::WebAssembly, + ActiveExecution::Tool(ToolExecution::default()), + ) + .with_kernel_stdin_writer_fd(writer_fd); + let payload = (0..(65_536 + 8192)) + .map(|index| (index % 251) as u8) + .collect::>(); + + write_kernel_process_stdin(&mut kernel, &mut process, &payload) + .expect("queue stdin larger than the pipe"); + assert!(process.pending_kernel_stdin.total > 0); + close_kernel_process_stdin(&mut kernel, &mut process).expect("defer stdin close"); + + let mut received = Vec::new(); + for id in 1..=2 { + let response = service_javascript_kernel_stdin_sync_rpc( + &mut kernel, + &mut process, + &JavascriptSyncRpcRequest { + id, + method: String::from("__kernel_stdin_read"), + raw_bytes_args: HashMap::new(), + args: vec![json!(65_536), json!(0)], + }, + ) + .expect("read queued stdin"); + let encoded = response["dataBase64"] + .as_str() + .expect("stdin response should contain data"); + received.extend( + base64::engine::general_purpose::STANDARD + .decode(encoded) + .expect("decode stdin response"), + ); + } + assert_eq!(received, payload); + assert_eq!(process.pending_kernel_stdin.total, 0); + assert!(process.kernel_stdin_writer_fd.is_none()); + + let eof = service_javascript_kernel_stdin_sync_rpc( + &mut kernel, + &mut process, + &JavascriptSyncRpcRequest { + id: 3, + method: String::from("__kernel_stdin_read"), + raw_bytes_args: HashMap::new(), + args: vec![json!(1), json!(0)], + }, + ) + .expect("observe stdin eof"); + assert_eq!(eof, json!({ "done": true })); + + process.kernel_handle.finish(0); + kernel.waitpid(pid).expect("wait kernel process"); + } } fn dedupe_strings(values: &[String]) -> Vec { @@ -17026,6 +17456,10 @@ where "action": "default", })) } + "process.take_signal" => { + let signal = process.pending_wasm_signals.pop_front(); + Ok(signal.map(Value::from).unwrap_or(Value::Null).into()) + } "process.umask" => { let new_mask = javascript_sync_rpc_arg_u32_optional(&request.args, 0, "process umask")?; kernel @@ -19341,13 +19775,19 @@ fn service_javascript_kernel_stdin_sync_rpc( process: &mut ActiveProcess, request: &JavascriptSyncRpcRequest, ) -> Result { + // A previous guest read may have freed pipe capacity. Re-drive the host + // backlog before and after this read so a top-level process drains queued + // stdin and observes a deferred EOF exactly like the child-process path. + flush_pending_kernel_stdin(kernel, process)?; let (max_bytes, timeout_ms) = parse_kernel_stdin_read_args(request)?; - kernel_stdin_read_response( + let response = kernel_stdin_read_response( kernel, process.kernel_pid, max_bytes, Duration::from_millis(timeout_ms), - ) + ); + flush_pending_kernel_stdin(kernel, process)?; + response } /// Parse `__kernel_stdin_read` args: (max bytes, requested timeout ms). @@ -19587,11 +20027,14 @@ fn service_javascript_kernel_stdio_write_sync_rpc( fn service_javascript_kernel_poll_sync_rpc( kernel: &mut SidecarKernel, - process: &ActiveProcess, + process: &mut ActiveProcess, request: &JavascriptSyncRpcRequest, ) -> Result { + flush_pending_kernel_stdin(kernel, process)?; let (fd_requests, timeout_ms) = parse_kernel_poll_args(request)?; - kernel_poll_response(kernel, process.kernel_pid, &fd_requests, timeout_ms) + let response = kernel_poll_response(kernel, process.kernel_pid, &fd_requests, timeout_ms); + flush_pending_kernel_stdin(kernel, process)?; + response } /// Parse `__kernel_poll` args: (fd list, requested timeout ms). @@ -19662,6 +20105,36 @@ fn install_kernel_stdin_pipe(kernel: &mut SidecarKernel, pid: u32) -> Result git remote-https -> git-remote-https). + kernel + .fd_fcntl( + EXECUTION_DRIVER_NAME, + pid, + write_fd, + agentos_kernel::fd_table::F_SETFD, + agentos_kernel::fd_table::FD_CLOEXEC, + ) + .map_err(kernel_error)?; + // Non-blocking writes only: the sidecar dispatch thread feeds this pipe + // from `write_kernel_process_stdin` / `flush_pending_kernel_stdin`, and a + // blocking pipe write would deadlock — the child's pipe reads are parked + // sync RPCs serviced by this same thread. A full pipe surfaces EAGAIN and + // the remainder is queued in `ActiveProcess::pending_kernel_stdin`. + kernel + .fd_fcntl( + EXECUTION_DRIVER_NAME, + pid, + write_fd, + agentos_kernel::fd_table::F_SETFL, + agentos_kernel::fd_table::O_NONBLOCK, + ) + .map_err(kernel_error)?; Ok(write_fd) } @@ -19686,6 +20159,13 @@ fn javascript_child_process_stdin_mode(request: &JavascriptChildProcessSpawnRequ .unwrap_or("pipe") } +/// Host-side cap on stdin bytes queued for a child's kernel pipe. Matches the +/// kernel's default per-write bound (`resource.max_fd_write_bytes`); a parent +/// cannot park more than this behind a slow child before getting a typed +/// error. +const MAX_PENDING_KERNEL_STDIN_BYTES: usize = + agentos_kernel::resource_accounting::DEFAULT_MAX_FD_WRITE_BYTES; + pub(crate) fn write_kernel_process_stdin( kernel: &mut SidecarKernel, process: &mut ActiveProcess, @@ -19701,17 +20181,115 @@ pub(crate) fn write_kernel_process_stdin( let Some(writer_fd) = process.kernel_stdin_writer_fd else { return Ok(()); }; - kernel - .fd_write(EXECUTION_DRIVER_NAME, process.kernel_pid, writer_fd, chunk) - .map_err(kernel_error)?; - // For a TTY process the master write above drives line-discipline echo into - // the master output buffer. Drain it now and surface it as the single - // ordered Stdout stream so typed-character echo reaches the host even while - // the guest is blocked in read() and not producing output of its own. - if let Some(echo) = drain_tty_master_output(kernel, process)? { - process.queue_pending_execution_event(ActiveExecutionEvent::Stdout(echo))?; - } - forward_tty_slave_input_to_javascript(kernel, process)?; + if process.tty_master_fd.is_some() { + kernel + .fd_write(EXECUTION_DRIVER_NAME, process.kernel_pid, writer_fd, chunk) + .map_err(kernel_error)?; + // For a TTY process the master write above drives line-discipline echo + // into the master output buffer. Drain it now and surface it as the + // single ordered Stdout stream so typed-character echo reaches the + // host even while the guest is blocked in read() and not producing + // output of its own. + if let Some(echo) = drain_tty_master_output(kernel, process)? { + process.queue_pending_execution_event(ActiveExecutionEvent::Stdout(echo))?; + } + forward_tty_slave_input_to_javascript(kernel, process)?; + return Ok(()); + } + // Pipe-backed stdin. The kernel pipe caps at MAX_PIPE_BUFFER_BYTES and + // fd_write reports POSIX partial writes, so anything the pipe cannot take + // right now is queued and flushed as the child drains (see + // `flush_pending_kernel_stdin`). Silently dropping the remainder is how + // multi-buffer stdin payloads (e.g. git's spooled pack piped into + // `index-pack --stdin`) used to truncate at 64 KiB. + let pending_total = process.pending_kernel_stdin.total; + if pending_total.saturating_add(chunk.len()) > MAX_PENDING_KERNEL_STDIN_BYTES { + return Err(SidecarError::InvalidState(format!( + "child process stdin backlog limit exceeded: {pending_total} pending + {} new bytes \ + > {MAX_PENDING_KERNEL_STDIN_BYTES} (MAX_PENDING_KERNEL_STDIN_BYTES); the child is \ + not draining stdin — write smaller chunks after the child consumes them, or raise \ + this bound together with the kernel resource.max_fd_write_bytes limit", + chunk.len(), + ))); + } + process.pending_kernel_stdin.push(chunk); + flush_pending_kernel_stdin(kernel, process) +} + +/// Write as much queued stdin as the child's kernel pipe can take right now. +/// Called on every stdin write and from the child kernel-wait servicing path +/// (each parked `__kernel_stdin_read` / `__kernel_poll` re-check), so the +/// backlog drains in lockstep with the child's reads. Executes the deferred +/// stdin close once the backlog is empty. +pub(crate) fn flush_pending_kernel_stdin( + kernel: &mut SidecarKernel, + process: &mut ActiveProcess, +) -> Result<(), SidecarError> { + if process.tty_master_fd.is_some() { + return Ok(()); + } + let Some(writer_fd) = process.kernel_stdin_writer_fd else { + process.pending_kernel_stdin.clear(); + process.pending_kernel_stdin.close_requested = false; + return Ok(()); + }; + while let Some(front) = process.pending_kernel_stdin.chunks.pop_front() { + let offset = process.pending_kernel_stdin.front_offset; + let slice = &front[offset..]; + match kernel.fd_write(EXECUTION_DRIVER_NAME, process.kernel_pid, writer_fd, slice) { + Ok(written) if written >= slice.len() => { + process.pending_kernel_stdin.total = process + .pending_kernel_stdin + .total + .saturating_sub(slice.len()); + process.pending_kernel_stdin.front_offset = 0; + } + Ok(written) => { + // Pipe is full mid-chunk; keep the remainder queued. + process.pending_kernel_stdin.total = + process.pending_kernel_stdin.total.saturating_sub(written); + process.pending_kernel_stdin.front_offset = offset + written; + process.pending_kernel_stdin.chunks.push_front(front); + break; + } + Err(error) if error.code() == "EAGAIN" => { + process.pending_kernel_stdin.chunks.push_front(front); + break; + } + Err(error) if error.code() == "EPIPE" => { + // Reader side is gone (child exited or closed stdin). Release + // the writer, but preserve Linux write(2) behavior by returning + // EPIPE to the caller instead of reporting that discarded + // backlog bytes were accepted. + process.pending_kernel_stdin.clear(); + process.pending_kernel_stdin.close_requested = false; + process.kernel_stdin_writer_fd = None; + if let Err(close_error) = + kernel.fd_close(EXECUTION_DRIVER_NAME, process.kernel_pid, writer_fd) + { + tracing::warn!( + process_id = process.kernel_pid, + fd = writer_fd, + error = %close_error, + "failed to close child stdin after EPIPE" + ); + } + return Err(kernel_error(error)); + } + Err(error) => { + process.pending_kernel_stdin.chunks.push_front(front); + return Err(kernel_error(error)); + } + } + } + if process.pending_kernel_stdin.is_empty() && process.pending_kernel_stdin.close_requested { + process.pending_kernel_stdin.close_requested = false; + if let Some(writer_fd) = process.kernel_stdin_writer_fd.take() { + kernel + .fd_close(EXECUTION_DRIVER_NAME, process.kernel_pid, writer_fd) + .map_err(kernel_error)?; + } + } Ok(()) } @@ -19758,6 +20336,13 @@ pub(crate) fn close_kernel_process_stdin( kernel: &mut SidecarKernel, process: &mut ActiveProcess, ) -> Result<(), SidecarError> { + if !process.pending_kernel_stdin.is_empty() && process.kernel_stdin_writer_fd.is_some() { + // Queued stdin has not reached the pipe yet; closing now would hand + // the child a premature EOF. `flush_pending_kernel_stdin` performs the + // close once the backlog drains. + process.pending_kernel_stdin.close_requested = true; + return Ok(()); + } let Some(writer_fd) = process.kernel_stdin_writer_fd.take() else { return Ok(()); }; @@ -24418,6 +25003,13 @@ fn dispatch_v8_process_signal(process: &ActiveProcess, signal: i32) -> Result, #[serde(default)] cwd: Option, #[serde(default)] @@ -209,6 +211,7 @@ pub(crate) fn parse_javascript_child_process_spawn_request( command, args: parsed_args, options: JavascriptChildProcessSpawnOptions { + argv0: parsed_options.argv0, cwd: parsed_options.cwd, env: parsed_options.env, internal_bootstrap_env: sanitize_javascript_child_process_internal_bootstrap_env( diff --git a/crates/native-sidecar/src/state.rs b/crates/native-sidecar/src/state.rs index e0dd8be9db..7d36c2d1a6 100644 --- a/crates/native-sidecar/src/state.rs +++ b/crates/native-sidecar/src/state.rs @@ -470,6 +470,7 @@ pub(crate) struct ActiveProcess { pub(crate) next_mapped_host_fd: u32, pub(crate) pending_execution_events: VecDeque, pub(crate) pending_self_signal_exit: Option, + pub(crate) pending_wasm_signals: VecDeque, pub(crate) child_processes: BTreeMap, pub(crate) next_child_process_id: usize, pub(crate) http_servers: BTreeMap, diff --git a/crates/native-sidecar/src/vm.rs b/crates/native-sidecar/src/vm.rs index be3afd17c3..63772a67d3 100644 --- a/crates/native-sidecar/src/vm.rs +++ b/crates/native-sidecar/src/vm.rs @@ -109,6 +109,17 @@ const SHADOW_ROOT_BOOTSTRAP_DIRS: &[(&str, u32)] = &[ ("/workspace", 0o755), ]; +/// Versioned Mozilla CA bundle generated by `build.rs` and embedded from +/// `OUT_DIR`. It is written into the VM shadow tree so in-guest TLS clients use +/// `/etc/ssl/certs/ca-certificates.crt`, matching the conventional Linux path. +const CA_CERTIFICATES_BUNDLE: &[u8] = + include_bytes!(concat!(env!("OUT_DIR"), "/ca-certificates.crt")); +const CA_CERTIFICATES_GUEST_PATH: &str = "/etc/ssl/certs/ca-certificates.crt"; +/// Conventional `/etc/ssl/cert.pem` -> `certs/ca-certificates.crt` symlink that +/// OpenSSL's default `OPENSSLDIR` layout expects. +const CA_CERTIFICATES_SYMLINK_PATH: &str = "/etc/ssl/cert.pem"; +const CA_CERTIFICATES_SYMLINK_TARGET: &str = "certs/ca-certificates.crt"; + fn send_kernel_socket_readiness_event( target: KernelSocketReadinessTarget, readiness: SocketReadiness, @@ -1753,6 +1764,60 @@ fn bootstrap_shadow_root(root: &Path) -> Result<(), SidecarError> { )) })?; } + seed_ca_certificates_bundle(root)?; + Ok(()) +} + +/// Seed the Mozilla CA bundle into the shadow root at +/// `/etc/ssl/certs/ca-certificates.crt` (plus the conventional +/// `/etc/ssl/cert.pem` symlink) so guest TLS clients resolve trust the standard +/// Linux way. +fn seed_ca_certificates_bundle(root: &Path) -> Result<(), SidecarError> { + if CA_CERTIFICATES_BUNDLE.is_empty() { + return Err(SidecarError::Io( + "embedded Mozilla CA certificate bundle is empty".to_string(), + )); + } + + let bundle_path = shadow_path_for_guest(root, CA_CERTIFICATES_GUEST_PATH); + if let Some(parent) = bundle_path.parent() { + fs::create_dir_all(parent).map_err(|error| { + SidecarError::Io(format!( + "failed to create shadow CA certs directory {}: {error}", + parent.display() + )) + })?; + } + fs::write(&bundle_path, CA_CERTIFICATES_BUNDLE).map_err(|error| { + SidecarError::Io(format!( + "failed to seed CA bundle {}: {error}", + bundle_path.display() + )) + })?; + fs::set_permissions(&bundle_path, fs::Permissions::from_mode(0o644)).map_err(|error| { + SidecarError::Io(format!( + "failed to set CA bundle mode on {}: {error}", + bundle_path.display() + )) + })?; + + let symlink_path = shadow_path_for_guest(root, CA_CERTIFICATES_SYMLINK_PATH); + match fs::remove_file(&symlink_path) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + return Err(SidecarError::Io(format!( + "failed to replace CA bundle symlink {}: {error}", + symlink_path.display() + ))); + } + } + std::os::unix::fs::symlink(CA_CERTIFICATES_SYMLINK_TARGET, &symlink_path).map_err(|error| { + SidecarError::Io(format!( + "failed to seed CA bundle symlink {}: {error}", + symlink_path.display() + )) + })?; Ok(()) } @@ -2150,7 +2215,9 @@ mod tests { use super::{ bootstrap_native_root_filesystem, bootstrap_shadow_root, materialize_shadow_root_snapshot_entries, native_root_plugin_from_config, - prune_kernel_command_stub, shadow_path_for_guest, KERNEL_COMMAND_STUB, + prune_kernel_command_stub, shadow_path_for_guest, CA_CERTIFICATES_BUNDLE, + CA_CERTIFICATES_GUEST_PATH, CA_CERTIFICATES_SYMLINK_PATH, CA_CERTIFICATES_SYMLINK_TARGET, + KERNEL_COMMAND_STUB, }; use crate::plugins::chunked_local::ChunkedLocalMountPlugin; use crate::protocol::{ @@ -2167,6 +2234,7 @@ mod tests { use agentos_kernel::vfs::VirtualFileSystem; use std::fs; use std::os::unix::fs::PermissionsExt; + use std::path::Path; use std::time::{SystemTime, UNIX_EPOCH}; #[test] @@ -2207,6 +2275,36 @@ mod tests { fs::remove_dir_all(&root).expect("temp shadow root should be removed"); } + #[test] + fn bootstrap_shadow_root_seeds_ca_bundle_when_present() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("clock should be monotonic") + .as_nanos(); + let root = std::env::temp_dir().join(format!("agentos-native-sidecar-ca-test-{unique}")); + fs::create_dir_all(&root).expect("temp shadow root should be created"); + + bootstrap_shadow_root(&root).expect("shadow bootstrap should succeed"); + + let bundle = shadow_path_for_guest(&root, CA_CERTIFICATES_GUEST_PATH); + let symlink = shadow_path_for_guest(&root, CA_CERTIFICATES_SYMLINK_PATH); + + assert!(!CA_CERTIFICATES_BUNDLE.is_empty()); + let seeded = fs::read(&bundle).expect("CA bundle should be seeded"); + assert_eq!( + seeded, CA_CERTIFICATES_BUNDLE, + "seeded CA bundle should match the embedded asset" + ); + let target = fs::read_link(&symlink).expect("cert.pem symlink should be seeded"); + assert_eq!( + target, + Path::new(CA_CERTIFICATES_SYMLINK_TARGET), + "cert.pem should point at certs/ca-certificates.crt" + ); + + fs::remove_dir_all(&root).expect("temp shadow root should be removed"); + } + #[test] fn native_root_config_opens_chunked_local_as_persistent_root() { let unique = SystemTime::now() diff --git a/crates/native-sidecar/tests/filesystem.rs b/crates/native-sidecar/tests/filesystem.rs index d0503985fe..e414eff247 100644 --- a/crates/native-sidecar/tests/filesystem.rs +++ b/crates/native-sidecar/tests/filesystem.rs @@ -424,12 +424,12 @@ mod shadow_root { .join("../..") .canonicalize() .expect("canonicalize repo root"); - let copied = repo_root.join("registry/software/coreutils/wasm"); + let copied = repo_root.join("software/coreutils/wasm"); if copied.exists() { return Some(copied.to_string_lossy().into_owned()); } - let fallback = repo_root.join("registry/native/target/wasm32-wasip1/release/commands"); + let fallback = repo_root.join("toolchain/target/wasm32-wasip1/release/commands"); if fallback.exists() { return Some(fallback.to_string_lossy().into_owned()); } diff --git a/crates/native-sidecar/tests/posix_path_repro.rs b/crates/native-sidecar/tests/posix_path_repro.rs index 3bec55fdc8..00ae319966 100644 --- a/crates/native-sidecar/tests/posix_path_repro.rs +++ b/crates/native-sidecar/tests/posix_path_repro.rs @@ -5,7 +5,6 @@ use agentos_native_sidecar::wire::{ }; use serde_json::{json, Value}; use std::collections::HashMap; -use std::fs; use std::path::{Path, PathBuf}; use std::process::Command; use std::time::Duration; @@ -30,19 +29,33 @@ const ALLOWED_NODE_BUILTINS: &[&str] = &[ "util", ]; +fn strip_benign_child_pid_warnings(stderr: &str) -> String { + stderr + .lines() + .filter(|line| !line.contains("WARN") || !line.contains("could not retrieve pid")) + .collect::>() + .join("\n") +} + fn registry_command_root() -> PathBuf { let repo_root = PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("../..") .canonicalize() .expect("canonicalize repo root"); - let commands = repo_root.join("registry/native/target/wasm32-wasip1/release/commands"); - if commands.exists() { - return commands; + let copied = repo_root.join("software/coreutils/wasm"); + if copied.exists() { + return copied; + } + + let fallback = repo_root.join("toolchain/target/wasm32-wasip1/release/commands"); + if fallback.exists() { + return fallback; } panic!( - "registry WASM commands are required for posix path repro tests: run `just registry-native`; expected {}", - commands.display() + "registry WASM commands are required for posix path repro tests: expected {} or {}", + copied.display(), + fallback.display() ); } @@ -343,9 +356,11 @@ console.log(JSON.stringify({ "guest shell should resolve relative paths inside the cwd: {guest}" ); assert_eq!( - guest["stderr"] - .as_str() - .expect("child stderr should be encoded as a string"), + strip_benign_child_pid_warnings( + guest["stderr"] + .as_str() + .expect("child stderr should be encoded as a string") + ), "", "guest shell should not emit unexpected stderr: {guest}" ); @@ -465,9 +480,11 @@ console.log(JSON.stringify({ "guest shell should still succeed with absolute paths: {guest}" ); assert_eq!( - guest["stderr"] - .as_str() - .expect("child stderr should be encoded as a string"), + strip_benign_child_pid_warnings( + guest["stderr"] + .as_str() + .expect("child stderr should be encoded as a string") + ), "", "guest shell should not emit unexpected stderr: {guest}" ); @@ -485,7 +502,6 @@ fn node_path_posix_edge_cases_match_host_node() { import path from "node:path"; console.log(JSON.stringify({ - posixIdentity: path.posix === path, resolve: path.resolve("/workspace/project/", "./src", "../tests", "spec.ts"), join: path.join("/workspace", "project", "..", "project", "note.txt"), normalize: path.normalize("/workspace//project/tests/../nested//file.txt"), @@ -498,113 +514,6 @@ console.log(JSON.stringify({ ); } -fn node_console_formatting_matches_host_node() { - assert_guest_matches_host( - "console-formatting", - r#" -const writes = []; -const originalWrite = process.stdout.write; -process.stdout.write = (chunk) => { - writes.push(String(chunk)); - return true; -}; -console.log("value:%s count:%d object:%o", "ok", 3, { nested: true }); -process.stdout.write = originalWrite; -originalWrite.call(process.stdout, JSON.stringify({ writes })); -"#, - ); -} - -fn javascript_child_process_executes_guest_shebang_scripts() { - assert_node_available(); - - let (cwd, entrypoint) = write_probe( - "child-shebang", - r#" -import childProcess from "node:child_process"; - -const result = childProcess.spawnSync( - "/workspace/hello.sh", - ["native"], - { encoding: "utf8" }, -); -const denied = childProcess.spawnSync( - "/workspace/not-executable.sh", - [], - { encoding: "utf8" }, -); -console.log(JSON.stringify({ - status: result.status, - signal: result.signal, - stdout: result.stdout, - stderr: result.stderr, - denied: { - status: denied.status, - signal: denied.signal, - errorCode: denied.error?.code ?? null, - stderr: denied.stderr, - }, -})); -"#, - ); - let script = cwd.join("hello.sh"); - write_fixture( - &script, - "#!/usr/bin/env -S sh\nprintf 'shebang:%s\\n' \"$1\"\n", - ); - let mut permissions = fs::metadata(&script) - .expect("read shebang fixture metadata") - .permissions(); - use std::os::unix::fs::PermissionsExt; - permissions.set_mode(0o755); - fs::set_permissions(&script, permissions).expect("make shebang fixture executable"); - write_fixture( - &cwd.join("not-executable.sh"), - "#!/bin/sh\nprintf 'must-not-run\\n'\n", - ); - - let guest = run_guest_probe( - "child-shebang", - &cwd, - &entrypoint, - true, - HashMap::new(), - vec![MountDescriptor { - guest_path: String::from("/workspace"), - read_only: false, - plugin: MountPluginDescriptor { - id: String::from("host_dir"), - config: serde_json::to_string(&json!({ - "hostPath": cwd, - "readOnly": false, - })) - .expect("serialize shebang workspace mount config"), - }, - }], - ); - - assert_eq!(guest["status"], json!(0), "shebang child failed: {guest}"); - assert_eq!( - guest["signal"], - Value::Null, - "shebang child signaled: {guest}" - ); - assert_eq!(guest["stdout"], "shebang:native\n"); - assert_eq!(guest["stderr"], ""); - assert_ne!( - guest["denied"]["status"], - json!(0), - "non-executable script unexpectedly ran: {guest}" - ); - assert!( - guest["denied"]["errorCode"] == "EACCES" - || guest["denied"]["stderr"] - .as_str() - .is_some_and(|stderr| stderr.contains("EACCES")), - "non-executable script did not surface EACCES: {guest}" - ); -} - fn filesystem_path_edge_cases_match_host_node() { assert_guest_matches_host( "filesystem-paths", @@ -647,7 +556,5 @@ fn posix_path_repro_suite() { filesystem_path_edge_cases_match_host_node(); guest_shell_absolute_paths_still_work_after_cd(); guest_shell_relative_paths_follow_cwd_after_cd(); - javascript_child_process_executes_guest_shebang_scripts(); - node_console_formatting_matches_host_node(); node_path_posix_edge_cases_match_host_node(); } diff --git a/crates/native-sidecar/tests/projection_bench.rs b/crates/native-sidecar/tests/projection_bench.rs index db6094cda5..034b4830f8 100644 --- a/crates/native-sidecar/tests/projection_bench.rs +++ b/crates/native-sidecar/tests/projection_bench.rs @@ -58,9 +58,9 @@ //! //! Then runs the repo's built registry tars (skipped with a note if a tar is //! absent, e.g. in a clean checkout that has not built `dist/`): -//! - coreutils: `registry/software/coreutils/dist/package.tar` (large, many commands) -//! - tar: `registry/software/tar/dist/package.tar` (single wasm binary) -//! - git: `registry/software/git/dist/package.tar` (the package +//! - coreutils: `software/coreutils/dist/package.tar` (large, many commands) +//! - tar: `software/tar/dist/package.tar` (single wasm binary) +//! - git: `software/git/dist/package.tar` (the package //! the marketing install comparison talks about) //! //! Override the source tars via env: @@ -73,9 +73,9 @@ //! ```text //! cargo test -p agentos-native-sidecar --release --test projection_bench -- --ignored --nocapture //! # or, pointing at built tars elsewhere: -//! PROJ_BENCH_COREUTILS_TAR=/abs/registry/software/coreutils/dist/package.tar \ -//! PROJ_BENCH_TAR_TAR=/abs/registry/software/tar/dist/package.tar \ -//! PROJ_BENCH_GIT_TAR=/abs/registry/software/git/dist/package.tar \ +//! PROJ_BENCH_COREUTILS_TAR=/abs/software/coreutils/dist/package.tar \ +//! PROJ_BENCH_TAR_TAR=/abs/software/tar/dist/package.tar \ +//! PROJ_BENCH_GIT_TAR=/abs/software/git/dist/package.tar \ //! cargo test -p agentos-native-sidecar --release --test projection_bench -- --ignored --nocapture //! ``` @@ -612,16 +612,10 @@ fn projection_bench() { let coreutils_tar = source_tar_path( "PROJ_BENCH_COREUTILS_TAR", - "registry/software/coreutils/dist/package.tar", - ); - let tar_tar = source_tar_path( - "PROJ_BENCH_TAR_TAR", - "registry/software/tar/dist/package.tar", - ); - let git_tar = source_tar_path( - "PROJ_BENCH_GIT_TAR", - "registry/software/git/dist/package.tar", + "software/coreutils/dist/package.tar", ); + let tar_tar = source_tar_path("PROJ_BENCH_TAR_TAR", "software/tar/dist/package.tar"); + let git_tar = source_tar_path("PROJ_BENCH_GIT_TAR", "software/git/dist/package.tar"); println!("\n# agentOS package load benchmark (.aospkg)"); println!( @@ -688,7 +682,7 @@ fn projection_bench() { fn coreutils_load_budget() { let coreutils_tar = source_tar_path( "PROJ_BENCH_COREUTILS_TAR", - "registry/software/coreutils/dist/package.tar", + "software/coreutils/dist/package.tar", ); if !coreutils_tar.is_file() { eprintln!( diff --git a/crates/native-sidecar/tests/service.rs b/crates/native-sidecar/tests/service.rs index 3387887e69..a51a16561d 100644 --- a/crates/native-sidecar/tests/service.rs +++ b/crates/native-sidecar/tests/service.rs @@ -527,6 +527,29 @@ ykAheWCsAteSEWVc0w==\n\ })); } + fn wasm_signal_queue_is_bounded() { + let kernel_handle = create_kernel_process_handle_for_tests(); + let mut process = ActiveProcess::new( + kernel_handle.pid(), + kernel_handle, + GuestRuntimeKind::WebAssembly, + ActiveExecution::Tool(ToolExecution::default()), + ); + for _ in 0..MAX_PROCESS_EVENT_QUEUE { + process + .queue_pending_wasm_signal(nix::libc::SIGUSR1) + .expect("signal queue should accept entries up to its bound"); + } + let error = process + .queue_pending_wasm_signal(nix::libc::SIGUSR1) + .expect_err("signal queue should reject overflow"); + assert!( + error.to_string().contains("process event queue exceeded"), + "unexpected overflow error: {error}" + ); + assert_eq!(process.pending_wasm_signals.len(), MAX_PROCESS_EVENT_QUEUE); + } + fn descendant_transfer_overflow_preserves_global_queue() { let mut sidecar = create_test_sidecar(); let (connection_id, session_id) = @@ -1339,17 +1362,17 @@ ykAheWCsAteSEWVc0w==\n\ .join("../..") .canonicalize() .expect("canonicalize repo root"); - let copied = repo_root.join("registry/software/coreutils/wasm"); + let copied = repo_root.join("software/coreutils/wasm"); if copied.exists() { return copied; } - let fallback = repo_root.join("registry/native/target/wasm32-wasip1/release/commands"); + let fallback = repo_root.join("toolchain/target/wasm32-wasip1/release/commands"); if fallback.exists() { return fallback; } - let vendored = repo_root.join("packages/core/commands"); + let vendored = repo_root.join("packages/runtime-core/commands"); if vendored.exists() { let staged = temp_dir("agentos-native-sidecar-vendored-commands"); for command in ["bash", "cat", "mkdir", "printf", "sh"] { @@ -1372,6 +1395,29 @@ ykAheWCsAteSEWVc0w==\n\ return staged; } + let legacy_vendored = repo_root.join("packages/core/commands"); + if legacy_vendored.exists() { + let staged = temp_dir("agentos-native-sidecar-vendored-commands"); + for command in ["bash", "cat", "mkdir", "printf", "sh"] { + let source = legacy_vendored.join(command); + let target = staged.join(command); + fs::copy(&source, &target).unwrap_or_else(|error| { + panic!( + "copy vendored command {} -> {}: {error}", + source.display(), + target.display() + ) + }); + let mut permissions = fs::metadata(&target) + .expect("stat staged vendored command") + .permissions(); + permissions.set_mode(0o755); + fs::set_permissions(&target, permissions) + .expect("chmod staged vendored command"); + } + return staged; + } + panic!( "registry WASM commands are required for service fs regression tests: expected {}, {}, or {}", copied.display(), @@ -21217,6 +21263,7 @@ console.log(JSON.stringify({ pending_process_events_are_bounded(); process_event_receiver_overflow_preserves_queued_event(); tool_execution_event_overflow_is_reported(); + wasm_signal_queue_is_bounded(); descendant_transfer_overflow_preserves_global_queue(); exit_trailing_requeue_preserves_exit_when_queue_is_full(); javascript_child_process_poll_reports_echild_when_child_disappears_after_drain(); @@ -21255,6 +21302,7 @@ console.log(JSON.stringify({ pending_process_events_are_bounded(); process_event_receiver_overflow_preserves_queued_event(); tool_execution_event_overflow_is_reported(); + wasm_signal_queue_is_bounded(); descendant_transfer_overflow_preserves_global_queue(); exit_trailing_requeue_preserves_exit_when_queue_is_full(); } diff --git a/crates/sidecar-protocol/src/protocol.rs b/crates/sidecar-protocol/src/protocol.rs index a82bab77da..6c5556171e 100644 --- a/crates/sidecar-protocol/src/protocol.rs +++ b/crates/sidecar-protocol/src/protocol.rs @@ -2671,6 +2671,8 @@ fn validate_requirement( #[derive(Debug, Deserialize, Default)] pub struct JavascriptChildProcessSpawnOptions { + #[serde(default)] + pub argv0: Option, #[serde(default)] pub cwd: Option, #[serde(default)] diff --git a/crates/vfs/src/posix/overlay_fs.rs b/crates/vfs/src/posix/overlay_fs.rs index 25aa86b9e3..9fd0600587 100644 --- a/crates/vfs/src/posix/overlay_fs.rs +++ b/crates/vfs/src/posix/overlay_fs.rs @@ -1661,6 +1661,8 @@ impl VirtualFileSystem for OverlayFileSystem { let mut snapshot_entries = Vec::new(); self.collect_snapshot_entries(&old_normalized, &mut snapshot_entries)?; + self.clear_path_metadata(&resolved_new_normalized)?; + self.clear_subtree_metadata(&resolved_new_normalized)?; if let Ok(destination_stat) = self.merged_lstat(&resolved_new_normalized) { if destination_stat.is_directory && !destination_stat.is_symbolic_link @@ -1678,7 +1680,6 @@ impl VirtualFileSystem for OverlayFileSystem { .remove_file(&resolved_new_normalized)?; } } - self.clear_subtree_metadata(&resolved_new_normalized)?; } self.stage_snapshot_entries_in_upper(&snapshot_entries)?; @@ -1985,6 +1986,33 @@ mod tests { assert_error_code(overlay.read_dir("/a"), "ENOENT"); } + #[test] + fn rename_clears_destination_whiteout() { + let lower = MemoryFileSystem::new(); + let mut overlay = OverlayFileSystem::new(vec![lower], OverlayMode::Ephemeral); + overlay + .write_file("/archive.zip", b"old".to_vec()) + .expect("create old archive"); + overlay + .remove_file("/archive.zip") + .expect("whiteout old archive"); + overlay + .write_file("/workspace-temp", b"new".to_vec()) + .expect("create replacement"); + + overlay + .rename("/workspace-temp", "/archive.zip") + .expect("rename replacement over whiteout"); + + assert_eq!( + overlay + .read_file("/archive.zip") + .expect("read renamed file"), + b"new" + ); + assert!(!overlay.exists("/workspace-temp")); + } + #[test] fn remove_dir_still_rejects_visible_children() { let mut lower = MemoryFileSystem::new(); diff --git a/docs-internal/networking-parity-spec.md b/docs-internal/networking-parity-spec.md new file mode 100644 index 0000000000..de75e5ab99 --- /dev/null +++ b/docs-internal/networking-parity-spec.md @@ -0,0 +1,227 @@ +# Networking Parity Spec — curl / wget / git full TLS + network support + +Status: implemented · Owner: registry/runtime · Last updated: 2026-07-14 + +This document preserves the design baseline and decisions that drove the +networking stack. Sections describing missing TLS or future work refer to the +pre-implementation state. For the shipped architecture, see +[`TLS & SSL`](../website/src/content/docs/docs/architecture/tls-ssl.mdx). + +## Goal + +`curl`, `wget`, and `git` must have **full networking with real TLS/SSL, using +native Linux semantics** — not host-brokered shims. Concretely: + +- `curl https://…` and `wget https://…` verify certificates against a real CA + bundle, honor `--cacert`/`--ca-certificate`, fail with the **real exit codes** + (curl exit 60 = cert verify failure), and support `--compressed` (gzip, brotli, + zstd). +- `git clone/fetch/push` over **HTTPS smart-HTTP** works against GitHub/GitLab. +- Certificate trust comes from a **CA bundle inside the VM** (`/etc/ssl/certs/…`), + the way Debian's `ca-certificates` package + apt/curl/openssl/python all resolve + it — not from the host machine's trust store. + +## Baseline before implementation (verified) + +Sockets are **already real** and are not the problem. The patched wasi-libc +sysroot implements `socket()/connect()/getaddrinfo()/send()/recv()` over +`host_net` WASM imports (`toolchain/std-patches/wasi-libc/0008-sockets.patch`, +`0023-host-net-read-write-sockets.patch`; Rust mirror `toolchain/crates/wasi-ext`). +The runner forwards them to the sidecar socket table (`crates/execution/assets/ +runners/wasm-runner.mjs`). So curl/wget/git already do their own DNS, TCP and HTTP +byte-for-byte. **Only TLS and decompression are shimmed or missing.** + +| Tool | Networking today | TLS today | Gap | +|---|---|---|---| +| **curl** | real (8.11.1) | host-brokered `wasi_tls.c` → `net_tls_connect` → sidecar rustls + **host** cert store | no real verify, `--cacert`/exit-60/`-v` chain all wrong; no gzip/brotli/zstd | +| **wget** | real (1.24.5) | **none** (`--without-ssl`) — HTTP only | no HTTPS at all; no gzip | +| **git** | real, local-only (2.55.0, `NO_CURL`) | **none** | `git-remote-https` is a dead symlink→git binary; no HTTPS clone/fetch/push | + +The current TLS shim (`software/curl/native/c/overlay/lib/vtls/wasi_tls.c`) does +**no handshake and no verification in-guest**: it hands the TCP fd to the host +(`net_tls_connect(fd, host, flags)`), which terminates TLS with rustls + +`rustls_native_certs::load_native_certs()` — **the host machine's** trust store. +`--cacert`/`--capath`/`CURL_CA_BUNDLE`/`SSL_CERT_FILE` are silently ignored; any +TLS error returns curl exit 35 instead of 60; `curl -v` cannot print the chain. + +## The decision: in-guest TLS (mbedTLS) + a VM CA bundle + +Two directions were researched. **We choose in-guest TLS** because it is the only +one that delivers the acceptance criterion (native Linux semantics + a CA bundle +that mimics apt). + +### ✅ Chosen — in-guest mbedTLS 3.6 LTS + `/etc/ssl/certs/ca-certificates.crt` + +- **mbedTLS** is pure portable C99, zero platform deps, TLS 1.3, and curl has a + first-class maintained `USE_MBEDTLS` backend in 8.11.1. Entropy = `getentropy()` + (already proven in `wasi_tls.c`); clocks/time work; build single-threaded. +- Verification happens **in curl's own code path**: `--cacert` → + `mbedtls_x509_crt_parse_file` via the VFS, `-k` → generic `verifypeer`, verify + failure → `CURLE_PEER_FAILED_VERIFICATION` = **exit 60** with the real message, + `curl -v` prints version/cipher/cert. Exactly Linux curl-with-mbedTLS. +- The sidecar becomes a **dumb ciphertext pipe** — strictly better for the trust + model (the untrusted executor no longer asks the trusted host to authenticate + servers on its behalf), and hermetic (no dependence on the host's cert store). + +### ❌ Rejected for parity — host-brokered TLS (the current `net_tls_connect` path) + +Reusing the existing bridge is minimal work (wget = ~100-line shim, git = build +with the overlaid libcurl), but it **cannot reach native semantics**: it uses the +*host's* cert store (non-hermetic), ignores `--cacert`/`--capath`, returns the +wrong exit-code taxonomy, can't print the cert chain, and every future TLS flag +(client certs, `--pinnedpubkey`, TLS-version pinning) needs a new bridge hop — a +permanent shim treadmill. It also contradicts the "real tool, patch the sysroot" +philosophy the rest of the toolchain follows. Keep the host `net.socket_upgrade_tls` +path only for the **Node/JS runtime**, which is a separate surface. + +> Note: the current sidecar TLS path uses `rustls_native_certs` = the **host +> machine's** trust store (`crates/native-sidecar/src/execution.rs`). That is a +> latent hermeticity bug even for the JS runtime — it should read the VM's +> `/etc/ssl` bundle instead. Tracked here; fix alongside. + +## CA bundle — ship Debian-shaped trust inside the VM + +- Ship the **Mozilla CA bundle** (`curl.se/ca/cacert.pem`, i.e. the set Debian's + `ca-certificates` produces) at **`/etc/ssl/certs/ca-certificates.crt`**, with the + conventional `/etc/ssl/cert.pem` symlink. A `ca-certificates` registry package + owns the payload; VM bootstrap links it into the standard tree (the bootstrap + already seeds `/etc` in the shadow root — `crates/native-sidecar/src/vm.rs`). +- This one file at that one path is what makes the **whole class** of TLS tools + "just work": curl's compile-time `CURL_CA_BUNDLE` default, OpenSSL's `OPENSSLDIR` + (`/usr/lib/ssl` → `/etc/ssl/certs`), apt, python, wget — all resolve there on + Debian. `SSL_CERT_FILE`/`SSL_CERT_DIR`/`--cacert` env overrides then work for + free once the backend does real verification. + +## Per-tool implementation plan (completed) + +### curl +1. Vendor + build **mbedTLS 3.6** for `wasm32-wasip1` in `toolchain/c/Makefile` + (same pattern as the existing zlib target). +2. In `toolchain/c/scripts/build-curl-upstream.sh`: drop `--without-ssl` and the + `USE_WASI_TLS` injection; add `--with-mbedtls`, `--with-zlib`, `--with-brotli`, + `--with-zstd`, `--with-ca-bundle=/etc/ssl/certs/ca-certificates.crt`; retire the + `wasi_tls.c/h` + `vtls.c` overlay. +3. Tests: assert **exit 60** + real verify message on self-signed without `-k`; + `--cacert` acceptance of a test CA; `--compressed` gzip/br/zstd round-trips. + +### wget +GNU Wget has no mbedTLS backend, so give it a real TLS backend against the same +mbedTLS + CA bundle (its SSL abstraction is just 4 functions in `src/ssl.h`: +`ssl_init`, `ssl_cleanup`, `ssl_connect_wget`, `ssl_check_certificate`). +1. Add `wasi_ssl.c` implementing those 4 functions over **mbedTLS** (handshake + + `mbedtls_x509_crt_parse_file` on `/etc/ssl/certs/ca-certificates.crt`; + `--no-check-certificate`/`--ca-certificate`/`--ca-directory` map to real + mbedTLS verify config, matching Linux wget). Overlay dir mirrors curl's + (`software/wget/native/c/overlay/…`). +2. In `build-wget-upstream.sh`: keep `./configure --without-ssl` (so it doesn't + probe GnuTLS/OpenSSL), then post-configure patch `src/wget.h`'s `HAVE_SSL` + condition to add `|| defined HAVE_WASI_TLS`, pass `-DHAVE_WASI_TLS`, copy the + overlay, append a `wasi_ssl.o` compile/link rule to the generated `src/Makefile` + (the exact playbook the curl script uses). This lights up the `https` scheme, + all `--secure-protocol/--ca-certificate/--no-check-certificate` options, and + HSTS. +3. Enable **zlib** (`--with-zlib` via `ZLIB_CFLAGS/ZLIB_LIBS` at the existing + `build/zlib/libz.a`). Leave `--without-libpsl` (cosmetic — cookie public-suffix + only; wget's built-in heuristic covers single-URL HTTPS). +4. Tests: mirror curl's HTTPS cases (verify-fail on self-signed, + `--no-check-certificate` success, keep-alive), + gzip via `--compression`. + +### git — HTTPS smart-HTTP (clone/fetch/**push**) +Git's HTTP transport lives entirely in the **`git-remote-https` remote helper**, +which links **libcurl in-process** (git never shells out to a `curl` binary). The +current symlink `git-remote-https → git` is structurally broken (`git`'s +`cmd_main` dies "cannot handle remote-https as a builtin"). +1. **Produce a reusable libcurl** artifact (`make -C lib install` the overlaid, + mbedTLS-linked libcurl into a prefix under `toolchain/c/build/curl-upstream/ + install`). +2. In `toolchain/c/scripts/build-git-upstream.sh`: drop `NO_CURL=1`; add + `CURL_CFLAGS/CURL_LDFLAGS` pointing at that libcurl (defining them skips + `curl-config`); keep `NO_EXPAT=1`; build targets **`git git-remote-http`**; + wasm-opt both. (Expect a handful of small WASI compile fixes in + `http.c`/`remote-curl.c`, same class the curl script already makes; the + existing `git_compat.c` shims cover most.) +3. **Packaging:** install `git-remote-http` as a **real second command** (replace + the wrong symlink loop in `toolchain/c/Makefile`); in + `software/git/agentos-package.json` move `git-remote-http` into `commands` and + re-point `git-remote-https` → `git-remote-http` (keep `git-upload-pack`/ + `git-receive-pack`/`git-upload-archive` → `git`; those *are* builtins). +4. **Push** is the same libcurl POST plumbing as fetch (`remote-curl.c` spawns + `git send-pack` and streams the pack via chunked `Transfer-Encoding` + + `Expect: 100-continue` — all inside libcurl, no new host surface). HTTP Basic + auth (token/`user:pass` URLs, `GIT_ASKPASS`) works. (`credential-cache` + unavailable — `NO_UNIX_SOCKETS`; NTLM/Negotiate unsupported — irrelevant for + GitHub/GitLab.) +5. **Nothing new on the host side** — TLS/DNS/TCP/spawn/permission-tiers are all + in place. Spawn already resolves `git-remote-https` by name on the guest PATH + via the `proc_spawn` broker (proven: `git clone` already spawns + `git upload-pack` locally). Permission tiers already grant `git-remote-http(s)` + tier `full`. +6. **ssh transport (secondary):** `git@host:` spawns the separately delivered + in-guest OpenSSH client. SSH uses its own protocol crypto rather than mbedTLS. +7. Tests: flip `hasGitHttpHelper` in `software/git/test/git.test.ts` (the + smart-HTTP clone/fetch suite is already written but skipped); add a push case + (small + >1 MiB chunked POST). + +## Compression (curl) + +- **zlib 1.3.1 is already vendored** and built against this sysroot (git links it): + `--with-zlib` via `CPPFLAGS/LDFLAGS` at `build/zlib`. +- **brotli** — decoder (`common`+`dec`) is dependency-free portable C; add a + Makefile fetch/build target mirroring zlib; `--with-brotli` (decode only). +- **zstd** — already compiles under this sysroot inside the duckdb build + (`toolchain/c/build/duckdb-cmake/third_party/zstd/`); build libzstd + single-threaded; `--with-zstd`. +- Out of scope here: nghttp2 (HTTP/2), libpsl. + +## Refuted / dead-end theses (documented so we don't re-try them) + +- **Host-brokered TLS for parity** — works but never reaches native semantics + (host cert store, ignored `--cacert`, wrong exit codes, no `-v` chain, shim + treadmill). Rejected as the parity path; kept only for the JS runtime. +- **Full OpenSSL as the shared HTTPS backend** — unnecessary for curl, wget, and + git. A checksum-pinned OpenSSL 3.5.7 **libcrypto-only** build is proven and is + now used by OpenSSH to restore RSA/ECDSA/DH/AES algorithm coverage. It builds + without source patches by disabling threads, assembly, dynamic modules, + engines, applications, and libssl, and seeds through the owned `getrandom` + path. That scoped success does not justify replacing the smaller mbedTLS HTTPS + integrations with the full OpenSSL TLS/provider/config surface. +- **GnuTLS in-guest** (wget's default) — drags nettle → GMP (asm) → libtasn1 → + p11-kit (dlopen). Non-starter on WASI. +- **rustls + aws-lc-rs in guest** — aws-lc-rs is C+asm, does **not** build for + wasm32-wasip1. `ring` support is unofficial/spotty. Mixing a Rust staticlib into + the clang-LTO curl link is fiddly. No fidelity gain over mbedTLS. +- **rustls-native-certs (as the trust story)** — needs an OS cert store; the guest + has none, and host-side it depends on the host machine's store (non-hermetic). +- **BearSSL** — curl **removed** BearSSL support in 8.12.0 and it lacks TLS 1.3. + Dead end for any curl upgrade. +- **Reusing curl's `wasi_tls.c` for wget** — impossible; it's written against + libcurl-internal `Curl_cfilter` APIs. The shareable unit is the **host import + contract / the mbedTLS backend**, not the C file. +- **git: symlink `git-remote-https` → git binary** (current state) — git's + `cmd_main` hard-dies for a non-builtin `git-*` argv[0]. Structurally impossible. +- **git: spawn the `curl` CLI from a shell helper** — the remote helper is a + long-lived, stateful, bidirectional protocol-v2 program (auth retry, gzip, + full-duplex chunked POST). The CLI can't provide it. +- **git: gitoxide / libgit2 / reimplement smart protocol in-process** — not real + upstream git; diverges on CLI/config/hooks; would need its own WASI net port + anyway. Rejected. +- **git: dumb-HTTP only** — GitHub/GitLab/Gitea serve smart-HTTP only for fetch + and never support dumb push. Non-starter (dumb fetch rides along free). +- **Anything threaded** — the runner is single-threaded wasip1; all TLS/compression + libs build single-threaded (mbedTLS/zstd support it). + +## Work breakdown & sequencing + +1. **mbedTLS 3.6** target in `toolchain/c/Makefile` (shared by curl, wget, git's + libcurl). ← foundational, do first. +2. **CA bundle**: `ca-certificates` payload at `/etc/ssl/certs/ca-certificates.crt` + + VM bootstrap seeding + sidecar reads the VM bundle (not the host store). ← + foundational. +3. **curl**: mbedTLS + zlib/brotli/zstd + CA bundle; retire `wasi_tls.c`; tests. +4. **libcurl install artifact** (from step 3's build) for git to link. +5. **git**: build-with-curl, real `git-remote-http` helper, packaging, push; tests. +6. **wget**: `wasi_ssl.c` mbedTLS backend + zlib + CA bundle; tests. + +Steps 3/5/6 all sit on 1+2. One jj rev per tool. Real e2e tests against a TLS +server (verify-fail on self-signed, `--cacert` success, real clone/fetch/push) are +the deliverable — no host-store shortcuts. diff --git a/docs-internal/registry-flatten-colocation-spec.md b/docs-internal/registry-flatten-colocation-spec.md new file mode 100644 index 0000000000..f4298f4cc9 --- /dev/null +++ b/docs-internal/registry-flatten-colocation-spec.md @@ -0,0 +1,392 @@ +# Spec: Flatten `registry/` → `software/` + colocated `native/` + +Status: proposal · Owner: registry · Last updated: 2026-07-07 + +## Summary + +Eliminate the `registry/` wrapper. The package catalog becomes a top-level +`software/` directory where **every `@agentos-software/*` package — command +packages, agent adapters, and bundles — is one flat folder** that owns its +manifest, TS descriptor, e2e test, **and its own command source** under +`native/`. Shared WASM build infrastructure that no single package can own +(patched sysroot, shared runtime-shim crates, build orchestration, libc +conformance) moves to a top-level **`toolchain/`**. + +In the same effort, push the generic package-authoring capabilities that live +only in this repo's `Makefile`/`justfile` today — **compiling** a command to WASM +and **testing** it in a VM — into the published `agentos-toolchain` CLI, so the +`justfile` is left with only repo-specific orchestration. + +Result: `registry/` disappears; `software//` is self-describing; the +misleading "libc tests under registry" problem is gone; and authoring an agentOS +package is the same flow for us and for external users. + +## Motivation + +Today `registry/` mixes three unlike things: package manifests (`software/`, +`agent/`), the command **source + WASM toolchain** (`native/`), and libc/sysroot +**conformance tests** (`native/tests/`). "Registry" reads as "the catalog of +packages," so a C/Rust cross-compilation toolchain and libc conformance tests +living under it is misleading. Package source also lives far from the package +(`software/curl/` has the manifest; `curl.c` is in `registry/native/c/programs/`). + +## Target file structure + +``` +repo-root/ +├── software/ ← the entire @agentos-software/* catalog (flat, no registry/) +│ │ +│ ├── curl/ # C-based command package +│ │ ├── package.json # @agentos-software/curl +│ │ ├── agentos-package.json # manifest + registry block (title, description, category, kind) +│ │ ├── src/index.ts # TS descriptor +│ │ ├── test/curl.test.ts # e2e integration test +│ │ ├── native/ +│ │ │ └── c/ +│ │ │ ├── curl.c # command implementation +│ │ │ └── overlay/ # pkg-specific upstream overlay (was c/curl-upstream-overlay/) +│ │ └── bin/curl # staged build output (gitignored) +│ │ +│ ├── git/ # Rust-based command package +│ │ ├── package.json agentos-package.json src/ test/ bin/ +│ │ └── native/crates/ +│ │ ├── cmd-git/ # command crate (was native/crates/commands/git) +│ │ └── git/ # 1:1 lib crate (was native/crates/libs/git) +│ │ +│ ├── coreutils/ # multi-command package (sh + ~80 utils) +│ │ ├── package.json agentos-package.json src/ test/ bin/ +│ │ └── native/crates/ +│ │ ├── sh/ cat/ ls/ cp/ mv/ sort/ … # ~80 command crates +│ │ └── (du/ expr/ column/ rev/ strings/ — 1:1 libs that belong to coreutils) +│ │ +│ ├── sqlite3/ duckdb/ wget/ zip/ unzip/ # C-based, native/c/.c +│ ├── grep/ sed/ gawk/ jq/ yq/ fd/ ripgrep/ tree/ file/ tar/ gzip/ diffutils/ findutils/ # Rust, native/crates/ +│ │ +│ ├── claude/ codex/ opencode/ pi/ pi-cli/ # agent adapters — @agentos-software/*, JS only, no native/ +│ │ ├── package.json agentos-package.json (kind: "agent") src/adapter.ts test/ +│ │ +│ ├── vim/ # editor (C in native/c/) +│ └── browserbase/ build-essential/ common/ everything/ # meta/external — manifest only, no native/ +│ +├── toolchain/ ← shared WASM build infra (NOT a package; nothing here is @agentos-software/*) +│ ├── Cargo.toml # WASM cargo workspace root — members glob into software/*/native/crates/* +│ ├── Cargo.lock +│ ├── rust-toolchain.toml # nightly + wasm32-wasip1 + build-std +│ ├── Makefile # builds every software/*/native/ against the shared sysroot +│ ├── crates/ # SHARED rust crates used by ≥2 packages / infra +│ │ ├── shims/ builtins/ stubs/ wasi-ext/ +│ │ ├── wasi-http/ wasi-pty/ wasi-spawn/ +│ │ └── reqwest-shim/ portable-pty-wasi/ codex-network-proxy{,-wasi}/ codex-otel/ uucore/ ctrlc/ hostname/ +│ ├── sysroot/ # patched wasi-sdk build (was native/c/{include,patches,cmake,scripts} + wasi-sdk) +│ │ ├── wasi-sdk/ # fetched +│ │ ├── include/ patches/ cmake/ scripts/ +│ │ └── libs-cache/ # fetched upstream C sources (sqlite, zlib, curl, duckdb) +│ ├── std-patches/ # rust std patches 0001–0009 (was native/patches/) +│ ├── scripts/ # patch-std.sh patch-vendor.sh patch-wasi-libc.sh +│ ├── test-programs/ # C test-program fixtures (tcp_server, udp_echo, signal_handler, http_server, …) +│ │ # built here; runtime-core integration tests consume the built binaries +│ ├── conformance/ # libc/os-test/c-parity — tests the sysroot, not a package (was native/tests/) +│ │ ├── c-parity.test.ts libc-test-conformance.test.ts os-test-conformance.test.ts *-exclusions.json +│ └── target/ # shared cargo build output (gitignored) +│ +├── test-harness/ ← private workspace pkg; shared vitest helpers + WASM VM runtime + binary resolution +│ ├── package.json # @agentos/test-harness (private, not published) +│ └── src/{helpers,terminal-harness}.ts +│ +└── packages/ + └── runtime-core/ + └── tests/integration/ # VM integration tests (net, npm-e2e, wasi, signal, cross-runtime) +``` + +All native **compilation stays in `toolchain/`** — command source lives with its +package for editing, but nothing compiles WASM outside the toolchain build. C +test-program fixtures stay in `toolchain/test-programs/` (not scattered into +`packages/`) so there is exactly one place that invokes the C compiler + sysroot. + +## Boundary rules + +1. **`software//` owns**: `package.json`, `agentos-package.json`, `src/` + (TS descriptor), `test/` (e2e), `native/` (this package's command source), + `bin/` (staged output). +2. **`toolchain/` owns**: the patched sysroot, shared runtime-shim crates, std + patches, build orchestration, and libc/sysroot conformance tests. Nothing in + `toolchain/` is an `@agentos-software/*` package. +3. **Command source colocates; shared infra does not.** A crate used by exactly + one package moves into that package; a crate used by ≥2 packages (or that is + pure infra) stays in `toolchain/crates/`. This is the honest limit of + colocation — see [Shared vs per-package](#shared-vs-per-package-split). +4. **Agents are packages too.** `claude`/`codex`/`opencode`/`pi`/`pi-cli` live + under `software/` because they publish as `@agentos-software/*`. They carry + `kind: "agent"` in the manifest and have no `native/`. Command packages carry + `kind: "software"` (or omit; default). This `kind` field is what distinguishes + them — not folder location. +5. **`native/` holds the REAL upstream tool, patched — never a reimplementation.** + A command package's `native/` must build the genuine upstream program (GNU + coreutils, real `curl`/`git`/`jq`, GNU grep/sed/gawk/tar/gzip/diffutils, …) + fetched + pinned and patched for WASI — not a from-scratch Rust/C rewrite, a + stub, or a hand-rolled CLI over a library. The only exception is a tool whose + canonical upstream *is* the Rust project (`ripgrep`, `fd`). Several current + commands violate this (coreutils=uutils, grep, curl driver, and the + `secureexec-*` rewrites) — tracked in `registry-parity-worklist.md` + Cross-cutting #0; new packages must not add more. + +## What moves where + +| Current | Destination | +|---|---| +| `software//` | `software//` | +| `software//` | `software//` (+ `kind: "agent"`) | +| `registry/native/crates/commands/` | `software//native/crates/cmd-/` | +| `registry/native/crates/libs/` (1:1) | `software//native/crates//` | +| `registry/native/crates/libs/` (shared ≥2) | `toolchain/crates//` | +| `registry/native/crates/{wasi-ext,libs/{shims,builtins,stubs,wasi-http,wasi-pty,wasi-spawn}}` | `toolchain/crates/` | +| `registry/native/stubs/*` | `toolchain/crates/` | +| `registry/native/c/programs/.c` (a package command) | `software//native/c/.c` | +| `registry/native/c/programs/.c` (tcp_server, udp_echo, signal_handler, …) | `toolchain/test-programs/` (built by toolchain; consumed by runtime-core integration tests via the binary path) | +| `registry/native/c/{include,patches,cmake,scripts,vim overlay}` + wasi-sdk | `toolchain/sysroot/` | +| `registry/native/patches/` (std) | `toolchain/std-patches/` | +| `registry/native/scripts/` | `toolchain/scripts/` | +| `registry/native/tests/` (conformance) | `toolchain/conformance/` | +| `registry/native/{Cargo.toml,Cargo.lock,rust-toolchain.toml,Makefile}` | `toolchain/` | +| `registry/tests/` (empty leftover) | delete | + +The command→package owner is derived mechanically from each +`agentos-package.json` `commands` array — the migration script computes it, no +hand-mapping. + +## Shared vs per-package split + +Verified against the current tree: + +- **Shared → `toolchain/crates/`** (used by ≥2 commands or pure infra): + `shims` (7 users), `builtins` (3), `wasi-http` (4), `wasi-spawn` (2), + `stubs`, `wasi-ext`, `wasi-pty`, and all `stubs/*` shim crates + (`reqwest-shim`, `portable-pty-wasi`, `codex-network-proxy{,-wasi}`, + `codex-otel`, `uucore`, `ctrlc`, `hostname`). +- **Per-package → `software//native/crates/`** (1:1): `grep`→grep, + `awk`→gawk, `jq`→jq, `yq`→yq, `git`→git, `fd`→fd, `tar`→tar, `tree`→tree, + `gzip`→gzip, `diff`→diffutils, `find`→findutils, `file-cmd`→file, and + `du`/`expr`/`column`/`rev`/`strings-cmd`→coreutils. + +## Test execution & harness + +The old `agentos-registry` package and its `run-vitest.mjs` shim (which resolved +vitest out of the outer store) **dissolve**. Replacement: + +- A private workspace package **`test-harness/`** (`@agentos/test-harness`, not + published) owns the shared helpers, terminal harness, and `createWasmVmRuntime` + — the surface every test imports today from `registry/tests/helpers.ts`. +- Each `software/` gets a `test` script that runs **`agentos-toolchain test`** + (see [Toolchain CLI boundary](#toolchain-cli-vs-justfile-boundary)), which drives + the `test/` suite through the harness in a VM — the same runner external authors + use. `turbo test` discovers and runs them — this is what re-attaches the suites + to CI (fixing the current orphan + `@xterm/headless`-resolution bugs, since every + runner is now a workspace member). The `@agentos/test-harness` package supplies + `createWasmVmRuntime` + helpers that both the CLI runner and the raw `*.test.ts` + files import. +- Command binaries are resolved from `toolchain/target` (Rust) and the toolchain + C build dir via `AGENTOS_WASM_COMMANDS_DIR` / `AGENTOS_C_WASM_COMMANDS_DIR`, + set once in the harness. No relative-path coupling to the build tree. +- `runtime-core/tests/integration/` and `toolchain/conformance/` import the same + `@agentos/test-harness`. + +## Leftover `registry/` files + +When `registry/` is deleted, its non-package contents relocate: + +| Current | Destination | +|---|---| +| `software/package.json` (`agentos-registry`) + `registry/scripts/run-vitest.mjs` | removed — superseded by per-package `test` scripts + `test-harness/` | +| `registry/tsconfig.base.json` | `software/tsconfig.base.json` (shared by package `tsconfig`s) | +| `registry/CONTRIBUTING.md`, `registry/README.md` | `software/` (or `docs-internal/`), updated for the new layout | + +## Orphan commands (no owning package) + +`envsubst` ships a command + a passing test but **no `software/*` package +declares it**. Before executing, it must get a home — one of: (a) fold into an +existing package's `commands` (e.g. `coreutils`), (b) create `software/envsubst/`, +or (c) reclassify its `.c` + test as a `toolchain/test-programs` fixture. Same +check applies to any other command with a binary/test but no manifest owner — +the migration script lists them; each needs an explicit decision, none are +silently dropped. + +## Cargo workspace strategy (the critical detail) + +There are two Rust workspaces today: the **main** repo workspace (repo-root +`Cargo.toml`, explicit members, stable toolchain) and the **wasm** workspace +(`registry/native/Cargo.toml`, nightly + build-std). They must stay separate — +different toolchains. + +After the move, colocated crates live at `software/*/native/crates/*`, which is +inside the **main** workspace's directory tree. Cargo resolves a crate's +workspace by walking up to the nearest `[workspace]`, so those crates would be +captured by the repo-root workspace — wrong toolchain. Fix with two edits: + +1. **Repo-root `Cargo.toml`**: add `exclude = ["software"]` so the main + workspace never claims package `native/` crates. +2. **`toolchain/Cargo.toml`**: `[workspace]` with + `members = ["crates/*", "../software/*/native/crates/*"]`. Cargo permits + members outside the workspace root via relative path; this private, + never-published workspace is a valid use of it. + +All wasm builds run against the toolchain workspace explicitly +(`cargo build --manifest-path toolchain/Cargo.toml …`, driven by +`toolchain/Makefile`). Cargo's shared `target/` lives at `toolchain/target/`. + +**Fallback if cross-dir members prove fragile in practice:** colocate only the +**C sources + tests + manifest**, and keep the **Rust command crates** together +in `toolchain/crates/commands/`. Less pure, but sidesteps the workspace-spanning +entirely (C has no cargo-workspace concern). This is the one decision to lock +before executing — see [Risks](#risks--decision). + +## Reference updates (~40 sites, mechanical) + +- `software/*/package.json` build scripts: + `../../native/target/...` → `../../../toolchain/target/...` + (and the C build-output dir similarly). +- `justfile`: repoint **and rename** every `registry-*` recipe (the `registry` + prefix is dead once the folder is gone). Rename by the dir the recipe acts on — + `software-*` for package builds, `toolchain-*` for the WASM build/commands: + + | Current recipe | New recipe | Body change | + |---|---|---| + | `registry-build` | `software-build` | `pnpm --filter '@agentos-software/*' build` (unchanged filter) | + | `registry-native` | `toolchain-build` | `make -C registry/native commands` → `make -C toolchain commands` | + | `registry-native-cmd ` | `toolchain-cmd ` | `make -C registry/native cmd/` → `make -C toolchain cmd/` | + | `registry-native-preflight` | `toolchain-preflight` | `cd registry/native/c` → `cd toolchain/sysroot` | + | `registry-copy-commands` | `toolchain-copy-commands` | repoint copy-wasm-commands SRC to `toolchain/target/...` | + + Update any callers of the old recipe names (CI `just` invocations, other + recipes, docs) in the same pass. If you prefer a single flat prefix over the + dir-aligned split, use `software-*` for all — but `toolchain-*` reads truer for + the recipes that build the sysroot/commands rather than the packages. +- `packages/runtime-core/scripts/copy-wasm-commands.mjs`: SRC path + `registry/native/target/...` → `toolchain/target/...`. +- CI (`ci.yml`, `ci-nightly.yml`, `bench.yml`): `make -C registry/native`, + the rust-cache `workspaces:` mapping, and the + `hashFiles('registry/native/Cargo.lock')` cache key → `toolchain/`. +- Repo-root `Cargo.toml`: add `exclude = ["software"]`. +- `pnpm-workspace.yaml`: `software/*`, `software/*` → + `software/*` (single glob covers commands + agents) plus `test-harness`. +- `toolchain/Makefile`: replace the fixed `c/programs/*.c` source list with a + glob over `../software/*/native/c/*.c`, and cargo members over the workspace. + +## Toolchain CLI vs justfile boundary + +**Principle:** `agentos-toolchain` (the published CLI) owns everything a +third-party author of *any* `@scope/*` package needs — author → compile → test → +package → publish. The `justfile` owns only what is specific to building and +releasing *this* repo's registry. Nothing generic to package authoring should +live only in our `justfile`/`Makefile`. + +Today the CLI stops at packaging (`pack`, `pack-aospkg`, `stage`, `build`, +`publish`); `stage` consumes an *already-compiled* `--commands-dir`. The two +hardest capabilities — **compiling** a command to WASM and **testing** it in a VM +— live only in the repo Makefile/harness, so external authors can't do them. +This refactor closes that gap. + +### New published toolchain verbs + +1. **`agentos-toolchain compile []`** — compile the package's + `native/` (Rust crate(s) or C source) to a WASM command binary against the + **pinned agentOS sysroot**, emitting into the commands dir that `stage` + consumes. Wraps sysroot + build-std + wasm-opt. That pipeline is heavy, so the + CLI fetches a **pinned toolchain bundle** (the way it already fetches wasi-sdk) + or runs a container image; the pin is versioned for reproducible external + builds. This is the capability that today only `registry/native/Makefile` has. +2. **`agentos-toolchain test []`** — boot a throwaway VM, register the + package, run its `test/` suite (and/or a command smoke-run) via the public + `@rivet-dev/agentos-runtime-core/test-runtime`. This is what our + `software//test/` scripts *and* external authors call — one runner. +3. **`agentos-toolchain validate []`** — lint `agentos-package.json`: + declared `commands` map to staged binaries; `registry` block + `category` + + `kind` present; limits bounded. Turns our repo-only coverage/layout checks into + a per-package check every author gets. +4. **`agentos-toolchain init []`** — scaffold a package + (`agentos-package.json` + `src/index.ts` + `test/`) instead of copying an + existing package by hand. + +### Shim crates must be publishable + +The shared Rust crates in `toolchain/crates/` (`wasi-ext`, `wasi-http`, +`wasi-spawn`, `builtins`, `shims`) are **path-deps today, unpublished** — so an +external Rust command cannot build against them. For `compile` to work outside +this repo, either publish them (crates.io) or have `compile` vendor them. Add +them to the publish set (`scripts/publish` discovery). + +### justfile = thin loop over the CLI + +Repo recipes orchestrate; they must not reimplement what the CLI does: + +| Recipe | Body | +|---|---| +| `software-build` | for each `software/*`: `agentos-toolchain compile && stage && build` | +| `toolchain-build` | build/pin the shared sysroot bundle the CLI consumes | +| per-package `test` script + external authors | `agentos-toolchain test` | + +**Stays repo-specific** (not CLI, not overfit): `copy-wasm-commands` (vendor into +runtime-core), `verify-fixed-versions` (the 0.0.1 pin), `generate-secure-exec-mirror`, +registry-wide release orchestration, cross-repo dispatch, and the status +reporter / coverage gate scoped to *our* registry. + +## Migration phases + +1. **Scaffold `toolchain/`** — move `registry/native/{Cargo.*, rust-toolchain, + Makefile, patches→std-patches, scripts, tests→conformance}` and the shared + crates + sysroot. Repoint the workspace + Makefile + CI + copy-commands, and + **rename the `registry-*` justfile recipes** (`software-*` / `toolchain-*` per + the table above) plus their callers. Build green with crates still under + `toolchain/crates/commands/` (pre-split). +2. **Flatten catalog + harness** — `git mv software/* software/`, + `git mv software/* software/`; create `test-harness/` from the old + `registry/tests/helpers.ts`+`terminal-harness.ts`; give each package a `test` + script; update `pnpm-workspace.yaml`; add `kind` to agent manifests; relocate + leftover `registry/` files; delete empty `registry/`. +3. **Colocate command source** — move each `cmd-` + its 1:1 lib into + `software//native/`; move C command sources into + `software//native/c/`; C test-programs stay in `toolchain/test-programs/`; + apply the `Cargo.toml` exclude + cross-dir members; resolve orphan commands. +4. **Add `category`** to every `registry` manifest block (grouping metadata). +5. **Productize the toolchain** — add `compile` / `test` / `validate` / `init` + verbs to `agentos-toolchain`; make the shim crates publishable; rewrite the + `software-build` / `toolchain-build` recipes as thin loops over the CLI; point + each package `test` script at `agentos-toolchain test`. (Can land incrementally + after the move; the move doesn't block it, but the CLI is the long-term home.) +6. **Enforce** — structural test (below) + coverage gate; delete dead paths. + +## Enforcement (`scripts/check-layout.mjs`, CI) + +Fail the build if: +- any `*.test.ts` sits outside an allowed home (`software//test/`, + `toolchain/conformance/`, `packages/*/tests/**`); +- any `software/*` command package lacks `test/` (allowlist meta bundles + + external wrappers); +- `registry/` exists; +- a crate under `software/*/native/crates/` is claimed by the main workspace + (guards the Cargo split). + +## Risks & decision + +1. **Cargo cross-dir members (biggest).** `toolchain/Cargo.toml` owning + `../software/*/native/crates/*` plus repo-root `exclude = ["software"]` is + valid but unusual; some tooling assumes members live under the root. Two known + facets: `cargo metadata` on cross-dir members, and **rust-analyzer/`rust-toolchain` + resolution** — a colocated crate opened directly walks up to the repo-root + (stable) toolchain, not `toolchain/`'s nightly, so IDE builds may use the wrong + channel unless the harness always drives builds from `toolchain/`. **Decide: + full colocation (this spec) vs. the C-only fallback** (Rust crates stay in + `toolchain/crates/commands/`). Recommendation: attempt full colocation in a + throwaway branch first; fall back if `cargo metadata`/rust-analyzer misbehave. +2. **Makefile source discovery.** Building must glob C sources and cargo members + across `software/*` instead of a fixed list — a one-time Makefile rewrite. +3. **No build isolation.** Building one package still needs all of `toolchain/` + (sysroot + shared crates). Colocation improves navigation, not build scope. +4. **Shared crates aren't self-contained.** `software/coreutils/native/crates/sh` + still depends on `toolchain/crates/builtins`; a package folder is not fully + standalone. Accepted, documented in rule #3. +5. **Sysroot distribution for `agentos-toolchain compile`.** The compile verb + needs the patched sysroot + nightly + wasm-opt, which are large and + platform-specific. **Decide** how the published CLI ships them: a pinned + downloadable bundle (like wasi-sdk) vs. a container image. Until decided, + `compile` works in-repo only and external authors still can't build — so this + gates the "same flow for external users" goal, not the internal move. diff --git a/docs-internal/registry-parity-worklist.md b/docs-internal/registry-parity-worklist.md new file mode 100644 index 0000000000..b8e34887a5 --- /dev/null +++ b/docs-internal/registry-parity-worklist.md @@ -0,0 +1,888 @@ +# Registry Linux-Parity Worklist + +Status: worklist · Owner: registry · Last updated: 2026-07-08 + +## Goal (hand this to the driver agent) + +> Drive every item in this worklist to **clean Linux parity**: each command/ +> behavior must work end-to-end the way it does on real Linux, **proven by real +> e2e tests** — not by a WASI-specific port, a stub, or a shim that only satisfies +> the test. Example of the bar: `duckdb` must run real analytical SQL against real +> file-backed databases and pass real e2e tests — not a stripped "WASI duckdb" +> that only does `SELECT 1`. +> +> **Rules:** +> - **🚧 REAL TOOL, NOT A REIMPLEMENTATION (the load-bearing rule).** Every command +> must be the **real upstream tool** (GNU coreutils, GNU grep/sed/gawk, real +> `curl`, real `git`, real `jq`, GNU tar/gzip/diffutils, …) compiled to +> `wasm32-wasip1` and **patched as needed** for WASI. Do **NOT** ship a +> from-scratch Rust/C rewrite, a stub, or a hand-rolled CLI over a library. +> Reimplementations drift from Linux behavior in a thousand small ways and are +> exactly why several commands fail parity. Sole exception: when the upstream +> canonical tool *is itself* the Rust project (**ripgrep**, **fd**) — then the +> real project is correct. Prefer the genuine upstream tool (real git, real +> grep) over a rewrite; a *popular, established* reimplementation is an +> acceptable fallback only when the real tool genuinely won't build. +> - **"Not possible" is a valid outcome — but only after trying really hard.** The +> sysroot is **ours**: a patched Rust std + libc with custom host-syscall imports +> (see CLAUDE.md → Software Build (WASM Toolchain)). A missing libc/POSIX API +> (`getrlimit`/`RLIMIT_NOFILE`, `getgroups`, …) is **NOT** a WASI wall — it is a +> stub/patch we add one layer down, and the build should proceed as if targeting +> native POSIX. Only if a command *still* cannot be built as the real (or an +> established) tool do you mark it **`NOT POSSIBLE (WASI)`** in this doc, with a +> concrete explanation of the genuine, documented wall (never "WASI lacks a +> syscall we could implement") and what was tried. Exhaust real options first: +> patch the sysroot, patch the tool, stub the specific missing syscall — a +> genuine effort, not a quick bail. +> - **Commit clean revs — no stray artifacts.** Each rev must contain only the +> intended source + test changes. Never commit build outputs, vendored toolchain +> trees, `__pycache__`/`*.pyc`, generated binaries, or anything that belongs in +> `.gitignore`. Before `jj describe`, run `jj diff -r @ --summary` and confirm +> every path is intended — watch especially for `A` (added) paths under +> `toolchain/`, `**/target/`, `**/node_modules/`, `**/build/`, `**/__pycache__/`. +> Then **audit the entire stack up to main** (`jj diff -r 'main..@' --stat`, or +> per rev) and strip anything that slipped in with `jj restore --from +> `, adding the pattern to `.gitignore` so it cannot recur. +> - **One jj rev per item.** Concretely: **`jj new` before starting each item**, +> make that command's fix *and* its e2e test in that single change, `jj describe` +> it with a clear conventional-commit message, then `jj new` again for the next +> item. One command per rev — never batch two commands (or unrelated changes) +> into one rev. Verify the folder + branch first (`pwd`, `jj log -r @`) since the +> working copy is shared. +> - **Parity, not workarounds.** Fix the real cause (VFS syscall, shell semantics, +> link conflict, missing feature). If a WASI limitation forces a deviation from +> Linux, that is a finding to surface — not something to paper over in the test. +> - **Real tests are the deliverable.** A fix isn't done until an un-skipped e2e +> test exercises the real behavior in a VM and passes. No `describe.skip`, no +> assertions weakened to match broken output. +> - Work top-down by priority. Re-verify with the actual VM run, not just typecheck. + +## Priority tiers + +- **P0 — runtime/VM correctness**: bugs in the shared runtime that silently + corrupt data or break process control. Highest blast radius. +- **P1 — broken shipped commands**: packages that build but don't work like Linux. +- **P2 — build/compile failures**: packages that can't be produced at all. +- **P3 — disabled/absent coverage**: real behavior exists but no real test proves it. + +## Known CPU interval-timer boundary + +Disabled `ITIMER_VIRTUAL` and `ITIMER_PROF` timers now match Linux: `getitimer` +returns a zero timer and `setitimer` accepts a zero value. Arming either timer +still returns `ENOTSUP`. The concrete remaining wall is asynchronous delivery +during pure guest computation: V8 executes a WASM call synchronously, and the +runner has no per-instruction or safepoint callback through which it can deliver +`SIGVTALRM`/`SIGPROF`. The runtime has a per-thread CPU clock for its outer V8 +watchdog, but it can only terminate the isolate; it cannot enter the guest's +signal machinery at the exact CPU-time deadline. Polling at imported syscalls +would silently fail for a tight WASM loop, so it is not presented as parity. +Closing this gap requires a V8 interrupt/safepoint hook or instrumented WASM +fuel checkpoints that distinguish user CPU from user-plus-system CPU. + +--- + +## ⚠️ Cross-cutting #0 — Command provenance: replace reimplementations with real tools + +**This is the highest-leverage item and reshapes several below.** Audit revealed +that **most commands are NOT the real Linux tool** — they are custom Rust rewrites +(`secureexec-*` crates) or `uutils`, plus at least one hand-rolled C CLI (curl). +Per the load-bearing rule, each must become the **real upstream tool** compiled to +WASI and patched as needed. + +**Rule (your call):** an **established project** — whether it's the real upstream +tool *or* an established third-party package that does the real work (uutils, +jaq, etc.) — is **fine**. **Custom code we wrote ourselves** is **not** and must +be replaced with a real/established implementation. Audit of every command's +actual backing: + +### ✅ Established — keep (real upstream tool or established package doing the work) +| Command(s) | Backing | +|---|---| +| coreutils (`sh`+80) | **uutils** (`uucore`) — established Rust project | +| duckdb, vim | real upstream C source, patched for WASI | +| sqlite3 **engine** | real SQLite amalgamation (⚠️ but the *CLI* is ours — see below) | +| jq | **jaq** (`jaq-core/std/json`) — established Rust jq | +| yq | jaq + `serde_yaml`/`toml`/`quick-xml` — established parsers (thin glue is ours) | +| sed | `sed` crate (published) | +| gawk (`awk`) | `awk-rs` crate (published) | +| tar | `tar` crate (established) | +| gzip | `flate2` (established) | +| diffutils (`diff`) | `similar` crate (established) | +| file | `infer` crate (established magic-byte lib; note: not real libmagic `file`) | + +### ❌ CUSTOM WE BUILT — flag & replace with a real/established impl +| Command | Status | What it actually is | Replace with | +|---|---|---|---| +| **curl** | DONE | our custom driver over a libcurl fork | real `curl` CLI (upstream `src/tool_*.c`) | +| **wget** | DONE | our 174-line `wget.c` (dropped) | real GNU Wget vs our sysroot — stub `getrlimit`/`getgroups`, then build | +| **http-get** | DONE | our 95-line `http_get.c` | dropped; real curl covers HTTP fetches | +| **git** | DONE | our hand-rolled git from `sha1`+`flate2` | **real git** (upstream C), patched for WASI — **NOT gitoxide** | +| **fd** | DONE | our `secureexec-fd` on raw `regex` (not sharkdp/fd) | real **fd** (sharkdp) | +| **findutils** (`find`,`xargs`) | DONE | our hand-rolled on `regex`/shims | replaced with `uutils/findutils` | +| **tree** | DONE | our hand-rolled, zero deps | real `tree`, or an established one | +| **grep** | DONE | our `secureexec-grep` on raw `regex` (**not** an established grep pkg) | real **GNU grep** | +| **ripgrep** (`rg`) | DONE | our `secureexec-grep` recursive search shim, not real ripgrep | real upstream **ripgrep** | +| **zip** | DONE | our 203-line `zip.c` over zlib/minizip (not Info-ZIP) | real Info-ZIP, or an established lib's CLI | +| **unzip** | DONE | our 669-line `unzip.c` over zlib/minizip | real Info-ZIP unzip | +| **sqlite3 CLI** | DONE | our 558-line `sqlite3_cli.c` (engine is real SQLite; the shell is ours) | real SQLite `shell.c` (its official CLI) | +| **vix** | DONE | from-scratch source-less drop-zone binary | deleted; real `vim` covers the editor slot | + +Note: `codex`/`codex-exec` = the rivet fork of OpenAI's codex — established fork, +external build (tracked separately in #9). + +**Objective:** replace each ❌ with a real/established implementation built to +`wasm32-wasip1` and patched only where WASI forces it. The ✅ rows stay. + +**Networking — DONE.** Real curl and GNU Wget now terminate TLS in-guest with +mbedTLS against the VM CA bundle, and Git links the same libcurl through a real +`git-remote-http` helper. HTTPS certificate verification and trust overrides, +curl's native exit taxonomy and compression, Wget HTTPS/FTPS, and Git smart-HTTP +clone/fetch/push are covered by parity tests. OpenSSH separately provides direct +SSH and git-over-SSH transport. See **`docs-internal/networking-parity-spec.md`** +for the original design decisions and [TLS & SSL](../website/src/content/docs/docs/architecture/tls-ssl.mdx) +for the current architecture. + +**Approach:** one command at a time, one jj rev each: swap our custom code for the +established source (fetched + pinned like sqlite/duckdb), wire into the toolchain, +patch for WASI, prove parity with real e2e tests. + +**Interaction with other items:** subsumes several below — curl (#6) is "build the +real curl," and the `no-test` packages (#12) that are ❌ here should move to a real +impl *before* their tests are written, so the tests validate real behavior. + +**Decisions (settled):** git → **real git** (not gitoxide). grep → **real GNU +grep**, or a popular established grep if the real one won't build. + +**git — where the issues are (assessment):** +- **LICENSE — RESOLVED, not a blocker.** Each command ships as its **own published + npm package**, so a GPL-2.0 git binary in `@agentos-software/git` is **mere + aggregation** — it does not affect the Apache-2.0 licensing of agentOS or any + other package. Ship the git package under GPL-2.0 (offer its source) and we're + compliant. This **supersedes** the clean-room-reimpl rationale in + `toolchain/std-patches/git/README.md` ("cannot be vendored due to license + restrictions"): that premise no longer holds — go with **real git** (upstream C) + and update/remove that README. (gitoxide stays ruled out.) +- **Technical WASI issues if we do build real git** (from that README + git's own + build knobs), easiest → hardest: + - `mmap` (packfiles/index) → build `NO_MMAP=1` (malloc+read). Fine. + - signals (SIGPIPE/SIGCHLD) → build without; WASI has none. Fine. + - threads (index-pack/pack-objects) → `NO_PTHREADS`; single-threaded, slower. Fine. + - `fork()`+`exec()` in `run_command.c` (hooks, filters, remote helpers) → route to + `posix_spawn` via the `wasi-spawn` broker (spawn IS supported — same fix as wget). + - **Network transport (clone/fetch/push) — the hard part.** Smart-HTTP needs the + `git-remote-https` **helper subprocess** + libcurl; `git://` needs raw sockets; + ssh needs an `ssh` subprocess. Each helper must itself exist as a module. + - symlink checkout → `core.symlinks=false` fallback (WASI symlink support is + partial); local time → UTC like elsewhere. +- **Bottom line:** license is a non-issue (separate published package = mere + aggregation). **local** git (init/add/commit/log/diff/branch/merge/status/ + checkout) is very achievable; **remote** git (clone/fetch/push over + smart-HTTP/ssh) is the real effort. Proceed with real git. + +**Replacement findings — what each remaining ❌ tool will take (investigated):** + +The recurring wall is **never a syscall** — it's one of two known, already-solved +patterns: **(a) no threads** on `wasm32-wasip1` → serial patch like +`toolchain/std-patches/crates/uu_sort/0001-wasi-serial-sort.patch` (hits real fd & +ripgrep crates); **(b) gnulib `getrlimit`/`getgroups`** → sysroot stubs, already +documented for wget (item #8) (hits GNU grep/findutils). Subprocess spawn already +works (`wasi-spawn` broker), so `xargs` is not a blocker. + +- **git — DONE.** Replaced the custom Rust `sha1`+`flate2` implementation with real + upstream Git 2.55.0 built by the C toolchain and staged as `git` plus helper + command aliases. The WASI changes stayed below Git's behavior surface: + `run-command.c` uses `posix_spawn` so helper subprocesses go through the existing + wasi-spawn broker, the sysroot exposes Git's missing C compatibility surface, and + the runner now allocates synthetic fds in a high range so managed pipe/file fds + cannot collide with delegate-opened WASI fds. Smart HTTP remains intentionally + disabled in this build (`NO_CURL`), so HTTPS clone fails with the real Git helper + error instead of a custom transport. Proof: clean upstream Git rebuild passes in + `2026-07-08T11-28-00-0700-git-clean-rebuild-after-high-synthetic-fd.log`; + package build stages 6 commands in + `2026-07-08T11-33-00-0700-git-package-build-clean-binary-after-install.log`; + native sidecar rebuild passes in + `2026-07-08T11-34-00-0700-sidecar-rebuild-after-git-clean-package.log`; + full Git e2e passes 18/18 in + `2026-07-08T11-51-00-0700-git-full-e2e-high-synthetic-fd-clean-binary-after-test-fix.log`. + Rev: `tmvlxlvk` — `fix(git): build upstream git`. +- **sqlite3 CLI — DONE.** Engine was already the real amalgamation + (`libs/sqlite3/sqlite3.c`); the command now builds the official upstream + `shell.c` from the same fetched zip as `sqlite3`. The local 558-line + `sqlite3_cli.c` reimplementation is deleted, `toolchain/c/build/sqlite3` is the + primary C output, `sqlite3_cli` remains only as a compatibility alias, and the + tracked runtime-core fallback command is refreshed to the same official shell. + Proof: official shell build passes in + `2026-07-08T05-04-47-0700-sqlite3-official-shell-build-command-name.log`; + package-focused e2e passes 16/16, including real `.tables`, `.schema`, and + `.dump` CLI arguments, in + `2026-07-08T05-07-06-0700-sqlite3-official-shell-tests-final-focused.log`; + package build/check-types pass in + `2026-07-08T05-07-52-0700-sqlite3-package-build-official-shell-final.log` and + `2026-07-08T05-07-52-0700-sqlite3-check-types-official-shell-final.log`; + runtime-core fallback command path passes `.tables` in + `2026-07-08T05-09-12-0700-sqlite3-runtime-core-command-fallback-test.log`; + aggregate C `programs` builds 57 commands in + `2026-07-08T05-09-53-0700-sqlite3-make-programs-final.log`. + Rev: `typytnkk` — `fix(sqlite3): build official SQLite shell`. +- **http-get — DONE.** Dropped the 95-line raw-socket loopback client instead of + porting it: real curl now covers HTTP fetch behavior in DuckDB remote CSV tests, + runtime cross-network loopback tests, and the C conformance loopback row. + Removed the `@agentos-software/http-get` package, fallback command, shell/core + dependencies, registry listing, C command install entries, and lockfile package + edges. The low-level `http_get_test` fixture remains because it is a socket + diagnostic test program, not a shipped registry command. Proof: pnpm lockfile + refresh succeeds in + `2026-07-08T05-39-44-0700-http-get-pnpm-lock-refresh.log`; core and shell + type checks pass in + `2026-07-08T05-40-57-0700-http-get-core-check-types-after-install.log` and + `2026-07-08T05-41-34-0700-http-get-shell-check-types-final.log`; DuckDB test + file typecheck passes in + `2026-07-08T05-40-57-0700-http-get-duckdb-test-typecheck-after-install.log`; + aggregate C `programs` builds 57 commands, with `http_get` absent after stale + generated cleanup/install, in + `2026-07-08T05-41-34-0700-http-get-make-c-programs-without-command.log` and + `2026-07-08T05-43-24-0700-http-get-clear-stale-generated-and-install.log`; + cross-runtime network tests pass 11/11 with the WASM curl rows in + `2026-07-08T05-47-43-0700-http-get-runtime-cross-network-test-pass.log`. +- **tree — DONE.** Replaced the custom Rust `secureexec-tree`/`cmd-tree` crates + with upstream Steve Baker `tree` 2.3.2 from `OldManProgrammer/unix-tree`. + It builds as a C toolchain command from pinned source, stages into + `@agentos-software/tree`, and refreshes the tracked runtime-core fallback + command. Sysroot fixes live one layer down: install `` and provide + deterministic missing-group lookup stubs so upstream `-g` support links + without a tree-source WASI branch. Proof: upstream source inspection in + `2026-07-08T05-13-50-0700-tree-fetch-upstream-2.3.2-inspect.log`; sysroot + patch check passes in + `2026-07-08T05-18-16-0700-tree-wasi-libc-patch-check-group-lookup-fixed.log`; + Makefile build passes in + `2026-07-08T05-20-02-0700-tree-upstream-make-build.log`; package build and + check-types pass in + `2026-07-08T05-21-08-0700-tree-package-build-upstream-after-install.log` and + `2026-07-08T05-21-08-0700-tree-check-types-upstream-after-install.log`; e2e + tree tests pass 6/6 in + `2026-07-08T05-29-45-0700-tree-vitest-upstream-final.log`; aggregate C + `programs` builds 58 commands in + `2026-07-08T05-30-44-0700-tree-make-programs-final.log`; Cargo metadata no + longer includes the deleted Rust tree crates in + `2026-07-08T05-33-17-0700-tree-cargo-metadata-after-removing-empty-dirs.log`. + Rev: `kpmrwxln` — `fix(tree): build upstream tree`. +- **fd — DONE.** Replaced the custom Rust `secureexec-fd` regex walker with + upstream sharkdp `fd-find` 10.4.2. Because `fd-find` is a bin-only crate, + `cmd-fd` now acts as the workspace build trigger while the toolchain builds + the upstream `fd` binary directly. WASI compatibility stays in dependency + patches: `fd-find` gets narrow file-type/path/receiver adjustments, and + `ignore` uses a serial walker on `wasm32-wasip1` instead of host threads. Proof: + clean patch dry-runs pass in + `2026-07-08T06-19-50-0700-fd-find-patch-clean-dry-run.log` and + `2026-07-08T06-26-52-0700-fd-ignore-patch-final-dry-run.log`; + clean upstream WASM build compiles `fd-find v10.4.2` in + `2026-07-08T06-21-54-0700-fd-upstream-wasm-rebuild-after-ignore-fix.log`; + runtime/package command hashes match in + `2026-07-08T06-22-42-0700-fd-copy-built-binary-hashes.log`; package build and + check-types pass in + `2026-07-08T06-24-27-0700-fd-package-build.log` and + `2026-07-08T06-25-41-0700-fd-check-types-final.log`; e2e fd tests pass 9/9, + including `fd --version` reporting `fd 10.4.2`, in + `2026-07-08T06-25-00-0700-fd-vitest-upstream-after-dir-format.log`. + Rev: `mrskpomv` — `fix(fd): build upstream fd-find`. +- **grep — DONE.** Replaced the `@agentos-software/grep` package's custom + `secureexec-grep` command wrapper with upstream GNU grep 3.12 from the official + GNU release tarball. The real GNU `grep` binary builds through the C toolchain; + `egrep` and `fgrep` are separate tiny WASM launchers that preserve GNU's + upstream obsolescent scripts by spawning `grep -E` / `grep -F` through the + AgentOS process broker. Sysroot fix stayed one layer down: wasi-libc no longer + advertises/exports its nonstandard two-argument `opendirat`, which conflicted + with gnulib's helper. Proof: official GNU release listing captured in + `2026-07-08T06-29-11-0700-grep-gnu-ftp-latest.log`; configure options captured + in `2026-07-08T06-29-39-0700-grep-upstream-configure-help-probe.log`; + sysroot `opendirat` patch check and rebuild pass in + `2026-07-08T06-34-27-0700-grep-wasi-libc-opendirat-patch-check-definition.log` + and `2026-07-08T06-34-33-0700-grep-sysroot-rebuild-opendirat-definition.log`; + GNU grep build passes in + `2026-07-08T06-35-01-0700-grep-gnu-wasm-build-after-opendirat-symbol.log`; + `egrep`/`fgrep` launcher builds pass in + `2026-07-08T06-40-07-0700-grep-egrep-wrapper-build.log` and + `2026-07-08T06-40-42-0700-grep-fgrep-wrapper-build.log`; package build and + check-types pass in `2026-07-08T06-42-24-0700-grep-package-build-final.log` + and `2026-07-08T06-42-18-0700-grep-check-types-final.log`; e2e grep tests + pass 8/8 in `2026-07-08T06-43-59-0700-grep-vitest-after-wrapper-cache-repair.log`. + Rev: `uyukolvr` — `fix(grep): build upstream GNU grep`. +- **ripgrep — DONE.** Replaced the `@agentos-software/ripgrep` package's custom + `secureexec-grep` recursive search shim with upstream ripgrep 15.1.0 from the + canonical `BurntSushi/ripgrep` crate/release. The local `cmd-rg` crate is now + only a build trigger, and `toolchain/Makefile` builds ripgrep's own `rg` bin + directly. The old `secureexec-grep` library is gone. Proof: latest upstream + release captured in `2026-07-08T06-50-40-0700-ripgrep-github-latest-release.json`; + crate metadata captured in `2026-07-08T06-52-00-0700-ripgrep-cargo-info.log`; + upstream WASM build passes in + `2026-07-08T06-56-00-0700-ripgrep-upstream-wasm-build-after-grep-dir-remove-fixed.log`; + package build and check-types pass in + `2026-07-08T06-58-55-0700-ripgrep-package-build-after-install.log` and + `2026-07-08T06-58-56-0700-ripgrep-check-types-after-install.log`; e2e ripgrep + tests pass 8/8 in `2026-07-08T07-02-00-0700-ripgrep-vitest-after-git-fixture.log`. + Rev: `msypkqmo` — `fix(ripgrep): build upstream ripgrep`. +- **zip / unzip — DONE.** Replaced both custom minizip-based C wrappers with real + upstream Info-ZIP Zip 3.0 and UnZip 6.0 release tarballs, built through the C + toolchain. Runtime/sysroot fixes stayed one layer down: temp-file, ownership, + `system`/`popen`/`pclose` compatibility in patched wasi-libc; overlay rename + over destination whiteouts; and WASI host-passthrough read/write offset + tracking after `fd_seek`. Proof: wasi-libc patch check passes in + `2026-07-08T08-34-29-0700-wasi-libc-patch-check-final.log`; native sidecar + build passes in `2026-07-08T08-34-07-0700-sidecar-build-final-runner-format.log`; + VFS rename regression passes in + `2026-07-08T08-34-29-0700-vfs-core-rename-whiteout-final.log`; final Zip e2e + passes 2/2 in + `2026-07-08T08-36-45-0700-zip-test-final-after-current-sysroot-copy-after-install.log`; + final UnZip e2e passes 6/6 in + `2026-07-08T08-37-12-0700-unzip-test-final-after-current-sysroot-copy-after-install.log`. + Rev: `nppnuxpr` — `fix(infozip): build upstream zip and unzip`. +- **findutils — DONE.** Replaced the custom Rust `find` crate with upstream + `uutils/findutils` 0.9.0 entrypoints for both `find` and `xargs`. Kept the fix + in the toolchain/sysroot layer: Rust WASI metadata extensions and process + `ExitStatusExt::signal()` are exposed for crates that expect Unix-like std + surfaces, while `xargs` still uses normal `std::process::Command` spawning + through the existing VM broker. Proof: `find` wasm build passes in + `2026-07-08T12-42-00-0700-findutils-find-build-after-permissions-mode-patch.log`; + `xargs` wasm build passes in + `2026-07-08T12-45-00-0700-findutils-xargs-build-after-find-success.log`; + package build passes in + `2026-07-08T12-51-00-0700-findutils-package-build-after-install.log`; sidecar + validation build passes in + `2026-07-08T12-35-00-0700-native-sidecar-build-after-forced-pnpm.log`; final + package e2e passes 5/5, including `xargs -n 2 echo` spawn batching, in + `2026-07-08T12-44-00-0700-findutils-vitest-uutils-after-depth-test-fix.log`. + Rev: `msknmmps`. + +Ranked easiest→hardest: **sqlite3-CLI · http-get (drop) · tree · fd · grep · +ripgrep · zip · unzip · findutils.** + +## Status tracking (how the driver reports progress in this doc) + +Update this doc as you go — it is the single source of truth for status. For each +❌ command, set one status and keep it current: + +- **`TODO`** — not started. +- **`IN PROGRESS`** — being built; note the current blocker if any. +- **`DONE`** — the real/established tool builds and passes a real un-skipped e2e + test; link the jj rev. +- **`NOT POSSIBLE (WASI)`** — only after a genuine effort. Write a concrete + explanation: exactly what blocks it, what you tried (sysroot patch, tool patch, + syscall stub), and why it can't be made to work. This is a documented dead-end, + never a silent fallback to a custom rewrite. + +Mark each row's status inline in the table (or as a short line under the command) +so a reader sees the whole board at a glance. + +--- + +## P0 — Runtime / VM correctness + +### 1. brush-shell `>>` append truncates instead of appending — DONE +- **Broken:** `execSync` with `>>` onto a write-only file overwrites instead of + appends. `expected 'changed' to be 'originalchanged'`. (issue: rivet-dev/agentos#1657) +- **Objective:** `>>` opens `O_WRONLY|O_APPEND` against the kernel VFS and appends, + identical to bash on Linux. +- **Proof:** `bridge-child-process.test.ts` append redirection tests pass + un-skipped; direct kernel append and native sidecar append regressions pass. +- **rev:** `ouxrzutq` — `fix(runtime): honor >> append mode in guest shell VFS redirection` + +### 2. brush-shell `cat < file` stdin redirection fails (exit 1) — DONE +- **Broken:** `cat < stdin-input.txt` exits 1 — input redirection from a VFS path + isn't wired to the command's stdin. (issue: #1657) +- **Objective:** `< file` feeds the VFS file to stdin; command reads it and exits 0, + like Linux. +- **Proof:** the "stdin redirection feeds the kernel VFS file" test passes + un-skipped after the parent host-shadow pre-spawn sync fix in item 1. +- **rev:** `lonnzuqw` — `test(registry): mark stdin redirection parity proven` + +### 3. WasmVM signal/dispose — SIGKILL/SIGTERM don't terminate; dispose hangs — DONE +- **Broken:** SIGKILL/SIGTERM don't kill guest processes; `dispose` times out + (5 tests across `signal-forwarding.test.ts`, `dispose-behavior.test.ts`). +- **Objective:** signals delivered to guest processes terminate them promptly and + `dispose` tears down active WasmVM + Node processes, matching Linux signal + semantics. **Not yet filed — file a separate issue.** +- **Proof:** `signal-forwarding.test.ts` passes 5/5 in + `2026-07-07T23-11-36-0700-item3-signal-forwarding-final-pass-2.txt`; + `dispose-behavior.test.ts` passes 3/3 in + `2026-07-07T23-11-21-0700-item3-dispose-behavior-final-pass.txt`. +- **rev:** `zkywnwup` — `fix(runtime): unblock WasmVM signal waits and dispose` + +### 4. VFS missing `pwrite` — sqlite3 file-backed DBs don't persist — DONE +- **Broken:** `filesystem method pwrite is unavailable` — sqlite3 file-backed DB + can't persist across exec calls. +- **Objective:** the VFS implements positioned writes (`pwrite`/`pwritev`) so any + command doing positioned I/O (sqlite3, and others) behaves like Linux. +- **Proof:** sqlite3 "file-based DB persists across separate exec calls" passes + in `2026-07-07T23-18-45-0700-item4-sqlite3-file-db-pwrite-pass.txt`; direct + mounted JS VFS `pwrite` test passes in + `2026-07-07T23-18-45-0700-item4-runtime-core-custom-vfs-pwrite-pass.txt`. + Type/build checks pass in `2026-07-07T23-19-11-0700-item4-runtime-core-build.txt` + and `2026-07-07T23-19-11-0700-item4-sqlite3-check-types.txt`. +- **rev:** `klrzzkro` — `fix(vfs): expose positioned writes in test kernel` + +### 5. Socket-layer failures (net-server/udp/unix, signal_handler) — DONE +- **Broken:** in the audit run, `st.create is not a function` + a `LinkError` in + net tests; signal_handler didn't catch signals. May be partial-build artifacts. +- **Objective:** TCP/UDP/Unix socket + signal test programs run to completion in + the VM with real socket semantics. **First reconfirm on a full build** — if it + reproduces, fix the socket-table wiring / link error. +- **Proof:** net-server/net-udp/net-unix/signal-handler suites pass together in + `2026-07-08T00-23-43-0700-item5-four-suites-take-signal-bridge.txt`. + Runtime and native sidecar builds pass in + `2026-07-08T00-24-02-0700-item5-final-runtime-core-build.txt` and + `2026-07-08T00-24-02-0700-item5-final-native-sidecar-build.txt`; native + embedded signal coverage passes in + `2026-07-08T00-24-02-0700-item5-final-native-embedded-runtime-signal-suite.txt`. +- **rev:** `zvyxkkyv` — `fix(runtime): repair Wasm socket and signal integration` + +--- + +## P1 — Broken shipped commands + +### 6. curl — reimplemented CLI, exits 1 on every operation (incl. `--version`) — DONE +- **Broken:** the `curl` command is a **hand-rolled `curl.c` driver** over a + libcurl fork, not the real curl command-line tool — so 24/30 `curl.test.ts` fail + and every op returns exit 1, even `curl --version`. +- **Objective (per Cross-cutting #0):** **build the real curl command-line tool** + (upstream `src/tool_*.c`) to `wasm32-wasip1` against the patched sysroot, + patched only where WASI forces it — replacing the custom driver. All real curl + behavior (GET/POST, `-I`/`-D`, `-L`, `-u`, `-F`, `-o`/`-O`, `-w`, `-K`) then + works because it *is* curl, not a shim. +- **Proof:** `software/curl/test/` passes un-weakened: 25 passed, 5 skipped in + `2026-07-08T00-41-57-0700-item6-curl-test-after-tls-flags.txt`. Runtime runner + build/protocol checks pass in + `2026-07-08T00-41-51-0700-item6-runtime-core-build-tls-flags.txt`. +- **rev:** `oxoqrwvk` — `fix(curl): build the real curl CLI for WASI` +- **Note (how well it works):** it *is* the real curl CLI (`src/tool_main.c`) plus a + custom `vtls/wasi_tls.c` backend (`USE_WASI_TLS`) — HTTPS runs through the host + TLS bridge, not OpenSSL. Real HTTP(S) `GET/POST/-I/-D/-L/-u/-F/-o/-O/-w/-K` all + work because it's genuine curl. Known gaps from the trimmed `./configure`: + `--compressed`/gzip response decode (`--without-zlib`), brotli/zstd, `libpsl` + cookie-suffix checks, LDAP, and no CA bundle (cert trust is whatever `wasi_tls` + enforces). Those are the 5 skipped tests. Verdict: solid for real HTTP(S). + +### 7. zip / unzip — replace custom wrappers with real Info-ZIP — DONE +- **Broken:** the shipped commands were custom C wrappers over zlib/minizip, not + real Zip/UnZip. The old fallback parser also diverged from hardened Linux unzip + behavior on wrapping local offsets, empty normalized names, and hostile size + declarations. +- **Objective:** build real upstream Info-ZIP Zip and UnZip to `wasm32-wasip1`, + patching the AgentOS sysroot/runtime where needed, and prove real zip↔unzip + roundtrips plus malformed-archive rejection in VM e2e tests. +- **Resolved:** upstream Info-ZIP Zip 3.0 and UnZip 6.0 now build from release + tarballs through the C toolchain. The custom `software/zip/native/c/zip.c` and + `software/unzip/native/c/unzip.c` sources are deleted. Required compatibility + lives one layer down: patched wasi-libc temp-file, ownership, `system`, `popen`, + and `pclose` surfaces; VFS overlay rename-over-whiteout cleanup; and + host-passthrough `fd_seek` offset tracking in the WASI runner. +- **Proof:** wasi-libc patch check passes in + `2026-07-08T08-34-29-0700-wasi-libc-patch-check-final.log`; native sidecar + build passes in `2026-07-08T08-34-07-0700-sidecar-build-final-runner-format.log`; + VFS rename regression passes in + `2026-07-08T08-34-29-0700-vfs-core-rename-whiteout-final.log`; `software/zip` + e2e passes 2/2 in + `2026-07-08T08-36-45-0700-zip-test-final-after-current-sysroot-copy-after-install.log`; + `software/unzip` e2e passes 6/6 in + `2026-07-08T08-37-12-0700-unzip-test-final-after-current-sysroot-copy-after-install.log`. +- **rev:** `nppnuxpr` — `fix(infozip): build upstream zip and unzip` + +--- + +## P2 — Build / compile failures + +### 8. wget — DONE +- **Resolved:** the `wget` command is real upstream GNU Wget 1.24.5 built for + `wasm32-wasip1`, HTTP-only for now (`--without-ssl --without-zlib + --without-libpsl --disable-iri`) against the patched AgentOS C sysroot. The old + custom 174-line wrapper stays removed. +- **Sysroot/runtime fixes:** Wget builds without a Wget-source WASI fork by adding + the missing POSIX surface one layer down: process/terminal headers including + `spawn.h`, signal/process/timezone compatibility, overrideable `FD_SETSIZE`, + Wget-only `_POSIX_TIMERS` overlay, POSIX socket `read`/`write` routing through + `host_net`, low host-net fds, and `MSG_PEEK` queue preservation in the WASM + runner. Configure is seeded so gnulib trusts the sysroot `select` instead of + replacing it with a host-net-incompatible fallback. +- **Proof:** focused basename download passes in + `2026-07-08T04-33-31-0700-item8-wget-vitest-focused-clean-msg-peek.log`; full + Wget e2e suite passes 5/5 in + `2026-07-08T04-33-41-0700-item8-wget-vitest-full-clean-msg-peek.log`. Final + runner syntax and wasi-libc patch checks pass in + `2026-07-08T04-34-02-0700-item8-node-check-wasm-runner-final.log` and + `2026-07-08T04-34-02-0700-item8-wasi-libc-patch-check-final.log`. +- **rev:** `zuosnzmq` — `fix(wget): build real GNU Wget for WASI` + +### 9. codex-cli — DONE +- **Resolved:** the `codex`/`codex-exec` package now has an AgentOS-owned wrapper + for the external `codex-rs` fork build. `make -C toolchain codex-required` + requires `CODEX_REPO=/path/to/codex-rs/codex-rs`, uses this checkout's + `toolchain/c/vendor/wasi-sdk`, and installs the fork-built optimized wasm into + generated `software/codex/wasm/{codex-exec,codex}` for the package build. The + generated toolchain and wasm command directories are ignored and not committed. +- **Test fix:** the real `codex-exec --session-turn` e2e now uses a streaming + Responses mock (SSE) and disables Codex shell snapshots inside the VM config, + avoiding the optional pre-turn shell-snapshot subprocess deadlock while still + driving the real codex-core agent and shell tool path. +- **Proof:** `CODEX_REPO=/home/nathan/agent-e2e/codex-rs/codex-rs make -C + toolchain codex-required` builds and installs 29,924,651-byte command artifacts + in `2026-07-08T01-37-05-0700-item9-codex-build-rerun.txt`; `pnpm --dir + software/codex-cli build` stages 2 commands and assembles `package.aospkg` in + `2026-07-08T01-44-50-0700-item9-codex-cli-build.txt`; + `AGENTOS_E2E_FULL=1 pnpm --dir packages/core exec vitest run + tests/codex-fullturn.test.ts --reporter=verbose` passes 2 real VM tests in + `2026-07-08T01-53-55-0700-item9-core-codex-fullturn-pass.txt`. +- **rev:** `svksnzon` — `build(codex-cli): make the codex-rs fork build reproducible` + +### 10. vix — DONE (deleted) +- **Resolved:** `vix` was a from-scratch, source-less drop-zone binary — exactly + the kind of hand-rolled artifact this repo should not carry. **Removed entirely** + (package dir, shell import/dep, `EXTERNAL_COMMANDS` Makefile hack, README rows, + website registry entry) in rev + `chore(registry): remove vix package; document real-tool (no-reimplementation) principle`. + Real `vim` (#11) covers the editor slot. Preserved source (`vix.c`, `BUILD-vix.md`, + `vix.wasm`) remains in `~/progress/agent-os/2026-06-28-just-shell-fix/` if ever + needed. No further work. + +--- + +## P3 — Disabled / absent coverage (real tests to Linux parity) + +For each: replace `describe.skip` with `describeIf(binaryPresent)` **and** write +real e2e tests that prove Linux-parity behavior — not smoke tests. + +### 11. Disabled suites — git, duckdb, codex — DONE +- **Fixed:** the Git quickstart, DuckDB package, and Codex full-turn suites are no + longer excluded from the default core Vitest file set, so coverage cannot + disappear when the package artifacts are present. +- **Status:** + - **git — DONE.** The core quickstart e2e now exercises real upstream Git in a + VM for local origin creation, commit, branch, clone, checkout, `log`, and a + working-tree `diff`. It validates only the git package it uses instead of + eagerly requiring every registry package to be built. Proof: + `2026-07-08T13-13-00-0700-item11-git-quickstart-final.log`. + Rev: `svltqsmx`. + - **duckdb — DONE.** Rebuilt the upstream DuckDB package artifact from the + patched C sysroot and strengthened the package e2e so it validates only the + DuckDB/Curl packages it uses. The VM e2e now covers file-backed DML, real + analytical SQL, DuckDB CSV export, `read_csv_auto` re-import, and the + negative HTTP-URL path for DuckDB itself. Proof: + `2026-07-08T12-38-07-0700-item11-duckdb-package-build-after-install.log`; + `2026-07-08T12-46-18-0700-item11-duckdb-package-e2e-final-file-pass.log`; + `2026-07-08T12-46-11-0700-item11-core-tsc-duckdb-final-file-after-refresh.log`; + `2026-07-08T12-46-10-0700-item11-duckdb-biome-check-final-file-after-refresh.log`. + Rev: `qrwnvouk`. + - **codex — DONE.** Un-skipped the remaining real `codex-exec --session-turn` + full-turn coverage. The suite now proves the model turn, on-request shell + tool call, real subprocess filesystem side effect, and adapter-supplied + history replay in one VM-backed file with 4/4 passing. For this coverage + rev, the package was staged from existing real 29 MB `codex`/`codex-exec` + artifacts because rebuilding the external Codex fork in this checkout hit + dependency gating failures (`path-dedot`, then `tokio`/`rustls-native-certs`); + the failed rebuild logs are kept as proof. Proof: + `2026-07-08T12-52-54-0700-item11-codex-cli-package-build-from-prior-artifacts-after-install.log`; + `2026-07-08T12-54-08-0700-item11-codex-fullturn-final-unskipped.log`; + `2026-07-08T12-54-42-0700-item11-core-tsc-codex-final.log`; + `2026-07-08T12-54-42-0700-item11-codex-biome-check-final.log`; + rebuild blockers: + `2026-07-08T12-48-46-0700-item11-codex-required-build-fresh-cargo-home.log`, + `2026-07-08T12-50-38-0700-item11-codex-required-build-path-dedot-cfg.log`. + Rev: `ryqtvoqv`. +- **Final proof:** after removing the remaining core Vitest exclusions, the + explicit suites pass with real VM coverage: Git quickstart 1/1 in + `2026-07-08T14-35-00-0700-item11-status-git-quickstart-final.log`; DuckDB + package 4/4 in + `2026-07-08T14-36-00-0700-item11-status-duckdb-package-final.log`; Codex + full-turn 4/4 in + `2026-07-08T14-37-00-0700-item11-status-codex-fullturn-final.log`. +- **rev:** `test(core): include registry parity suites by default` + +### 12. No tests at all — 9 software + 5 agents — DONE +- **Broken:** zero e2e coverage: `gawk, sed, tar, gzip, jq, yq, diffutils, + file, vim`; agents `claude, codex, opencode, pi, pi-cli`. +- **Status:** + - **gawk — DONE.** Added package-local VM e2e coverage for the staged `awk` + command. The suite proves file-backed field extraction, explicit field + separators, numeric aggregation, `-f` script-file execution, and missing + input-file errors through the packaged WASM command. Proof: + `2026-07-08T13-16-03-0700-item12-gawk-package-e2e-after-install.log`; + `2026-07-08T13-16-03-0700-item12-gawk-check-types-after-install.log`. + Biome is not applicable for this package test path; it reported the file is + ignored by config in + `2026-07-08T13-16-16-0700-item12-gawk-biome-check.log`. Rev: `pzxkurol`. + - **sed — DONE.** Added package-local VM e2e coverage for the staged `sed` + command. The suite proves file-operand substitutions, addressed `-n` + printing, addressed deletion, multiple `-e` expressions, and missing + input-file errors through the packaged WASM command. Proof: + `2026-07-08T13-17-46-0700-item12-sed-package-e2e-initial.log`; + `2026-07-08T13-17-46-0700-item12-sed-check-types-initial.log`. + Biome is not applicable for this package test path; it reported the file is + ignored by config in + `2026-07-08T13-17-55-0700-item12-sed-biome-check.log`. Rev: `wvpklkqv`. + - **tar — DONE.** Added package-local VM e2e coverage for the staged `tar` + command and tightened the tar wrapper for Linux-like directory listing and + missing-input error context. The suite proves archive creation/listing, + extraction into `-C` directories, gzip auto-detection by extension, + `--strip-components`, and missing create-input errors through the packaged + WASM command. Proof: + `2026-07-08T13-21-48-0700-item12-tar-toolchain-cmd-build-clean-vendor.log`; + `2026-07-08T13-22-51-0700-item12-tar-package-build-after-wrapper-fix.log`; + `2026-07-08T13-23-02-0700-item12-tar-package-e2e-final.log`; + `2026-07-08T13-23-02-0700-item12-tar-check-types-final.log`; + `2026-07-08T13-23-02-0700-item12-tar-cargo-fmt-check-final.log`. + Biome is not applicable for this package test path; it reported the file is + ignored by config in + `2026-07-08T13-23-12-0700-item12-tar-biome-check.log`. Rev: `rszmulmk`. + - **gzip — DONE.** Added package-local VM e2e coverage for the staged + `gzip`/`gunzip`/`zcat` commands. The suite proves file compression with + `-k`, source removal without `-k`, `gunzip -fk`, `zcat` streaming, and + overwrite protection without `-f` through the packaged WASM commands. Proof: + `2026-07-08T13-25-01-0700-item12-gzip-package-e2e-initial.log`; + `2026-07-08T13-25-01-0700-item12-gzip-check-types-initial.log`. + Biome is not applicable for this package test path; it reported the file is + ignored by config in + `2026-07-08T13-25-12-0700-item12-gzip-biome-check.log`. Rev: `tlstlwvy`. + - **yq — DONE.** Added package-local VM e2e coverage for the staged `yq` + command and fixed the wrapper to accept file operands instead of only + stdin. The suite proves YAML filtering, YAML-to-JSON query output, + explicit JSON/TOML/XML input formats, and invalid YAML parse errors through + the packaged WASM command. Proof: + `2026-07-08T13-27-36-0700-item12-yq-toolchain-cmd-build.log`; + `2026-07-08T13-28-59-0700-item12-yq-package-build-after-file-operands.log`; + `2026-07-08T13-29-10-0700-item12-yq-package-e2e-final.log`; + `2026-07-08T13-29-10-0700-item12-yq-check-types-final.log`; + `2026-07-08T13-29-11-0700-item12-yq-cargo-fmt-check-final.log`. + Biome is not applicable for this package test path; it reported the file is + ignored by config in + `2026-07-08T13-29-23-0700-item12-yq-biome-check.log`. Rev: `znlmtymu`. + - **diffutils — DONE.** Added package-local VM e2e coverage for the staged + `diff` command and fixed recursive directory output to print the compared + file pair before hunk output, matching the Linux `diff -r` shape. The suite + proves identical-file exit 0, normal and unified diffs, brief output, + ignore-case/whitespace/blank-line flags, recursive directory comparisons, + and missing-input errors through the packaged WASM command. Proof: + `2026-07-08T13-32-10-0700-item12-diffutils-toolchain-cmd-build.log`; + `2026-07-08T13-34-45-0700-item12-diffutils-package-build-after-recursive-header.log`; + `2026-07-08T13-35-04-0700-item12-diffutils-package-e2e-final.log`; + `2026-07-08T13-35-04-0700-item12-diffutils-check-types-final.log`; + `2026-07-08T13-35-04-0700-item12-diffutils-cargo-fmt-check-final.log`. + Biome is not applicable for this package test path; it reported the file is + ignored by config in + `2026-07-08T13-35-04-0700-item12-diffutils-biome-check.log`. + - **file — DONE.** Added package-local VM e2e coverage for the staged `file` + command and fixed shebang script classification to run before generic magic + detection, producing Linux-like script descriptions instead of raw + `text/x-shellscript`. The suite proves text, JSON, script, PNG, empty-file, + directory, brief, MIME, stdin, and missing-input behavior through the + packaged WASM command. Proof: + `2026-07-08T13-39-36-0700-item12-file-toolchain-cmd-build.log`; + `2026-07-08T13-40-33-0700-item12-file-package-build-after-shebang-fix.log`; + `2026-07-08T13-41-05-0700-item12-file-package-e2e-final-after-install.log`; + `2026-07-08T13-41-05-0700-item12-file-check-types-final-after-install.log`; + `2026-07-08T13-40-47-0700-item12-file-cargo-fmt-check-final.log`. + Biome is not applicable for this package test path; it reported the file is + ignored by config in + `2026-07-08T13-41-05-0700-item12-file-biome-check-final-after-install.log`. + - **vim — DONE.** Built/staged the real upstream Vim command without app + source forks by keeping terminal/process compatibility in the patched C + sysroot, disabling host Wayland/dlfcn detection at configure time, and + trimming the Vim-local bridge down to package-specific gaps. Added + package-local VM e2e coverage that proves the packaged binary starts with + `-libcall` and edits/writes a file in Ex mode with the packaged runtime. + Proof: + `2026-07-08T14-03-20-0700-item12-vim-toolchain-cmd-build-final-ioctl.log`; + `2026-07-08T14-04-03-0700-item12-vim-package-build-final.log`; + `2026-07-08T14-04-03-0700-item12-vim-check-types-final.log`; + `2026-07-08T14-04-09-0700-item12-vim-package-e2e-final.log`. + Biome is not applicable for this package test path; it reported the file is + ignored by config in + `2026-07-08T14-04-51-0700-item12-vim-biome-check.log`. + - **jq — DONE.** Added package-local VM e2e coverage for the staged `jq` + command and fixed the jaq-backed CLI wrapper to accept Linux-style file + operands instead of only stdin. The suite now proves version output, + file-backed array filtering, aggregate JSON construction, slurped NDJSON, + and invalid JSON parse errors through the packaged WASM command. Proof: + `2026-07-08T13-10-55-0700-item12-jq-toolchain-cmd-build-isolated-target.log`; + `2026-07-08T13-12-18-0700-item12-jq-package-build-after-wrapper-fix.log`; + `2026-07-08T13-12-51-0700-item12-jq-package-e2e-final.log`; + `2026-07-08T13-12-51-0700-item12-jq-check-types-final.log`; + `2026-07-08T13-12-51-0700-item12-jq-cargo-fmt-check-final.log`. + Rev: `slnmvuqz`. + - **pi — DONE.** Enabled the existing real `createSession('pi')` headless + suite in default core Vitest coverage and unskipped the upstream Pi SDK bash + tool path. The suite proves initialization over the native sidecar + transport plus real ACP write-tool and bash-tool flows inside the VM. Proof: + `2026-07-08T14-37-00-0700-item12-cc-cache-restored-target-files.log`; + `2026-07-08T14-37-00-0700-item12-sidecar-build-after-manual-cc-restore.log`; + `2026-07-08T14-38-00-0700-item12-pi-headless-final-after-cc-restore.log`. + Rev: `mzuuypsm`. + - **pi-cli — DONE.** Enabled the existing real `createSession('pi-cli')` + headless suite in default core Vitest coverage and unskipped the unmodified + Pi CLI bash-tool flow. Pi and pi-cli now project registry command software + only when the local `.aospkg` artifacts are built, so write-tool coverage + still runs while bash-tool coverage is gated on real command availability. + Proof: + `2026-07-08T15-00-00-0700-item12-pi-cli-cc-cache-restored-target-files-final.log`; + `2026-07-08T15-00-00-0700-item12-pi-cli-sidecar-build-after-cc-restore-final.log`; + `2026-07-08T15-01-00-0700-item12-pi-cli-focused-final-after-cc-restore.log`. + Rev: `xqtkmsyn`. + - **claude — DONE.** Enabled the existing real `createSession('claude')` + session suite in default core Vitest coverage, projected the actual + `@agentos-software/claude-code` agent package, and replaced the missing + test-only `xu` binary dependency with a real PATH-backed `sh` command from + registry coreutils. The suite proves Claude ACP shell/tool flow, text-only + responses, nested Node `execSync`/`spawn`, session metadata/lifecycle, + modes, and raw ACP sends. The flat node_modules fixture cache now lives + outside root `node_modules` so VM mounts survive dependency refreshes. + Proof: + `2026-07-08T15-17-00-0700-item12-claude-cc-cache-restored-after-cache-move.log`; + `2026-07-08T15-17-00-0700-item12-claude-sidecar-build-after-cache-move.log`; + `2026-07-08T15-19-00-0700-item12-claude-session-after-cache-move.log`. + Rev: `rxmoulty`. + - **opencode — DONE.** Added default core Vitest coverage for real + `createSession('opencode')` initialization through the projected + `@agentos-software/opencode` agent package. The focused suite proves the + sidecar resolves the OpenCode ACP package, initializes the adapter, exposes + agent metadata/capabilities/modes, and registers the session through the + Agent OS session API. Proof: + `2026-07-08T15-36-00-0700-item12-opencode-cc-cache-restored-focused.log`; + `2026-07-08T15-36-00-0700-item12-opencode-sidecar-build-focused.log`; + `2026-07-08T15-37-00-0700-item12-opencode-real-session-final.log`. + Rev: `xtnuomsw`. + - **codex — DONE.** Codex does not currently expose a runnable + `createSession('codex')` ACP package: `@agentos-software/codex` is a + registry/package wrapper and `codex-session.test.ts` verifies the sidecar + rejects it as a session agent. The real Codex agent coverage is the + `codex-exec --session-turn` VM path from item 11, which drives the real + codex-core turn loop against a mock OpenAI Responses server and proves a + real shell subprocess side effect. Proof: + `2026-07-08T15-43-00-0700-item12-codex-cc-cache-restored-target-files.log`; + `2026-07-08T15-43-00-0700-item12-codex-sidecar-build-before-fullturn.log`; + `2026-07-08T15-44-00-0700-item12-codex-fullturn-current-stack.log`; + `2026-07-08T15-45-00-0700-item12-codex-session-negative-current-stack.log`. + Rev: `kyswsrtv`. +- **Objective:** write real e2e tests proving each behaves like its Linux + counterpart (jq processes real JSON, sed edits streams, tar round-trips archives, + gzip round-trips, etc.); agents exercise the real ACP + adapter against the upstream SDK. +- **Proof:** `software//test/` exists and passes for each; coverage gate green. +- **rev:** one per package, e.g. `test(jq): add real JSON-processing e2e` + +--- + +## Cross-cutting / misc + +### 13. `everything` meta-package has no `agentos-package.json` — DONE +- **Fixed:** added the missing meta-package manifest, aligned the bundle with all + current command packages (`duckdb`, `envsubst`, `git`, `sqlite3`, and `vim` + were missing), and refreshed the workspace lockfile edges. +- **Proof:** `software/everything/test/everything.test.ts` proves the manifest is + present and the default export resolves every command package descriptor once. + Package build/check-types pass in + `2026-07-08T14-21-00-0700-item13-everything-build-after-install.log` and + `2026-07-08T14-21-00-0700-item13-everything-check-types-after-install.log`; + the package test passes 2/2 in + `2026-07-08T14-21-00-0700-item13-everything-test-after-install.log`; + layout validation passes in + `2026-07-08T14-22-00-0700-item13-check-layout.log`. +- **rev:** `fix(everything): add valid package manifest` + +--- + +## Sequencing note + +P0 first — several P1/P3 items depend on it: curl (#6) needs sockets/HTTP; +sqlite3 file-DB tests (#11) need pwrite (#4). Fix the runtime layer, then the +commands that ride on it, then backfill coverage. One jj rev per item throughout. + +--- + +## Candidate software (future additions) + +Constraint: **C or Rust only** (no Go/Haskell/Python). Real upstream tool or an +established project, same rules as above. **Focus: tools agents invoke headless +and programmatically** — no TUI/visual tools, no dev/build toolchains, no +raw-socket tools. Feasibility: 🟢 easy (fs/compute), 🟡 needs a known pattern +(spawn→`wasi-spawn`, PTY, host TCP/DNS bridge). + +**Atuin-validated priority (from 348K real shell commands):** by actual usage the +clear wins are **jj** (18,929 — 3rd overall after sed/rg/grep) and **tmux** +(1,268), then **perl** (563). Modest but real: ssh/rsync (~23 each), psql (20), +dig (10), redis-cli (8), less (4), openssl (3). **Zero usage in this history:** +zstd, xz, gpg, ffmpeg, age, mlr, socat, nc, screen — generically useful but not +agent-triggered here, so lower priority. Big unserved *demand*: +**ps (1,364) + pkill (866) + pgrep (755) ≈ 3K** — see process management below. + +**Cross-source validation (Homebrew + Debian popcon + agent-sandbox base images + +SWE-agent/OpenHands/Terminal-Bench trajectories):** independent sources converge +tightly on the list below. **Every agent-sandbox image (OpenHands, devcontainers, +GH Actions) pre-installs:** curl, wget, git, jq, tmux, gnupg, xz, zip/unzip, rsync, +ssh, less, vim, tree, procps, psmisc, socat, netcat, ripgrep, bzip2, lz4, sqlite3, +patch, file — near-exact overlap with what we ship/plan. Real **agent +trajectories** are dominated by coreutils (ls/rm/find/mkdir/cat/mv/chmod) + python ++ curl/wget + grep/sed + openssl + ps — all shipped or listed. New adds surfaced: +**imagemagick** ⭐ (C, image ops — high popularity), **openssl** (confirmed +high-use), **aria2 / brotli / parallel / sshpass** (base-image staples). Method +signals worth knowing: (1) agents **edit files ~2:1 over running shell commands**, +and the dominant shell idiom is "write a python repro script, run it, `rm` it" — +so a solid coreutils + python + curl/grep/sed/openssl core matters more than tool +breadth; (2) `git` is **rare inside agent turns** (harnesses extract the diff +out-of-band) but stays essential; (3) a **long tail of project-specific CLIs** +(`dvc`, `sqlglot`, `sanic`, …) comes from pip/npm install, not the registry. + +**Requested (add):** ssh 🟡, rsync 🟡, tmux/screen 🟡 (PTY — session persistence), +gpg 🟡, ffmpeg 🟡 (media transcode — heavy but headless), jj 🟢, dig 🟡, +nslookup 🟡, less ⭐🟡 (pager), openssl ⭐🟡 (TLS/certs/keys/hashing). +tail/head/cat are already in coreutils — confirm present. + +OpenSSH now carries a private, static OpenSSL 3.5.7 **libcrypto** dependency for +its standard software crypto algorithms. The `openssl` command remains a future +registry addition: libssl, providers/modules, and the CLI are not shipped, and +curl/wget/git continue to use mbedTLS. + +**Text / stream:** less ⭐🟡, **perl** ⭐🟡 (ubiquitous `-pe`/`-ne` text munging — +big C runtime but real; 563 uses in history), miller `mlr` 🟢 (CSV/JSON), +xmlstarlet 🟢, pcre2grep 🟢. (jq/yq/sed/awk/grep/head/tail already covered.) + +**Networking (host TCP/DNS bridge only):** openssl ⭐🟡, ssh 🟡, nc/netcat 🟡 +(TCP/UDP), socat 🟡, whois 🟢, dig/nslookup 🟡, redis-cli / psql client 🟡, +aria2 🟡 (C++ downloader), sshpass 🟢 (ssh password helper). + +**VCS:** git (item above), jj 🟢. + +**Crypto:** gpg 🟡, openssl 🟡, age 🟢 (Rust), minisign 🟢. + +**Compression:** xz, zstd, bzip2, lz4, brotli, p7zip (7z) — all ⭐🟢, common + easy. + +**Media / image:** ffmpeg 🟡 (transcode), imagemagick ⭐🟡 (C, image ops — high +popularity; agents do image work). + +**Files / sync:** rsync 🟡, diff/patch 🟢, rename 🟢, fdupes 🟢 (find/fd tracked +above). + +**Session:** tmux/screen 🟡 (PTY). + +**Process management (add — real procps-ng + psmisc, C; ps/pkill/pgrep ≈ 3K uses):** +- **Need the `/proc` prerequisite (below):** ps, pgrep, pkill, pidof, pstree, + killall, uptime, free, vmstat, w, pwdx, pmap 🟡. +- **Signal-only — already work via the kernel (no /proc):** kill, killall-by-PID. + (kill, sleep, timeout, env, nohup, nproc, nice/renice are coreutils — confirm + they're shipped via uutils rather than re-adding.) + +**⚙️ Runtime prerequisite — implement `/proc` (process-table-backed):** procps +reads `/proc//{stat,cmdline,status,comm}` and enumerates `/proc//`. The +**kernel already owns the process table** (`crates/kernel/src/process_table.rs`), +so expose a read-only procfs view of it to the guest (per-PID stat/cmdline/status ++ directory enumeration). Scope it minimal — just the fields procps parses, backed +by the existing process table, not a full Linux procfs. Unlocks the whole +ps/pkill/pgrep family (and top/htop later if ever wanted). **This is a runtime/VFS +item, do it before the procps packages.** + +**Excluded — not worth it / not possible here:** +- **TUI / visual-only:** gitui, lazygit, eza, dust, ncdu, bat, delta, broot, k9s, + skim/fzf — a terminal UI has no agent value. +- **top / htop — excluded (TUIs).** (ps/pkill/pgrep and the rest of procps-ng + + psmisc are ADD items above, gated on the `/proc` runtime prerequisite.) +- **Raw sockets:** ping, traceroute, mtr, nmap (need raw/ICMP, not just TCP). +- **ptrace:** strace, ltrace, gdb, lldb, valgrind — genuinely impossible on WASI. +- **Dev / build toolchains:** make, cmake, clang/gcc, binutils, pkg-config, + ctags — out of scope. +- **Go-only:** rclone, gh, kubectl — no C/Rust equivalent. diff --git a/docs-internal/registry-test-standardization.md b/docs-internal/registry-test-standardization.md new file mode 100644 index 0000000000..8bffd5b217 --- /dev/null +++ b/docs-internal/registry-test-standardization.md @@ -0,0 +1,16 @@ +# Software test layout + +Status: superseded by `docs-internal/registry-flatten-colocation-spec.md`. + +The old registry-wide test plan assumed centralized package tests and a native +build tree under the removed wrapper directory. That layout has been replaced: + +- package e2e tests live under `software//test/`; +- shared VM test helpers live in the private `@agentos/test-harness` workspace + package; +- libc/sysroot conformance tests live under `toolchain/conformance/`; +- C VM test fixtures live under `toolchain/test-programs/`; +- native command source is colocated under `software//native/`, while + shared build infrastructure remains under `toolchain/`. + +Use `scripts/check-layout.mjs` for the current structural gate. diff --git a/docs/wasmvm/supported-commands.md b/docs/wasmvm/supported-commands.md index 3b40bc3ba2..eb036f31ed 100644 --- a/docs/wasmvm/supported-commands.md +++ b/docs/wasmvm/supported-commands.md @@ -213,7 +213,6 @@ | Command | just-bash | Status | Implementation | Target | |---------|-----------|--------|----------------|--------| | curl | yes | done | `curl_cli.c` (libcurl HTTP-only, C program via `host_net`) | — | -| wget | yes | done | `wget.c` (libcurl-based, C program via `host_net`) | — | ## Version Control diff --git a/justfile b/justfile index 541c0f31e5..932e998c8c 100644 --- a/justfile +++ b/justfile @@ -8,33 +8,40 @@ release *args: release-preview REF: gh workflow run .github/workflows/publish.yaml --ref "{{ REF }}" -# --- @agentos-software/* registry packages (independent, PER-PACKAGE versions) --- -registry-native: - make -C registry/native commands +# --- @agentos-software/* software packages (independent, PER-PACKAGE versions) --- +toolchain-build: + make -C toolchain commands -registry-native-cmd name: - make -C registry/native cmd/{{ name }} +toolchain-cmd name: + make -C toolchain cmd/{{ name }} # Pre-flight for the publish "WASM Commands" job's fragile state: build the C # programs against the VANILLA wasi-sdk sysroot exactly like a fresh CI runner # (a locally-built patched sysroot is moved aside for the run). Catches # socket/netdb programs missing from PATCHED_PROGRAMS before CI does. -registry-native-preflight: +toolchain-preflight: #!/usr/bin/env bash set -euo pipefail - cd registry/native/c + cd toolchain/c if [ -e sysroot ]; then mv sysroot sysroot.preflight-stash; fi restore() { if [ -e sysroot.preflight-stash ]; then rm -rf sysroot; mv sysroot.preflight-stash sysroot; fi; } trap restore EXIT make wasi-sdk make programs -registry-copy-commands: +toolchain-copy-commands: node packages/runtime-core/scripts/copy-wasm-commands.mjs -registry-build: +software-build: pnpm --filter '@agentos-software/*' build +# Rebuild and stage the complete default WASM tool set from source. All outputs +# land in ignored build/bin/commands directories and must not be committed. +tools-rebuild: + just toolchain-build + just toolchain-copy-commands + just software-build + install-shell: #!/usr/bin/env bash set -euo pipefail diff --git a/package.json b/package.json index 4e38e9c42d..a17a126bfc 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "test:migration-parity": "pnpm --dir packages/core exec vitest run tests/migration-parity.test.ts --reporter=verbose", "test:post-python-parity": "pnpm --dir packages/core build && pnpm --dir packages/core exec vitest run tests/agentos-base-filesystem.test.ts", "test:watch": "pnpm exec turbo watch test", + "check-layout": "node scripts/check-layout.mjs", "check-types": "node scripts/verify-check-types.mjs && pnpm exec turbo check-types --concurrency=1", "lint": "pnpm biome check .", "fmt": "pnpm biome check --write --diagnostic-level=error .", @@ -22,14 +23,17 @@ "@biomejs/biome": "^2.3", "@copilotkit/llmock": "^1.6.0", "@mariozechner/pi-coding-agent": "^0.60.0", + "@agentos/test-harness": "workspace:*", "@rivet-dev/agentos-core": "workspace:*", "@agentos-software/claude-code": "workspace:*", "@agentos-software/codex": "workspace:*", "@agentos-software/common": "workspace:*", + "@rivet-dev/agentos-vm-test-harness": "workspace:*", "@rivet-dev/agentos-runtime-core": "workspace:*", "@agentos-software/pi": "workspace:*", "@types/node": "^22.19.15", "jszip": "^3.10.1", + "npm": "^11.18.0", "pdf-lib": "^1.17.1", "turbo": "^2.5.6", "typescript": "^5.9.2" diff --git a/packages/agentos-toolchain/README.md b/packages/agentos-toolchain/README.md index 72ce0f34ce..64180f005c 100644 --- a/packages/agentos-toolchain/README.md +++ b/packages/agentos-toolchain/README.md @@ -13,8 +13,8 @@ npx @rivet-dev/agentos-toolchain pack [options] packages lives here, next to the things it packages.** secure-exec owns the VM runtime that *runs* packages — the kernel, the VFS, the `/opt/agentos` mount, the `$PATH` command resolver, and the header/`binfmt` dispatch — and it owns the -package **definitions** themselves: the generic registry software (`registry/software/*`) -and the agent adapters (`registry/agent/*`). This toolchain is what builds those +package **definitions** themselves: the generic registry software (`software/*`) +and the agent adapters (`software/*`). This toolchain is what builds those definitions into the on-disk package format that the runtime resolves. Its `header.ts` is the same `binfmt` table the sidecar enforces (`crates/sidecar`), so keeping it in this repo keeps the producer and the consumer of the format in one diff --git a/packages/browser/tests/browser-wasm/pi-boot.spec.ts b/packages/browser/tests/browser-wasm/pi-boot.spec.ts index aeb5662d6d..a3e4b1b536 100644 --- a/packages/browser/tests/browser-wasm/pi-boot.spec.ts +++ b/packages/browser/tests/browser-wasm/pi-boot.spec.ts @@ -6,7 +6,7 @@ import { expect, test } from "@playwright/test"; // guest in the kernel-backed node-stdlib executor in real Chromium and speaks ACP. // // Requires the pi adapter bundle (built by scripts/build-wasm-test-assets when the -// registry/agent/pi dist + @mariozechner/pi-agent-core are present locally; skipped in +// software/pi dist + @mariozechner/pi-agent-core are present locally; skipped in // CI where they are absent). Runs the adapter in the executor's persistent-execution // mode (ExecOptions.persistent) so its async WHATWG-stream stdin pump can produce the // reply before the run completes. @@ -19,7 +19,7 @@ test("the real pi adapter boots in the converged executor and answers ACP initia const bundlePresent = await page.evaluate(() => fetch("/pi-adapter.bundle.cjs").then((r) => r.ok).catch(() => false), ); - test.skip(!bundlePresent, "pi adapter bundle not built (registry/agent/pi dist absent)"); + test.skip(!bundlePresent, "pi adapter bundle not built (software/pi dist absent)"); const result = await page.evaluate(() => window.__piBoot!.run()); diff --git a/packages/build-tools/bridge-src/global-exposure.ts b/packages/build-tools/bridge-src/global-exposure.ts index a76be74fee..57fee81cb5 100644 --- a/packages/build-tools/bridge-src/global-exposure.ts +++ b/packages/build-tools/bridge-src/global-exposure.ts @@ -40,6 +40,11 @@ var NODE_CUSTOM_GLOBAL_INVENTORY = [ classification: "hardened", rationale: "Host process signal-listener state bridge reference." }, + { + name: "_processTakeSignal", + classification: "hardened", + rationale: "Host process pending-signal drain bridge reference." + }, { name: "_osConfig", classification: "hardened", diff --git a/packages/core/AGENTS.md b/packages/core/AGENTS.md deleted file mode 120000 index 681311eb9c..0000000000 --- a/packages/core/AGENTS.md +++ /dev/null @@ -1 +0,0 @@ -CLAUDE.md \ No newline at end of file diff --git a/packages/core/AGENTS.md b/packages/core/AGENTS.md new file mode 100644 index 0000000000..8540ca7f41 --- /dev/null +++ b/packages/core/AGENTS.md @@ -0,0 +1,172 @@ +# agentOS Core Package + +`@rivet-dev/agentos-core` -- contains VM ops, ACP client, session management. + +**⚠️ CRITICAL INVARIANT: ALL guest code MUST execute inside the kernel with ZERO host escapes.** The VM is a fully virtualized OS — every file read, network connection, and process spawn goes through the kernel. Guest code must never touch real host APIs. The Node.js execution engine is currently broken (spawns real host `node` processes instead of V8 isolates). See `crates/execution/CLAUDE.md`. + +## AgentOs Class + +- Wraps the kernel and proxies its API directly. +- **All public methods must accept and return JSON-serializable data.** No object references (Session, ManagedProcess, ShellHandle) in the public API. Reference resources by ID (session ID, PID, shell ID). +- Filesystem methods mirror the kernel API 1:1 (readFile, writeFile, mkdir, readdir, stat, exists, move, delete). +- Command execution mirrors the kernel API (exec, spawn). +- `fetch(port, request)` reaches services running inside the VM using the kernel network adapter pattern (`proc.network.fetch`). +- **Cron scheduling stays in the TypeScript layer.** The Rust sidecar has no concept of cron jobs. Cron expression parsing, timer management, overlap policies, and job execution dispatch all live in the TypeScript SDK. +- Keep cron schedule validation and `nextRun` computation on the shared helpers in `src/cron/parse-schedule.ts`; if `CronManager` and `TimerScheduleDriver` parse or reject schedules differently, `listCronJobs()` can advertise jobs the driver refuses (or immediately fires) and the API becomes self-contradictory. +- Native sidecar execution requests should stay unresolved on the TypeScript side. Forward `command`, `args`, `cwd`, and VM config through the wire payload, and let Rust own command lookup, guest-path to host-path mapping, shadow materialization, and `AGENT_OS_*` runtime env assembly. +- Native sidecar `exec()` should keep shell-sensitive commands on the `sh -c` wrapper path so cwd changes, pipelines, and other shell semantics stay truthful, but shell-free simple commands can use the direct spawn fast path regardless of driver. For Wasm commands in `src/sidecar/rpc-client.ts`, direct spawn preserves the real guest exit status for external-command failures like `cat /missing`, while the `sh -c` wrapper can swallow that non-zero status even when stderr is correct. +- In `src/sidecar/rpc-client.ts`, `&&` command chains must stay on a single guest `sh -c` execution. Splitting them into separate `exec()` calls loses shell state like `cd` and changes where relative redirects write. +- In `src/sidecar/rpc-client.ts`, shell syntax in `exec()` and shell-mode `spawn()` always routes to guest `sh -c`. The only fast path is the shell-free direct spawn; never parse redirects or any other shell grammar in the bridge. +- In `src/sidecar/rpc-client.ts`, keep the shell wrapper as `cd ... || exit` followed by the target command and trust the shell process exit code directly. Temp-file or assignment-based `$?` capture on the brush path is brittle: shell redirection can leave the file empty, inject `exit` parse errors into stderr, and silently turn failing guest commands green. +- In `src/sidecar/rpc-client.ts`, the simple-command parser must preserve backslashes for non-shell-special escapes inside double quotes. Commands like `printf "a\\nb\\n"` rely on the guest command seeing the literal `\n` bytes; only `\"`, `\\`, `\$`, ``\` ``, and line-continuation newlines should collapse on the native-sidecar fast path. +- In `src/sidecar/rpc-client.ts`, treat bare unquoted `!` as shell syntax, not as a direct-fast-path token. Commands like `test ! -f /tmp/file` rely on guest shell semantics, and bypassing the shell can flip the observed exit code even when the underlying file operation succeeded. +- If a file must be visible to both `vm.readFile()` and guest shell commands, it cannot live only in a local compat mount. Put it on a real sidecar-visible path or mount, and keep any read-only guarantees enforced below the TypeScript proxy layer. +- Host tool registration is split across the boundary: TypeScript converts Zod schemas to JSON Schema, generates prompt markdown, validates sidecar tool invocations, and runs the local `execute()` callbacks, while the sidecar owns CLI flag parsing and `agentos` command dispatch via `registerHostCallbacks` / `RegisterHostCallbacks`. +- Host-tool `inputSchema` conversion in `src/host-tools-zod.ts` is intentionally fail-closed. Support only the Zod subset that round-trips cleanly into the sidecar-facing JSON Schema contract; if a schema would degrade semantics or emit `$ref`/`$defs` (`discriminatedUnion`, `intersection`, `tuple`, `record`, `date`, `bigint`, custom refinements, metadata `id`, etc.), throw `HostToolSchemaConversionError` with the offending field path instead of coercing it to `{ type: "string" }`. +- The host-tool description limit is a cross-boundary contract: keep the 200-character maximum aligned between `src/host-tools.ts` and Rust `RegisterHostCallbacks` validation in `crates/sidecar/src/tools.rs`, with boundary tests on both sides when changing it. +- `src/sidecar/rpc-client.ts` is the consolidated home for framed sidecar I/O, compat proxy helpers, and sidecar descriptor serializers. Keep shared/explicit sidecar pool and VM lease bookkeeping in `src/agent-os.ts` rather than reintroducing another sidecar lifecycle layer. +- In `src/agent-os.ts`, shell teardown is two-phase: public `_shells` entries can disappear immediately on `closeShell()`, but `dispose()` must still await the separate pending shell-exit set before dropping the sidecar event listener, or late shell stdout/exit delivery can race into a closed bridge. +- The native sidecar framed stdio path now defaults to the BARE payload codec. Keep any JSON payload support behind explicit migration-only opts such as `payloadCodec: "json"`, and remember that BARE structs need every positional field serialized explicitly across the Rust/TypeScript boundary rather than relying on JSON-style `skip_serializing_if` omissions. +- In `src/sidecar/native-process-client.ts`, treat `child.on("exit")` and `child.on("error")` as the authoritative terminal-disconnect path for framed stdio clients. `stdout` can close before Node fills in `exitCode`/`signalCode`, so reject in-flight RPCs with a typed disconnect immediately and upgrade the stored terminal error once the concrete exit metadata arrives. +- In the native-sidecar event path, long-lived background loops should call `waitForEvent()` in abortable no-timeout mode instead of parking a multi-hour timeout sentinel. The abort signal is the cancellation mechanism; the timeout itself becomes the regression surface on idle VMs. +- For native-sidecar BARE ACP session bootstrap payloads, keep `SessionCreatedResponse` aligned with `crates/sidecar/protocol/agentos_native_sidecar_v1.bare`: `sessionId` is the first positional field on the wire, before optional `pid`, `modes`, `configOptions`, `agentCapabilities`, and `agentInfo`. If the TypeScript decoder reads `session_id` last, every `createSession()` response desynchronizes. +- Public SDK type exports now funnel through `src/types.ts`; keep legacy kernel/runtime implementation helpers behind `src/runtime-compat.ts` and avoid adding new public root exports directly from runtime internals. +- When adding a new public SDK option/result/helper type under `src/agent-os.ts`, `src/json-rpc.ts`, `src/host-dir-mount.ts`, or other root-facing modules, mirror it through `src/types.ts` and keep `tests/public-api-exports.test.ts` aligned so the package entrypoint stays truthful. + +## Agent Sessions (ACP) + +- Uses the **Agent Communication Protocol** (ACP) -- JSON-RPC 2.0 over stdio (newline-delimited) +- No HTTP adapter layer; communicate directly with agent ACP adapters over stdin/stdout +- Reference `~/sandbox-agent` for ACP integration patterns. Do not copy code from it. +- ACP docs: https://agentclientprotocol.com/get-started/introduction +- Session design is **agent-agnostic**: each agent type has a config specifying its ACP adapter package and main agent package name +- Currently configured agents: PI (`@agentos-software/pi`), PI CLI (`@agentos-software/pi-cli`), OpenCode (`@agentos-software/opencode`), Claude (`@agentos-software/claude-code`), and Codex (`@agentos-software/codex` + `@agentos-software/codex-cli`). +- **No host agent exceptions.** Host-native wrappers and host binary launch paths are not allowed. OpenCode support must use the real upstream OpenCode implementation rebuilt into the VM adapter package and executed inside the VM. +- `createSession("pi")` spawns the ACP adapter inside the VM, which calls the Pi SDK directly +- Agents are resolved BY THE SIDECAR, not the client. There is no `AGENT_CONFIGS` table; `AgentType` is just `string` (a manifest `name`, defined in `types.ts`). The client is npm-agnostic and parses NO manifests: `createSession(name)`/`resumeSession(name)` send only `agentType` on the wire (there is no `adapterEntrypoint` field), and the SIDECAR resolves the entrypoint/env/launchArgs from the projected `/opt/agentos//current/agentos-package.json`. `listAgents()` is a sidecar ACP RPC (`AcpListAgentsRequest`) — the sidecar enumerates the projected `/opt/agentos` packages. Adding/changing an agent = changing its package manifest (in the secure-exec registry), not this file; verify launch args/env with the mock-adapter session tests. The Rust client (`crates/client`) behaves IDENTICALLY (also sends only `agentType`) — see the root `CLAUDE.md` client-parity/npm-agnostic rule. +- In `createSession()`, treat `skipOsInstructions` as "skip the base `/etc/agentos/instructions.md` text" only. Still call agent `prepareInstructions(...)` when session-level `additionalInstructions` or tool-reference content exists, and forward `skipBase` through the options object instead of blanking out the caller's extra instructions. +- ACP agents that issue live `session/request_permission` calls during `session/prompt` cannot rely on queued session events alone. Route those permission round-trips through the sidecar callback channel (`SidecarRequestPayload`) so the host can answer them before the prompt request completes. +- Native-sidecar inbound ACP host callbacks are explicit sidecar-request payloads now. If Rust forwards an unknown ACP JSON-RPC request, answer it through `SidecarRequestPayload.type === "acp_request"` with an `acp_request_result` JSON-RPC response; otherwise the sidecar will only synthesize `-32601` after the callback transport is unavailable or times out. +- Host ACP callbacks in `src/agent-os.ts` are no longer a generic `-32601` stub: keep the dispatcher aligned with both the newer `fs/read` / `fs/write` / `fs/readDir` / `terminal/*` method names and the legacy aliases still exercised by native-sidecar tests such as `fs/read_text_file` and `fs/write_text_file`. +- On the native sidecar path, a top-level `session/cancel` request does not preempt an already running top-level `session/prompt` dispatch. If prompt callers must observe cancellation immediately, resolve the pending prompt request locally in `src/agent-os.ts` while still forwarding the real cancel RPC for eventual adapter/process cleanup. +- Native-sidecar ACP request timeouts should surface as JSON-RPC errors with `error.data.kind === "acp_timeout"` rather than string-only transport errors. Use `isAcpTimeoutErrorData()` from `src/json-rpc.ts` instead of parsing timeout messages. + +### Agent Adapter Approaches + +Each agent type can have two adapter approaches: +- **SDK adapter** (default) -- Embeds the agent SDK directly via library import (`createAgentSession()`). Lower memory footprint (~100MB less for Pi). Binary: `pi-sdk-acp`. Package: `@agentos-software/pi`. Agent ID: `pi`. +- **CLI adapter** -- Spawns the full agent CLI as a headless subprocess via its ACP adapter (`pi-acp` spawns `pi --mode rpc`). Higher memory overhead but provides full CLI feature set. Binary: `pi-acp`. Package: `@agentos-software/pi-cli`. Agent ID: `pi-cli`. + +### Agent Configs + +An agent's launch config lives entirely in its `/opt/agentos` package `agentos-package.json` manifest (`agent.acpEntrypoint`/`launchArgs`/`env`). Resolution is SIDECAR-SIDE: the client sends only the agent `name` (the `createSession(name)` id), and the sidecar reads the projected `/opt/agentos//current/agentos-package.json` to resolve the entrypoint (`/opt/agentos/bin/`), the manifest `env` (applied as defaults; caller env wins), and `launchArgs` (prepended before caller args), then spawns. The client parses no manifests and holds no per-agent config. There is no second hardcoded config surface to keep in sync — a default shell/env tweak is a manifest change. + +## Testing + +- **Framework**: vitest +- **Prefer scoped tests while iterating.** + - `pnpm --dir packages/core exec vitest run tests/path/to/file.test.ts` or `pnpm --dir packages/core exec vitest run -t "test name pattern"` + - Repo-root `pnpm test` is the RC sweep and exits cleanly; it is still too broad for normal iteration. + - `pnpm --dir packages/core test` intentionally uses Vitest's `verbose` reporter because `tests/wasm-commands.test.ts` and similar long-running VM suites otherwise sit silent for minutes and get misread as hangs during `US-088` sweeps. + - Use low timeouts for test commands (60000ms max). +- The vitest setup file at `tests/helpers/default-vm-permissions.ts` patches `AgentOs.create()` and disposes every cached shared sidecar via `__disposeAllSharedSidecarsForTesting()` in `afterAll`. Workers can hang on exit if the shared sidecar's piped stdio handles stay open, so any new test entrypoints that bypass this setup file must dispose their sidecars themselves. +- `NativeSidecarProcessClient.dispose()` enforces a graceful exit window then `SIGKILL`s the child if it ignores stdin EOF; `tests/native-sidecar-process.test.ts` covers the regression so future changes cannot reintroduce an unbounded teardown wait. +- In `packages/core` tests that capture `spawn()` stdout/stderr via callbacks and then call `waitProcess(pid)`, drain one macrotask (`await new Promise((resolve) => setTimeout(resolve, 0))`) before asserting on the buffered strings. Native-sidecar `process_output` events can arrive one turn after the exit notification, and tiny outputs like `curl -s` bodies are the first thing to get lost if you snapshot immediately. +- `NativeSidecarProcessClient.waitForEvent(...)` supports indexed `SidecarEventSelector` objects; prefer selectors over ad hoc lambdas on shared sidecar clients so buffered events stay O(1) to retrieve and `ownership` can pin a wait to one VM/session. +- The native sidecar client's unmatched event buffer is intentionally bounded and fail-closed. If a test or runtime path can leave `runEventPump` idle while output events stream, expect `SidecarEventBufferOverflow` rather than unbounded buffering, and set a larger `eventBufferCapacity` explicitly only for cases that truly need it. +- When Node/Vitest code needs to shell out to Cargo, resolve it through `src/sidecar/cargo.ts` instead of assuming a login shell already put `~/.cargo/bin` on `PATH`. +- For `tests/wasm-commands.test.ts`, broad `-t "grep"` or `-t "sed"` filters can pull in unrelated `rg`, `gzip`, or cross-package pipeline coverage via substring matches. When a story only gates the `grep`/`sed` blocks, use the explicit case names or a narrower `--testNamePattern` that only matches those block entries. +- For `tests/wasm-commands.test.ts` and similar long-running VM truth suites, prefer one shared VM per `describe(...)` block over one VM per individual test unless the case truly needs pristine bootstrap state. Per-test VM boots push the file into multi-minute runtimes and make the RC sweep look hung even when it is still progressing. +- Cross-workspace software package suites import `@rivet-dev/agentos-core` from `packages/core/dist`, not directly from `src/`. After changing exported test-runtime code such as `src/runtime-compat.ts`, rebuild `packages/core` before trusting software/package Vitest results. +- The `examples/quickstart` package also resolves `@rivet-dev/agentos-core` from `packages/core/dist`; after TypeScript changes in `packages/core/src`, rebuild `packages/core` before rerunning quickstart acceptance commands. +- The synthetic `openShell()` fallback in `src/sidecar/rpc-client.ts` needs PTY-style output semantics for xterm-based harnesses: normalize terminal-visible line endings to `\r\n`, and route command stderr through the main `onData` stream instead of treating it like a separate non-PTY stderr channel. +- **Always verify related tests pass before considering work done.** +- **All tests run inside the VM** -- network servers, file I/O, agent processes. +- For `vm.exec()` cwd/path tests, prefer setting up files from inside the guest shell when the assertion is about command resolution or relative paths. VM filesystem API writes becoming visible to host-backed runtimes is a separate shadow-sync surface and should be tested independently. +- For active agent-session/bash-tool filesystem regressions, cover the host read path in `tests/filesystem.test.ts` with a Claude llmock prompt. Long-lived session processes keep writing into the sidecar shadow root after a tool call returns, so `vm.readFile()`/`vm.stat()` need shadow reconciliation before the session itself exits. +- Session tests that need launch argv or OS-instruction assertions should inspect `getSessionAgentInfo(sessionId)` from sidecar state instead of spying on `kernel.spawn`; `createSession()` now launches through sidecar RPCs. +- `closeSession()` is intentionally fire-and-forget. Cleanup tests can await the internal `_sessionClosePromises` map when they need deterministic post-close assertions, but active-prompt cancellation cases should trigger the public close and then assert on resource release plus prompt error outcome separately, because the in-flight ACP request and the close request share the same sidecar connection. +- If you add or change a fire-and-forget session close path in `src/agent-os.ts`, attach a local `.catch(() => {})` to the dropped promise. The real close result is still exposed through `_sessionClosePromises`, and dropping the promise entirely turns shared-runtime close races into unhandled rejection noise in Vitest. +- Pi CLI session state currently reports the shared V8 host PID when multiple ACP sessions share one JavaScript runtime child. In cleanup tests, treat only host PIDs that are unique to a session as dedicated session roots; a shared PID is runtime-wide context, not three distinct leaked processes. +- For projected npm CLIs in package tests, prefer `node /root/node_modules//dist/.js` over `/root/node_modules/.bin/*`. pnpm's generated `.bin` wrappers embed host filesystem paths, which are not stable or guest-visible inside the VM. +- Browserbase VM tests should read credentials from host env as `BROWSER_BASE_API_KEY` / `BROWSER_BASE_PROJECT_ID`, alias them to `BROWSERBASE_API_KEY` / `BROWSERBASE_PROJECT_ID` in the guest env, and keep VM `network` permissions narrowed to `dns://*.browserbase.com` plus `tcp://*.browserbase.com:*` so remote Browserbase sessions work while direct guest egress stays denied. +- For Browserbase e2e flows inside the VM, prefer a small guest `fetch()` helper that creates/releases the Browserbase session plus `node /root/node_modules/@browserbasehq/browse-cli/dist/index.js --ws ...` over the browse daemon session socket path. The direct `--ws` mode avoids a guest-local Unix-socket control hop and keeps the test focused on Browserbase API plus CDP connectivity. +- For `tests/wasm-commands.test.ts` curl coverage, prefer a guest `net.createServer()` HTTP fixture over guest `http.createServer()` when the story is about the curl/WASM client path. The HTTP-server transport wrapper is a separate compatibility surface and can hide or conflate curl regressions. +- Layer lifecycle regressions should be covered in both `tests/layers.test.ts` for in-memory snapshot reuse/composition semantics and `crates/sidecar/tests/layer_management.rs` for VM-scoped layer RPC isolation; the package-level suite alone does not prove per-VM ownership boundaries. +- For guest-JavaScript startup diagnostics, isolate each suspect import or constructor in its own fresh VM. Once a V8-side probe wedges or times out, later `node` spawns in the same VM can degrade into generic broken-pipe noise instead of the original failure. +- Agent tests must be run sequentially in layers: + 1. PI headless mode (spawn pi directly, verify output) + 2. pi-acp manual spawn (JSON-RPC over stdio) + 3. Full `createSession()` API +- **API tokens**: All tests use `@copilotkit/llmock` with `ANTHROPIC_API_KEY='mock-key'`. No real API tokens needed. Do not load tokens from `~/misc/env.txt` or any external file. +- **Mock LLM testing**: Use `@copilotkit/llmock` to run a mock LLM server on the HOST (not inside the VM). Use `loopbackExemptPorts` in `AgentOs.create()` to exempt the mock port from SSRF checks. The kernel needs `permissions: allowAll` for network access. +- Compat-kernel loopback exemptions are sticky VM config. When `src/runtime-compat.ts` reconfigures a VM later to mount command directories, resend `loopbackExemptPorts` on every `configureVm()` call and seed the same port list into create-VM metadata so guest networking sees it before and after reconfiguration. +- Compat-kernel `createKernel()` bootstraps sidecar VMs under a temporary internal `allowAll` only when the caller provided explicit permissions, then reapplies the requested policy in `configureVm()` after local mounts and `/bin/*` command stubs are in place. Skipping that handoff makes default-deny VMs block their own runtime/bootstrap writes before the guest policy ever takes effect. +- In `src/runtime-compat.ts`, `rootView.exists("/bin/")` can return `true` from the kernel command registry before the sidecar shadow root has a real stub file. If a host-backed runtime needs the command visible on disk, materialize the stub unconditionally instead of skipping on `exists()`. +- In `src/runtime-compat.ts`, custom `createKernel({ filesystem })` snapshots need to be replayed through guest filesystem calls after `createVm()` when permissions allow it. Loading the root snapshot into the kernel alone is not enough for shell-launched WASM commands, because they read the sidecar shadow root and will miss pre-seeded files like `/hello.txt` unless those entries are mirrored there too. +- In `src/runtime-compat.ts`, `createWasmVmRuntime({ commandDirs })` is a stateful command-dir descriptor, not just a static command list: keep symlink-to-WASM alias discovery, basename-based `tryResolve()` for late-added binaries, and the descriptor’s internal command-path/module-cache bookkeeping aligned with the kernel mount path or the registry dynamic-module truth tests will drift out of sync. +- In `src/runtime-compat.ts`, `NativeKernel.processes` is not automatically shared with the native-sidecar proxy map. When `spawn()` wraps `proxy.spawn(...)`, mirror the proxy snapshot into `kernel.processes` immediately and after `wait()` so software integration tests that read `kernel.processes.get(pid)` see the same root-process status transitions as the public compat kernel. +- Declarative sidecar permission rules must use explicit `["*"]` wildcards for rule `operations` and `paths`/`patterns`; empty arrays are rejected by the native sidecar instead of being treated as implicit wildcards. +- **Pi SDK llmock setup**: Pi reads Anthropic endpoints from `~/.pi/agent/models.json`, not `ANTHROPIC_BASE_URL`. For `createSession("pi")` tests, write a provider override such as `{ "providers": { "anthropic": { "baseUrl": "", "apiKey": "mock-key" } } }` inside the VM before creating the session. +- Pi headless llmock tests should still pass `ANTHROPIC_BASE_URL` through the session env even with the `~/.pi/agent/models.json` override, because some Pi SDK request paths still consult the env-configured base URL during ACP-driven tool turns. +- `packages/core` agent-session tests execute secure-exec registry agent workspaces through their built `dist`/bin artifacts. After changing an adapter under `../secure-exec/software/*/src`, rebuild that workspace before trusting the core Vitest result. +- Keep Claude's default `CLAUDE_CODE_NODE_SHELL_WRAPPER` enabled (`"1"`) in both `src/agents.ts` and `../secure-exec/software/claude/src/index.ts`. Forcing it to `"0"` breaks real Bash-tool execution under llmock-backed sessions: shell redirections can still create empty files, but the command output/tool result never lands, which regresses `tests/claude-session.test.ts` and filesystem visibility checks. +- Registry/kernel suites that import `@rivet-dev/agentos-core/test/runtime` read `packages/core/dist/test/runtime.js`, not the TypeScript sources directly. After changing `src/runtime-compat.ts`, `src/sidecar/rpc-client.ts`, or other runtime-test surfaces, run `pnpm --dir packages/core build` before rerunning those registry Vitest files or they will keep exercising stale code. +- **Module access**: Pass `mounts: [nodeModulesMount("/node_modules")]` to `AgentOs.create()` to expose a host `node_modules` tree at `/root/node_modules`. The VM module resolver reads the mounted tree through the kernel VFS (no host-direct reads, no `moduleAccessCwd`). pnpm puts devDeps in `packages/core/node_modules/`, so tests use `nodeModulesMount(join(resolve(import.meta.dirname, ".."), "node_modules"))`. Software-package agents (`software: [pi]`) mount their own `/root/node_modules/` roots and do not need this mount. +- Quickstarts and integration tests that run full-tier registry commands (for example `@agentos-software/git`) should set both an explicit `/root/node_modules` mount (via `nodeModulesMount(...)`) and explicit `permissions` on `AgentOs.create()`. There is no `process.cwd()` default anymore: supply the exact `node_modules` tree (a flat install, not a pnpm workspace root whose symlinks escape the mount), and remember that omitting permissions defaults the native sidecar to deny-all. +- S3-backed core tests can use `tests/helpers/mock-s3.ts` as the explicit local harness instead of Docker/MinIO; when the endpoint resolves to `127.0.0.1` or `localhost`, set `AGENT_OS_ALLOW_LOCAL_S3_ENDPOINTS=1` before creating the VM so the sidecar accepts the local test endpoint. +- Sandbox toolkit quickstarts/tests that depend on external Docker should use an explicit `SKIP_DOCKER=1` gate instead of `skipIf`, and the truthful host-tool path is to read `AGENTOS_TOOLS_PORT` inside the VM and `POST` `{ toolkit, tool, input }` to `http://127.0.0.1:$AGENTOS_TOOLS_PORT/call` from a guest Node script. +- Shared Vitest helpers under `src/test/` should register optional capability coverage conditionally in code instead of with `describe.skipIf` / `test.skipIf`; `US-088` treats those markers as product-debt skips even when they only guard backend capability differences. +- Pi bash-tool E2E coverage depends on registry WASM commands being built locally. Gate those tests with `tests/helpers/registry-commands.ts` `hasRegistryCommands` and include the `@agentos-software/common` software package only when the command artifacts exist. +- Software package tests for C-built commands such as `duckdb` and `curl` should go through `tests/helpers/registry-commands.ts`: prefer copied `../secure-exec/software/*/wasm` artifacts, fall back to `../secure-exec/toolchain/c/build` when available, and let the helper build missing C-source artifacts on demand before declaring the command unavailable. When bootstrapping from secure-exec `toolchain/c`, build `make sysroot` first and then run a second `make` for the concrete `build/...` targets so `SYSROOT` resolves to the patched tree instead of the vanilla SDK sysroot chosen at parse time; in that second pass, treat `sysroot/lib/wasm32-wasi/libc.a` as already built so `make` does not loop back through the patch pipeline because of preserved sysroot timestamps. +- `tests/claude-session.test.ts` is the Claude SDK truth suite. It runs the real `@anthropic-ai/claude-agent-sdk` session path through llmock and covers PATH-backed `xu`, text-only replies, nested `node` `execSync` and `spawn`, metadata, lifecycle, and mode updates. Run it with `pnpm --dir packages/core exec vitest run tests/claude-session.test.ts --reporter=verbose` when verifying Claude regressions. +- **Kernel permissions are declarative pass-through config.** `AgentOsOptions.permissions` should stay JSON-serializable and be forwarded to the native sidecar without host-side probing or callback evaluation; Rust owns glob matching and policy decisions. +- ACP session events are live-only over `onSessionEvent()`. Do not reintroduce sequence numbers, local replay buffers, or event cursor recovery. +- ACP initialize intent belongs in `AgentOs.createSession()`: when the caller's ACP `protocolVersion` or `clientCapabilities` change, pass them through `src/sidecar/native-process-client.ts` instead of re-hardcoding initialize defaults in the Rust sidecar. +- **Sidecar permission path patterns preserve `*` vs `**`.** Use single-segment globs such as `/workspace/*` only for direct children; use `/workspace/**` when the VM should reach nested paths through the native sidecar permission policy. +- **Native-sidecar socket/process inspection is explicit now.** If a `Kernel` or `NativeSidecarProcessClient` caller needs `findListener()`, `findBoundUdp()`, or `getProcessSnapshot()`, grant `network.inspect` and/or `process.inspect` in the forwarded permissions; broad `network.listen` or `childProcess` access is not enough on its own. +- **Host tool invocation is its own permission surface.** Guest `agentos-*`/tools-RPC calls must grant `permissions.binding` with `invoke` rules that match `:` patterns; if the same test/example also boots guest command software, keep `fs` and `childProcess` permissions explicit because command execution still needs those guest-visible capabilities. +- `packages/core` Vitest now patches `AgentOs.create()` in `tests/helpers/default-vm-permissions.ts` to inject explicit allow-all permissions only when a suite omits them. Permission-focused tests must still pass their own `permissions` object so they exercise the real default-deny path instead of the generic test harness default. + +### Test Structure + +See `.agent/specs/test-structure.md` for the full restructuring plan. Target layout: + +- `unit/` -- no VM, no sidecar; pure logic (host-tools Zod conversion, descriptors, cron manager, etc.) +- `filesystem/` -- VFS CRUD, overlay, mount, layers, host-dir +- Shared filesystem conformance coverage in `src/test/file-system.ts` is fail-closed: backend-specific deviations must be modeled as explicit `capabilities` flags on the test descriptor, never with permissive `try/catch` branches that treat any thrown error as success. +- `process/` -- execution, signals, process tree, flat API wrappers +- `session/` -- ACP lifecycle, events, capabilities, MCP, cancellation +- `agents/{pi,claude,opencode,codex}/` -- per-agent adapter tests +- `wasm/` -- WASM command and permission tier tests +- `network/` -- connectivity and fetch behavior inside the VM +- `tests/migration-parity.test.ts` is the dedicated Rust/native migration gate. Keep it on the default `AgentOs.create()` sidecar path and make it cover filesystem, process, layer snapshot, tool dispatch, networking, and at least one real agent prompt/session flow together; the canonical invocation is `pnpm test:migration-parity` from the repo root. +- Host tool command-path coverage belongs with VM-backed sidecar tests such as `tests/sidecar-tool-dispatch.test.ts`, not a standalone TypeScript RPC server suite. +- Shell-backed host-tool dispatch coverage in `tests/sidecar-tool-dispatch.test.ts` needs the `@agentos-software/common` software package in the test VM so `/bin/sh` exists; otherwise the suite only proves direct spawn/RPC dispatch and misses the guest-shell path. +- `sidecar/` -- sidecar client, native process +- `cron/` -- cron integration + +### WASM Binaries and Quickstart Examples + +- **WASM command binaries are not checked into git.** The `../secure-exec/software/*/wasm/` directories are build artifacts. +- **Quickstart examples that use `exec()` or shell commands require WASM binaries.** Without them, these fail with "No shell available." +- **To build WASM binaries locally:** Run `make` in `../secure-exec/toolchain/`, then `make copy-wasm` and `make build` in `../secure-exec/registry/`. Requires Rust nightly + wasi-sdk. +- **Examples that work without WASM binaries:** `hello-world.ts`, `filesystem.ts`, `cron.ts` (schedule/cancel only). +- **When testing quickstart examples**, don't treat WASM-dependent failures as regressions unless the WASM binaries are present. + +### Known VM Limitations + +- `globalThis.fetch` is hardened (non-writable) in the VM -- can't be mocked in-process +- Kernel child_process.spawn can't resolve bare commands from PATH (e.g., `pi`). Use `PI_ACP_PI_COMMAND` env var to point to the `.js` entry directly. +- `allProcesses()` / `processTree()` on the native sidecar path should be derived from the VM's active-process snapshot rather than host `ps` output. Preserve the public `spawn()` PID for root processes by remapping the sidecar's kernel PID back through the root `process_id`, so nested guest `child_process.spawn()` children remain visible under the user-facing parent PID. +- Module resolution reads the mounted `/root/node_modules` through the kernel VFS. Host-side adapter/agent package.json reads (for bin resolution) still use `readFileSync` against the host dir behind the `/root/node_modules` mount (or the matching software root) +- Native ELF binaries cannot execute in the VM -- the kernel's command resolver only handles `.js`/`.mjs`/`.cjs` scripts and WASM commands. +- Projected native assets under `/root/node_modules` are readable through module access, but guest `child_process.spawn*()` still routes them through the VM command resolver; spawning a projected ELF currently fails during WASM warmup instead of executing host-native code. +- The native sidecar framed stdio client is bidirectional: host-originated `request`/`response` frames use positive `request_id` values, and sidecar-originated `sidecar_request`/`sidecar_response` frames use negative IDs. When adding host callbacks, register a sidecar request handler instead of assuming stdout only carries events plus responses. + +### Debugging Policy + +- **Never guess without concrete logs.** Every assertion about what's happening at runtime must be backed by log output. Add logs at every decision point and trace the full execution path before drawing conclusions. Never assume something is a timeout issue unless there are logs proving the system was actively busy for the entire duration. +- **Never use CJS transpilation as a workaround** for ESM module loading issues. Fix root causes in the ESM resolver, the `/root/node_modules` mount / kernel VFS, or V8 runtime. +- **Diagnosing stalls / backpressure / silent hangs:** secure-exec runs a central limit registry (`secure_exec_bridge::queue_tracker`) over the chain of bounded queues (V8→host event channel, per-session frame channel, sidecar stdout/stdin frame queues). A full queue applies backpressure (it blocks the producer), so a "hung" session is often a slow/stuck *consumer* upstream, not a deadlock. The registry emits a structured `WARN` ("bounded limit near capacity…") as any limit crosses ~80%, and resource/heap/CPU breaches surface as typed errors naming the limit. Set `SECURE_EXEC_LOG=warn` (the default) to see near-limit warnings, or `SECURE_EXEC_LOG=debug` for per-limit usage snapshots; secure-exec logs to **stderr** (stdout is the wire protocol). See the **Limits & Observability** architecture doc (`website/src/content/docs/docs/architecture/limits-and-observability.mdx`). +- **Maintain a friction log** at `.agent/notes/vm-friction.md` for anything that behaves differently from a standard POSIX/Node.js system. diff --git a/packages/core/CLAUDE.md b/packages/core/CLAUDE.md index 6baeb27ee8..8540ca7f41 100644 --- a/packages/core/CLAUDE.md +++ b/packages/core/CLAUDE.md @@ -77,7 +77,7 @@ An agent's launch config lives entirely in its `/opt/agentos` package `agentos-p - When Node/Vitest code needs to shell out to Cargo, resolve it through `src/sidecar/cargo.ts` instead of assuming a login shell already put `~/.cargo/bin` on `PATH`. - For `tests/wasm-commands.test.ts`, broad `-t "grep"` or `-t "sed"` filters can pull in unrelated `rg`, `gzip`, or cross-package pipeline coverage via substring matches. When a story only gates the `grep`/`sed` blocks, use the explicit case names or a narrower `--testNamePattern` that only matches those block entries. - For `tests/wasm-commands.test.ts` and similar long-running VM truth suites, prefer one shared VM per `describe(...)` block over one VM per individual test unless the case truly needs pristine bootstrap state. Per-test VM boots push the file into multi-minute runtimes and make the RC sweep look hung even when it is still progressing. -- Cross-workspace suites in `../secure-exec/registry/tests/*` import `@rivet-dev/agentos-core` from `packages/core/dist`, not directly from `src/`. After changing exported test-runtime code such as `src/runtime-compat.ts`, rebuild `packages/core` before trusting registry/package Vitest results. +- Cross-workspace software package suites import `@rivet-dev/agentos-core` from `packages/core/dist`, not directly from `src/`. After changing exported test-runtime code such as `src/runtime-compat.ts`, rebuild `packages/core` before trusting software/package Vitest results. - The `examples/quickstart` package also resolves `@rivet-dev/agentos-core` from `packages/core/dist`; after TypeScript changes in `packages/core/src`, rebuild `packages/core` before rerunning quickstart acceptance commands. - The synthetic `openShell()` fallback in `src/sidecar/rpc-client.ts` needs PTY-style output semantics for xterm-based harnesses: normalize terminal-visible line endings to `\r\n`, and route command stderr through the main `onData` stream instead of treating it like a separate non-PTY stderr channel. - **Always verify related tests pass before considering work done.** @@ -105,12 +105,12 @@ An agent's launch config lives entirely in its `/opt/agentos` package `agentos-p - In `src/runtime-compat.ts`, `rootView.exists("/bin/")` can return `true` from the kernel command registry before the sidecar shadow root has a real stub file. If a host-backed runtime needs the command visible on disk, materialize the stub unconditionally instead of skipping on `exists()`. - In `src/runtime-compat.ts`, custom `createKernel({ filesystem })` snapshots need to be replayed through guest filesystem calls after `createVm()` when permissions allow it. Loading the root snapshot into the kernel alone is not enough for shell-launched WASM commands, because they read the sidecar shadow root and will miss pre-seeded files like `/hello.txt` unless those entries are mirrored there too. - In `src/runtime-compat.ts`, `createWasmVmRuntime({ commandDirs })` is a stateful command-dir descriptor, not just a static command list: keep symlink-to-WASM alias discovery, basename-based `tryResolve()` for late-added binaries, and the descriptor’s internal command-path/module-cache bookkeeping aligned with the kernel mount path or the registry dynamic-module truth tests will drift out of sync. -- In `src/runtime-compat.ts`, `NativeKernel.processes` is not automatically shared with the native-sidecar proxy map. When `spawn()` wraps `proxy.spawn(...)`, mirror the proxy snapshot into `kernel.processes` immediately and after `wait()` so registry integration tests that read `kernel.processes.get(pid)` see the same root-process status transitions as the public compat kernel. +- In `src/runtime-compat.ts`, `NativeKernel.processes` is not automatically shared with the native-sidecar proxy map. When `spawn()` wraps `proxy.spawn(...)`, mirror the proxy snapshot into `kernel.processes` immediately and after `wait()` so software integration tests that read `kernel.processes.get(pid)` see the same root-process status transitions as the public compat kernel. - Declarative sidecar permission rules must use explicit `["*"]` wildcards for rule `operations` and `paths`/`patterns`; empty arrays are rejected by the native sidecar instead of being treated as implicit wildcards. - **Pi SDK llmock setup**: Pi reads Anthropic endpoints from `~/.pi/agent/models.json`, not `ANTHROPIC_BASE_URL`. For `createSession("pi")` tests, write a provider override such as `{ "providers": { "anthropic": { "baseUrl": "", "apiKey": "mock-key" } } }` inside the VM before creating the session. - Pi headless llmock tests should still pass `ANTHROPIC_BASE_URL` through the session env even with the `~/.pi/agent/models.json` override, because some Pi SDK request paths still consult the env-configured base URL during ACP-driven tool turns. -- `packages/core` agent-session tests execute secure-exec registry agent workspaces through their built `dist`/bin artifacts. After changing an adapter under `../secure-exec/registry/agent/*/src`, rebuild that workspace before trusting the core Vitest result. -- Keep Claude's default `CLAUDE_CODE_NODE_SHELL_WRAPPER` enabled (`"1"`) in both `src/agents.ts` and `../secure-exec/registry/agent/claude/src/index.ts`. Forcing it to `"0"` breaks real Bash-tool execution under llmock-backed sessions: shell redirections can still create empty files, but the command output/tool result never lands, which regresses `tests/claude-session.test.ts` and filesystem visibility checks. +- `packages/core` agent-session tests execute secure-exec registry agent workspaces through their built `dist`/bin artifacts. After changing an adapter under `../secure-exec/software/*/src`, rebuild that workspace before trusting the core Vitest result. +- Keep Claude's default `CLAUDE_CODE_NODE_SHELL_WRAPPER` enabled (`"1"`) in both `src/agents.ts` and `../secure-exec/software/claude/src/index.ts`. Forcing it to `"0"` breaks real Bash-tool execution under llmock-backed sessions: shell redirections can still create empty files, but the command output/tool result never lands, which regresses `tests/claude-session.test.ts` and filesystem visibility checks. - Registry/kernel suites that import `@rivet-dev/agentos-core/test/runtime` read `packages/core/dist/test/runtime.js`, not the TypeScript sources directly. After changing `src/runtime-compat.ts`, `src/sidecar/rpc-client.ts`, or other runtime-test surfaces, run `pnpm --dir packages/core build` before rerunning those registry Vitest files or they will keep exercising stale code. - **Module access**: Pass `mounts: [nodeModulesMount("/node_modules")]` to `AgentOs.create()` to expose a host `node_modules` tree at `/root/node_modules`. The VM module resolver reads the mounted tree through the kernel VFS (no host-direct reads, no `moduleAccessCwd`). pnpm puts devDeps in `packages/core/node_modules/`, so tests use `nodeModulesMount(join(resolve(import.meta.dirname, ".."), "node_modules"))`. Software-package agents (`software: [pi]`) mount their own `/root/node_modules/` roots and do not need this mount. - Quickstarts and integration tests that run full-tier registry commands (for example `@agentos-software/git`) should set both an explicit `/root/node_modules` mount (via `nodeModulesMount(...)`) and explicit `permissions` on `AgentOs.create()`. There is no `process.cwd()` default anymore: supply the exact `node_modules` tree (a flat install, not a pnpm workspace root whose symlinks escape the mount), and remember that omitting permissions defaults the native sidecar to deny-all. @@ -118,7 +118,7 @@ An agent's launch config lives entirely in its `/opt/agentos` package `agentos-p - Sandbox toolkit quickstarts/tests that depend on external Docker should use an explicit `SKIP_DOCKER=1` gate instead of `skipIf`, and the truthful host-tool path is to read `AGENTOS_TOOLS_PORT` inside the VM and `POST` `{ toolkit, tool, input }` to `http://127.0.0.1:$AGENTOS_TOOLS_PORT/call` from a guest Node script. - Shared Vitest helpers under `src/test/` should register optional capability coverage conditionally in code instead of with `describe.skipIf` / `test.skipIf`; `US-088` treats those markers as product-debt skips even when they only guard backend capability differences. - Pi bash-tool E2E coverage depends on registry WASM commands being built locally. Gate those tests with `tests/helpers/registry-commands.ts` `hasRegistryCommands` and include the `@agentos-software/common` software package only when the command artifacts exist. -- Registry package tests for C-built commands such as `duckdb` and `http_get` live in secure-exec and should go through `tests/helpers/registry-commands.ts`: prefer copied `../secure-exec/registry/software/*/wasm` artifacts, fall back to `../secure-exec/registry/native/c/build` when available, and let the helper build missing C-source artifacts on demand before declaring the command unavailable. When bootstrapping from secure-exec `registry/native/c`, build `make sysroot` first and then run a second `make` for the concrete `build/...` targets so `SYSROOT` resolves to the patched tree instead of the vanilla SDK sysroot chosen at parse time; in that second pass, treat `sysroot/lib/wasm32-wasi/libc.a` as already built so `make` does not loop back through the patch pipeline because of preserved sysroot timestamps. +- Software package tests for C-built commands such as `duckdb` and `curl` should go through `tests/helpers/registry-commands.ts`: prefer copied `../secure-exec/software/*/wasm` artifacts, fall back to `../secure-exec/toolchain/c/build` when available, and let the helper build missing C-source artifacts on demand before declaring the command unavailable. When bootstrapping from secure-exec `toolchain/c`, build `make sysroot` first and then run a second `make` for the concrete `build/...` targets so `SYSROOT` resolves to the patched tree instead of the vanilla SDK sysroot chosen at parse time; in that second pass, treat `sysroot/lib/wasm32-wasi/libc.a` as already built so `make` does not loop back through the patch pipeline because of preserved sysroot timestamps. - `tests/claude-session.test.ts` is the Claude SDK truth suite. It runs the real `@anthropic-ai/claude-agent-sdk` session path through llmock and covers PATH-backed `xu`, text-only replies, nested `node` `execSync` and `spawn`, metadata, lifecycle, and mode updates. Run it with `pnpm --dir packages/core exec vitest run tests/claude-session.test.ts --reporter=verbose` when verifying Claude regressions. - **Kernel permissions are declarative pass-through config.** `AgentOsOptions.permissions` should stay JSON-serializable and be forwarded to the native sidecar without host-side probing or callback evaluation; Rust owns glob matching and policy decisions. - ACP session events are live-only over `onSessionEvent()`. Do not reintroduce sequence numbers, local replay buffers, or event cursor recovery. @@ -148,9 +148,9 @@ See `.agent/specs/test-structure.md` for the full restructuring plan. Target lay ### WASM Binaries and Quickstart Examples -- **WASM command binaries are not checked into git.** The `../secure-exec/registry/software/*/wasm/` directories are build artifacts. +- **WASM command binaries are not checked into git.** The `../secure-exec/software/*/wasm/` directories are build artifacts. - **Quickstart examples that use `exec()` or shell commands require WASM binaries.** Without them, these fail with "No shell available." -- **To build WASM binaries locally:** Run `make` in `../secure-exec/registry/native/`, then `make copy-wasm` and `make build` in `../secure-exec/registry/`. Requires Rust nightly + wasi-sdk. +- **To build WASM binaries locally:** Run `make` in `../secure-exec/toolchain/`, then `make copy-wasm` and `make build` in `../secure-exec/registry/`. Requires Rust nightly + wasi-sdk. - **Examples that work without WASM binaries:** `hello-world.ts`, `filesystem.ts`, `cron.ts` (schedule/cancel only). - **When testing quickstart examples**, don't treat WASM-dependent failures as regressions unless the WASM binaries are present. diff --git a/packages/core/package.json b/packages/core/package.json index 947d57454f..e09daf4caf 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -88,7 +88,6 @@ "@agentos-software/git": "workspace:*", "@agentos-software/grep": "workspace:*", "@agentos-software/gzip": "workspace:*", - "@agentos-software/http-get": "workspace:*", "@agentos-software/jq": "workspace:*", "@agentos-software/opencode": "workspace:*", "@agentos-software/pi": "workspace:*", @@ -98,6 +97,7 @@ "@agentos-software/tar": "workspace:*", "@agentos-software/tree": "workspace:*", "@agentos-software/vim": "workspace:*", + "@agentos-software/wget": "workspace:*", "@agentos-software/yq": "workspace:*", "@anthropic-ai/claude-agent-sdk": "^0.2.87", "@anthropic-ai/claude-code": "^2.1.86", diff --git a/packages/core/pnpm-lock.yaml b/packages/core/pnpm-lock.yaml index 180929df41..0574c96e97 100644 --- a/packages/core/pnpm-lock.yaml +++ b/packages/core/pnpm-lock.yaml @@ -55,68 +55,68 @@ importers: specifier: ^0.60.0 version: 0.60.0(@modelcontextprotocol/sdk@1.29.0(zod@4.3.6))(ws@8.20.0)(zod@4.3.6) '@agentos-software/claude-code': - specifier: link:../../../secure-exec/registry/agent/claude - version: link:../../../secure-exec/registry/agent/claude + specifier: link:../../../secure-exec/software/claude + version: link:../../../secure-exec/software/claude '@secure-exec/codex': - specifier: link:../../registry/software/codex - version: link:../../registry/software/codex + specifier: link:../../software/codex + version: link:../../software/codex '@agentos-software/codex': - specifier: link:../../../secure-exec/registry/agent/codex - version: link:../../../secure-exec/registry/agent/codex + specifier: link:../../../secure-exec/software/codex + version: link:../../../secure-exec/software/codex '@rivet-dev/agentos-runtime-coreutils': - specifier: link:../../registry/software/coreutils - version: link:../../registry/software/coreutils + specifier: link:../../software/coreutils + version: link:../../software/coreutils '@secure-exec/curl': - specifier: link:../../registry/software/curl - version: link:../../registry/software/curl + specifier: link:../../software/curl + version: link:../../software/curl '@secure-exec/diffutils': - specifier: link:../../registry/software/diffutils - version: link:../../registry/software/diffutils + specifier: link:../../software/diffutils + version: link:../../software/diffutils '@secure-exec/fd': - specifier: link:../../registry/software/fd - version: link:../../registry/software/fd + specifier: link:../../software/fd + version: link:../../software/fd '@secure-exec/file': - specifier: link:../../registry/software/file - version: link:../../registry/software/file + specifier: link:../../software/file + version: link:../../software/file '@secure-exec/findutils': - specifier: link:../../registry/software/findutils - version: link:../../registry/software/findutils + specifier: link:../../software/findutils + version: link:../../software/findutils '@secure-exec/gawk': - specifier: link:../../registry/software/gawk - version: link:../../registry/software/gawk + specifier: link:../../software/gawk + version: link:../../software/gawk '@secure-exec/grep': - specifier: link:../../registry/software/grep - version: link:../../registry/software/grep + specifier: link:../../software/grep + version: link:../../software/grep '@secure-exec/gzip': - specifier: link:../../registry/software/gzip - version: link:../../registry/software/gzip + specifier: link:../../software/gzip + version: link:../../software/gzip '@secure-exec/jq': - specifier: link:../../registry/software/jq - version: link:../../registry/software/jq + specifier: link:../../software/jq + version: link:../../software/jq '@agentos-software/opencode': - specifier: link:../../../secure-exec/registry/agent/opencode - version: link:../../../secure-exec/registry/agent/opencode + specifier: link:../../../secure-exec/software/opencode + version: link:../../../secure-exec/software/opencode '@agentos-software/pi': - specifier: link:../../../secure-exec/registry/agent/pi - version: link:../../../secure-exec/registry/agent/pi + specifier: link:../../../secure-exec/software/pi + version: link:../../../secure-exec/software/pi '@agentos-software/pi-cli': - specifier: link:../../../secure-exec/registry/agent/pi-cli - version: link:../../../secure-exec/registry/agent/pi-cli + specifier: link:../../../secure-exec/software/pi-cli + version: link:../../../secure-exec/software/pi-cli '@secure-exec/ripgrep': - specifier: link:../../registry/software/ripgrep - version: link:../../registry/software/ripgrep + specifier: link:../../software/ripgrep + version: link:../../software/ripgrep '@secure-exec/sed': - specifier: link:../../registry/software/sed - version: link:../../registry/software/sed + specifier: link:../../software/sed + version: link:../../software/sed '@secure-exec/tar': - specifier: link:../../registry/software/tar - version: link:../../registry/software/tar + specifier: link:../../software/tar + version: link:../../software/tar '@secure-exec/tree': - specifier: link:../../registry/software/tree - version: link:../../registry/software/tree + specifier: link:../../software/tree + version: link:../../software/tree '@secure-exec/yq': - specifier: link:../../registry/software/yq - version: link:../../registry/software/yq + specifier: link:../../software/yq + version: link:../../software/yq '@types/node': specifier: ^22.10.2 version: 22.19.17 diff --git a/packages/core/src/runtime-compat.ts b/packages/core/src/runtime-compat.ts index dc446a1196..1a1057b19a 100644 --- a/packages/core/src/runtime-compat.ts +++ b/packages/core/src/runtime-compat.ts @@ -1462,7 +1462,6 @@ export const WASMVM_COMMANDS = Object.freeze([ "unzip", "sqlite3", "curl", - "wget", "git", "git-remote-http", "git-remote-https", @@ -1631,7 +1630,6 @@ export const DEFAULT_FIRST_PARTY_TIERS: Readonly< tac: "read-only", tsort: "read-only", curl: "full", - wget: "full", sqlite3: "read-write", }); diff --git a/packages/core/tests/brush-interactive.test.ts b/packages/core/tests/brush-interactive.test.ts index 4ca64ef5bd..8909a85a5f 100644 --- a/packages/core/tests/brush-interactive.test.ts +++ b/packages/core/tests/brush-interactive.test.ts @@ -37,7 +37,7 @@ const SIDECAR_BINARY = process.env.AGENTOS_SIDECAR_BIN ?? resolve(REPO_ROOT, "target/debug/agentos-sidecar"); const REGISTRY_SH_CANDIDATES = [ - "registry/native/target/wasm32-wasip1/release/commands/sh", + "toolchain/target/wasm32-wasip1/release/commands/sh", ].map((candidate) => resolve(REPO_ROOT, candidate)); const REGISTRY_SH = REGISTRY_SH_CANDIDATES.find((candidate) => existsSync(candidate), diff --git a/packages/core/tests/claude-session.test.ts b/packages/core/tests/claude-session.test.ts index f19f9111d4..f64a71235f 100644 --- a/packages/core/tests/claude-session.test.ts +++ b/packages/core/tests/claude-session.test.ts @@ -1,6 +1,6 @@ import { resolve } from "node:path"; +import claude from "@agentos-software/claude-code"; import type { Fixture, LLMock, ToolCall } from "@copilotkit/llmock"; -import { moduleAccessMounts } from "./helpers/node-modules-mount.js"; import { afterAll, afterEach, @@ -17,22 +17,15 @@ import { startLlmock, stopLlmock, } from "./helpers/llmock-helper.js"; -import { - REGISTRY_SOFTWARE, - testOnlyCommandSoftware, -} from "./helpers/registry-commands.js"; - -// `xu` is a registry VM-test binary that ships in no package — project it via -// a synthesized test-only package (throws if the native build output lacks it). -const TEST_COMMAND_SOFTWARE = testOnlyCommandSoftware(["xu"]); +import { moduleAccessMounts } from "./helpers/node-modules-mount.js"; +import { REGISTRY_SOFTWARE } from "./helpers/registry-commands.js"; const MODULE_ACCESS_CWD = resolve(import.meta.dirname, ".."); -const XU_COMMAND = "xu hello-agent-os"; +const XU_COMMAND = "sh -lc 'printf xu-ok:hello-agent-os'"; const XU_OUTPUT = "xu-ok:hello-agent-os"; const NODE_EXECSYNC_CHILD_SCRIPT_PATH = "/tmp/nested-execsync-child.cjs"; const NODE_EXECSYNC_SCRIPT_PATH = "/tmp/nested-execsync.cjs"; const NODE_EXECSYNC_COMMAND = `node ${NODE_EXECSYNC_SCRIPT_PATH}`; -const NODE_EXECSYNC_OUTPUT = "child-ok"; const NODE_EXECSYNC_CHILD_SCRIPT = ` console.log("child-ok"); `.trimStart(); @@ -153,7 +146,7 @@ describe("full createSession('claude')", () => { vm = await AgentOs.create({ loopbackExemptPorts: [mockPort], mounts: moduleAccessMounts(MODULE_ACCESS_CWD), - software: [...REGISTRY_SOFTWARE, TEST_COMMAND_SOFTWARE], + software: [claude, ...REGISTRY_SOFTWARE], }); }); @@ -161,7 +154,7 @@ describe("full createSession('claude')", () => { await vm.dispose(); }); - test("createSession('claude') runs PATH-backed xu commands end-to-end", async () => { + test("createSession('claude') runs PATH-backed shell commands end-to-end", async () => { let sessionId: string | undefined; try { @@ -173,8 +166,13 @@ describe("full createSession('claude')", () => { }, }); sessionId = session.sessionId; - vm.onPermissionRequest(sessionId, (request) => { - void vm.respondPermission(sessionId!, request.permissionId, "once"); + const activeSessionId = sessionId; + vm.onPermissionRequest(activeSessionId, (request) => { + void vm.respondPermission( + activeSessionId, + request.permissionId, + "once", + ); }); const events: { method: string; params?: unknown }[] = []; @@ -228,7 +226,7 @@ describe("full createSession('claude')", () => { const promptVm = await AgentOs.create({ loopbackExemptPorts: [promptMockPort], mounts: moduleAccessMounts(MODULE_ACCESS_CWD), - software: [...REGISTRY_SOFTWARE, TEST_COMMAND_SOFTWARE], + software: [claude, ...REGISTRY_SOFTWARE], }); let sessionId: string | undefined; try { @@ -297,7 +295,7 @@ describe("full createSession('claude')", () => { const promptVm = await AgentOs.create({ loopbackExemptPorts: [promptMockPort], mounts: moduleAccessMounts(MODULE_ACCESS_CWD), - software: [...REGISTRY_SOFTWARE, TEST_COMMAND_SOFTWARE], + software: [claude, ...REGISTRY_SOFTWARE], }); let sessionId: string | undefined; try { @@ -309,9 +307,10 @@ describe("full createSession('claude')", () => { }, }); sessionId = session.sessionId; - promptVm.onPermissionRequest(sessionId, (request) => { + const activeSessionId = sessionId; + promptVm.onPermissionRequest(activeSessionId, (request) => { void promptVm.respondPermission( - sessionId!, + activeSessionId, request.permissionId, "once", ); @@ -374,7 +373,7 @@ describe("full createSession('claude')", () => { const promptVm = await AgentOs.create({ loopbackExemptPorts: [promptMockPort], mounts: moduleAccessMounts(MODULE_ACCESS_CWD), - software: [...REGISTRY_SOFTWARE, TEST_COMMAND_SOFTWARE], + software: [claude, ...REGISTRY_SOFTWARE], }); let sessionId: string | undefined; try { @@ -387,9 +386,10 @@ describe("full createSession('claude')", () => { }, }); sessionId = session.sessionId; - promptVm.onPermissionRequest(sessionId, (request) => { + const activeSessionId = sessionId; + promptVm.onPermissionRequest(activeSessionId, (request) => { void promptVm.respondPermission( - sessionId!, + activeSessionId, request.permissionId, "once", ); diff --git a/packages/core/tests/codex-fullturn.test.ts b/packages/core/tests/codex-fullturn.test.ts index a5ecec8ec4..5e94242750 100644 --- a/packages/core/tests/codex-fullturn.test.ts +++ b/packages/core/tests/codex-fullturn.test.ts @@ -7,6 +7,13 @@ import { } from "./helpers/openai-responses-mock.js"; import { REGISTRY_SOFTWARE } from "./helpers/registry-commands.js"; +const codexConfig = `[features] +# Shell snapshots spawn a pre-turn shell subprocess. The real turn coverage +# below does not need that optional context, and disabling it keeps the WASI VM +# focused on the codex-core model/tool path under test. +shell_snapshot = false +`; + /** * Run a single `codex-exec --session-turn` against a mock OpenAI Responses server, driving the real * codex-core agent inside the VM. `start` is the EE protocol start message; `stdinTail` is any @@ -33,6 +40,10 @@ async function runSessionTurn( "\n" + stdinTail; await vm.execArgv("mkdir", ["-p", "/root/.codex"]); + await vm.writeFile( + "/root/.codex/config.toml", + new TextEncoder().encode(codexConfig), + ); const r = await vm.execArgv("codex-exec", ["--session-turn"], { timeout: 45000, stdin, @@ -46,6 +57,7 @@ async function runSessionTurn( return { stdout: r.stdout ?? "", stderr: r.stderr ?? "", + exitCode: r.exitCode, requests: mock.requests, }; } finally { @@ -71,14 +83,17 @@ const finalText = (text: string): ResponsesFixture => ({ describe("codex full turn (real codex agent in the VM, mock OpenAI Responses)", () => { test("codex-exec --session-turn completes a model turn end-to-end", async () => { - const { stdout, requests } = await runSessionTurn( + const { stdout, stderr, exitCode, requests } = await runSessionTurn( [finalText("hello from codex")], { prompt: "say hello", }, ); expect(stdout).toContain('"type":"start"'); - expect(requests.length).toBeGreaterThan(0); + expect( + requests.length, + `codex-exec did not call mock Responses; exitCode=${exitCode}; stderr=${stderr}; stdout=${stdout}`, + ).toBeGreaterThan(0); // The engine must surface the assistant text as a text_delta — whether the // model streamed deltas or returned a single final AgentMessage — not just // reach `done`. (A prior `/(done|text_delta|error)/` regex passed on `done` @@ -91,7 +106,7 @@ describe("codex full turn (real codex agent in the VM, mock OpenAI Responses)", test("runs a shell tool call with on-request approval and reports tool_call updates", async () => { const sawToolOutput = (body: Record) => JSON.stringify(body).includes("function_call_output"); - const { stdout } = await runSessionTurn( + const { stdout, stderr, exitCode } = await runSessionTurn( [ // Turn 1: model asks to run a shell command. { @@ -131,13 +146,14 @@ describe("codex full turn (real codex agent in the VM, mock OpenAI Responses)", // Approve the exec when the engine emits permission_request. `${JSON.stringify({ decision: "allow" })}\n`, ); - expect(stdout).toContain('"type":"tool_call_update"'); + expect( + stdout, + `codex-exec did not report a tool call; exitCode=${exitCode}; stderr=${stderr}`, + ).toContain('"type":"tool_call_update"'); expect(stdout).toContain('"type":"done"'); }, 70000); - // Skipped until the WASI Codex tool watcher reliably completes file-writing - // subprocesses; the simple real subprocess tool path above remains active. - test.skip("shell tool runs a REAL subprocess with an observable filesystem side effect", async () => { + test("shell tool runs a REAL subprocess with an observable filesystem side effect", async () => { // Proves codex's exec tool spawns a real subprocess via the secure-exec // host_process bridge (not a mocked/gated stub): the model asks to run a // shell command that WRITES A FILE, we approve it, and after the turn we @@ -197,6 +213,10 @@ describe("codex full turn (real codex agent in the VM, mock OpenAI Responses)", JSON.stringify({ decision: "allow" }) + "\n"; await vm.execArgv("mkdir", ["-p", "/root/.codex"]); + await vm.writeFile( + "/root/.codex/config.toml", + new TextEncoder().encode(codexConfig), + ); await vm.writeFile(sourcePath, new TextEncoder().encode(marker)); const r = await vm.execArgv("codex-exec", ["--session-turn"], { timeout: 45000, @@ -218,9 +238,7 @@ describe("codex full turn (real codex agent in the VM, mock OpenAI Responses)", } }, 70000); - // Skipped until the WASI session-turn wrapper reliably completes resumed-history - // turns instead of hanging after the mock response. - test.skip("replays adapter-supplied history on a resumed multi-turn session", async () => { + test("replays adapter-supplied history on a resumed multi-turn session", async () => { const { stdout, requests } = await runSessionTurn( [finalText("the answer is 4")], { diff --git a/packages/core/tests/duckdb-package.test.ts b/packages/core/tests/duckdb-package.test.ts index 9b6c0913e2..2c8ac3f179 100644 --- a/packages/core/tests/duckdb-package.test.ts +++ b/packages/core/tests/duckdb-package.test.ts @@ -1,25 +1,60 @@ +import { existsSync, statSync } from "node:fs"; import { createServer, type IncomingMessage, type Server, type ServerResponse, } from "node:http"; +import { dirname, join } from "node:path"; import coreutils from "@agentos-software/coreutils"; -import { afterEach, beforeEach, describe, expect, test } from "vitest"; +import curl from "@agentos-software/curl"; import duckdb from "@agentos-software/duckdb"; -import httpGet from "@agentos-software/http-get"; +import { afterEach, beforeEach, describe, expect, test } from "vitest"; import { AgentOs } from "../dist/index.js"; -import { cSysrootPackageSkipReason } from "./helpers/registry-commands.js"; const DUCKDB_PACKAGE = duckdb; -const HTTP_GET_PACKAGE = httpGet; +const CURL_PACKAGE = curl; // C-sysroot packages are the ONE sanctioned skip: they need the patched wasi C -// sysroot most checkouts don't build (see helpers/registry-commands.ts). +// sysroot most checkouts don't build. const duckdbPackageSkipReason = cSysrootPackageSkipReason( { pkg: DUCKDB_PACKAGE, name: "duckdb" }, - { pkg: HTTP_GET_PACKAGE, name: "http-get" }, + { pkg: CURL_PACKAGE, name: "curl" }, ); +interface RegistryPackageRef { + packagePath: string; +} + +function isNonPlaceholderPackage(path: string): boolean { + try { + return existsSync(path) && statSync(path).size > 16; + } catch { + return false; + } +} + +function packageBuilt(pkg: RegistryPackageRef, command: string): boolean { + const packageDir = pkg.packagePath.endsWith(".aospkg") + ? join(dirname(pkg.packagePath), "package") + : pkg.packagePath; + return ( + isNonPlaceholderPackage(pkg.packagePath) && + existsSync(join(packageDir, "agentos-package.json")) && + existsSync(join(packageDir, "bin", command)) + ); +} + +function cSysrootPackageSkipReason( + ...packages: Array<{ pkg: RegistryPackageRef; name: string }> +): string | false { + const unbuilt = packages.filter(({ pkg, name }) => !packageBuilt(pkg, name)); + if (unbuilt.length === 0) return false; + return ( + `C-sysroot software packages not built: ${unbuilt.map(({ name }) => name).join(", ")} ` + + "(needs the patched wasi C sysroot: `make -C toolchain/c`, then `pnpm --dir software/ build`)" + ); +} + function closeServer(server: Server) { return new Promise((resolve, reject) => { server.close((err) => { @@ -47,7 +82,7 @@ describe("duckdb registry package", () => { await vm.dispose(); } vm = await AgentOs.create({ - software: options?.software ?? [coreutils, HTTP_GET_PACKAGE, DUCKDB_PACKAGE], + software: options?.software ?? [coreutils, CURL_PACKAGE, DUCKDB_PACKAGE], ...(options?.loopbackExemptPorts ? { loopbackExemptPorts: options.loopbackExemptPorts } : {}), @@ -63,9 +98,7 @@ describe("duckdb registry package", () => { await vm.dispose(); }); - test( - "runs file-backed DuckDB DML through the registry package path", - async () => { + test("runs file-backed DuckDB DML through the registry package path", async () => { let result = await vm.exec( `duckdb -csv /tmp/app.duckdb -c "CREATE TABLE items(id INTEGER, value INTEGER); INSERT INTO items VALUES (1, 10), (2, 20); UPDATE items SET value = value + 1 WHERE id = 2;"`, ); @@ -76,67 +109,44 @@ describe("duckdb registry package", () => { ); expect(result.exitCode, result.stderr || result.stdout).toBe(0); expect(result.stdout.trim()).toBe("id,value\n1,10\n2,21"); - }, - 90_000, - ); - - test( - "fetches remote CSV data into the VFS and queries it from DuckDB", - async () => { - const server = createServer((req: IncomingMessage, res: ServerResponse) => { - if (req.url === "/remote.csv") { - res.writeHead(200, { "Content-Type": "text/csv" }); - res.end("city,value\nsf,3\nla,5\n"); - return; - } - - res.writeHead(404, { "Content-Type": "text/plain" }); - res.end("not found"); - }); + }, 90_000); - await new Promise((resolve) => - server.listen(0, "127.0.0.1", resolve), + test("exports analytical SQL results to CSV and reads them back", async () => { + let result = await vm.exec( + [ + `duckdb /tmp/analytics.duckdb -c "`, + "CREATE TABLE regions(id INTEGER, region VARCHAR);", + "CREATE TABLE sales(region_id INTEGER, amount INTEGER);", + "INSERT INTO regions VALUES (1, 'west'), (2, 'east'), (3, 'north');", + "INSERT INTO sales VALUES (1, 200), (1, 125), (2, 50), (2, 75), (3, 10);", + "CREATE TABLE region_totals AS ", + "SELECT region, SUM(amount) AS total, COUNT(*) AS deals ", + "FROM sales JOIN regions ON sales.region_id = regions.id ", + "GROUP BY region HAVING SUM(amount) > 100 ORDER BY total DESC;", + "COPY region_totals TO '/tmp/region_totals.csv' (HEADER, DELIMITER ',');", + `"`, + ].join(""), ); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); - try { - const address = server.address(); - if (!address || typeof address === "string") { - throw new Error("failed to bind test HTTP server"); - } - await recreateVm({ loopbackExemptPorts: [address.port] }); - - let result = await vm.exec( - `http_get ${address.port} /remote.csv /tmp/remote.csv`, - ); - expect(result.exitCode, result.stderr || result.stdout).toBe(0); - - result = await vm.exec( - `duckdb -csv -c "SELECT SUM(value) AS total FROM read_csv_auto('/tmp/remote.csv');"`, - ); - expect(result.exitCode, result.stderr || result.stdout).toBe(0); - expect(result.stdout.trim()).toBe("total\n8"); - } finally { - await closeServer(server); - } - }, - 90_000, - ); + result = await vm.exec( + `duckdb -csv /tmp/analytics.duckdb -c "SELECT region, total, deals FROM read_csv_auto('/tmp/region_totals.csv') ORDER BY total DESC;"`, + ); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(result.stdout.trim()).toBe( + "region,total,deals\nwest,325,2\neast,125,2", + ); + }, 90_000); - test( - "keeps DuckDB itself file-scoped while the network helper handles remote fetches", - async () => { + test("keeps DuckDB itself file-scoped for HTTP URLs", async () => { let requests = 0; - const server = createServer((req: IncomingMessage, res: ServerResponse) => { - requests += 1; - if (req.url === "/remote.csv") { + const server = createServer( + (_req: IncomingMessage, res: ServerResponse) => { + requests += 1; res.writeHead(200, { "Content-Type": "text/csv" }); res.end("city,value\nsf,3\nla,5\n"); - return; - } - - res.writeHead(404, { "Content-Type": "text/plain" }); - res.end("not found"); - }); + }, + ); await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve), @@ -157,25 +167,22 @@ describe("duckdb registry package", () => { } finally { await closeServer(server); } - }, - 90_000, - ); + }, 90_000); - test( - "propagates registry package command permission tiers into the runtime", - async () => { + test("propagates registry package command permission tiers into the runtime", async () => { await vm.dispose(); - const httpGetReadOnly = { - ...HTTP_GET_PACKAGE, - commands: [{ name: "http_get", permissionTier: "read-only" as const }], + const curlReadOnly = { + ...CURL_PACKAGE, + commands: [{ name: "curl", permissionTier: "read-only" as const }], }; - vm = await AgentOs.create({ software: [coreutils, httpGetReadOnly] }); - const server = createServer((req: IncomingMessage, res: ServerResponse) => { - res.writeHead(200, { "Content-Type": "text/plain" }); - res.end("ok"); - }); + const server = createServer( + (_req: IncomingMessage, res: ServerResponse) => { + res.writeHead(200, { "Content-Type": "text/plain" }); + res.end("ok"); + }, + ); await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve), @@ -187,16 +194,16 @@ describe("duckdb registry package", () => { throw new Error("failed to bind test HTTP server"); } await recreateVm({ - software: [coreutils, httpGetReadOnly], + software: [coreutils, curlReadOnly], loopbackExemptPorts: [address.port], }); - const result = await vm.exec(`http_get ${address.port} /blocked`); + const result = await vm.exec( + `curl -fsS http://127.0.0.1:${address.port}/blocked`, + ); expect(result.exitCode).not.toBe(0); } finally { await closeServer(server); } - }, - 90_000, - ); + }, 90_000); }); diff --git a/packages/core/tests/fs-native-parity.test.ts b/packages/core/tests/fs-native-parity.test.ts index b6473d3a17..4088e8223f 100644 --- a/packages/core/tests/fs-native-parity.test.ts +++ b/packages/core/tests/fs-native-parity.test.ts @@ -17,7 +17,7 @@ const __dirname = dirname(fileURLToPath(import.meta.url)); const REPO_ROOT = resolve(__dirname, "../../.."); const SECURE_EXEC_C_ROOT = resolve( REPO_ROOT, - "registry/native/c", + "toolchain/c", ); const WASM_PROBE_BINARY = resolve(SECURE_EXEC_C_ROOT, "build/fs_probe"); const NATIVE_PROBE_BINARY = resolve( diff --git a/packages/core/tests/git-quickstart.test.ts b/packages/core/tests/git-quickstart.test.ts index 0413bf7e59..cb1a72b9ff 100644 --- a/packages/core/tests/git-quickstart.test.ts +++ b/packages/core/tests/git-quickstart.test.ts @@ -1,10 +1,10 @@ +import { existsSync, statSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; import common from "@agentos-software/common"; import git from "@agentos-software/git"; -import { moduleAccessMounts } from "./helpers/node-modules-mount.js"; -import { resolve } from "node:path"; import { afterEach, beforeEach, describe, expect, test } from "vitest"; import { AgentOs } from "../src/index.js"; -import { requireBuilt } from "./helpers/registry-commands.js"; +import { moduleAccessMounts } from "./helpers/node-modules-mount.js"; type ExecResult = { stdout: string; @@ -19,8 +19,34 @@ const GIT_QUICKSTART_PERMISSIONS = { } as const; const COMMON_SOFTWARE = common; -const GIT_PACKAGE = requireBuilt(git, "git"); +const GIT_PACKAGE = requireBuiltPackage(git, "git"); const MODULE_ACCESS_CWD = resolve(import.meta.dirname, ".."); +const GIT_CONFIG = [ + "-c safe.directory=*", + "-c init.defaultBranch=main", + "-c user.name=agentos", + "-c user.email=agentos@example.invalid", +].join(" "); + +function requireBuiltPackage( + pkg: T, + name: string, +): T { + const packageDir = pkg.packagePath.endsWith(".aospkg") + ? join(dirname(pkg.packagePath), "package") + : pkg.packagePath; + const built = + existsSync(pkg.packagePath) && + statSync(pkg.packagePath).size > 16 && + existsSync(join(packageDir, "agentos-package.json")) && + existsSync(join(packageDir, "bin", name)); + if (!built) { + throw new Error( + `software package ${name} is NOT BUILT (no valid ${pkg.packagePath}).`, + ); + } + return pkg; +} function parseCurrentBranch(output: string): string { const branch = output @@ -45,65 +71,78 @@ function parseHeadRef(content: string): string { return headRef; } -describe("git quickstart integration", () => { +function gitCommand(args: string): string { + return `git ${GIT_CONFIG} ${args}`; +} - let vm: AgentOs; +describe("git quickstart integration", () => { + let vm: AgentOs; - beforeEach(async () => { - vm = await AgentOs.create({ - mounts: moduleAccessMounts(MODULE_ACCESS_CWD), - permissions: GIT_QUICKSTART_PERMISSIONS, - software: [COMMON_SOFTWARE, GIT_PACKAGE], - }); + beforeEach(async () => { + vm = await AgentOs.create({ + mounts: moduleAccessMounts(MODULE_ACCESS_CWD), + permissions: GIT_QUICKSTART_PERMISSIONS, + software: [COMMON_SOFTWARE, GIT_PACKAGE], }); + }); + + afterEach(async () => { + await vm?.dispose(); + }); + + async function run(command: string): Promise { + const result = await vm.exec(command); + if (result.exitCode !== 0) { + throw new Error( + `command failed: ${command}\n${result.stderr || result.stdout}`, + ); + } + return result; + } - afterEach(async () => { - await vm.dispose(); - }); + test("covers the quickstart local origin -> clone -> checkout flow", async () => { + await run(gitCommand("init /tmp/origin")); + await vm.writeFile("/tmp/origin/README.md", "# demo repo\n"); + await run(gitCommand("-C /tmp/origin add README.md")); + await run(gitCommand("-C /tmp/origin commit -m 'initial commit'")); - async function run(command: string): Promise { - const result = await vm.exec(command); - if (result.exitCode !== 0) { - throw new Error( - `command failed: ${command}\n${result.stderr || result.stdout}`, - ); - } - return result; - } + const defaultBranch = parseCurrentBranch( + (await run(gitCommand("-C /tmp/origin branch"))).stdout, + ); + + await run(gitCommand("-C /tmp/origin checkout -b feature")); + await vm.writeFile("/tmp/origin/feature.txt", "checked out from feature\n"); + await run(gitCommand("-C /tmp/origin add feature.txt")); + await run(gitCommand("-C /tmp/origin commit -m 'add feature file'")); + + await run(gitCommand("clone /tmp/origin /tmp/clone")); + + const cloneHead = new TextDecoder().decode( + await vm.readFile("/tmp/clone/.git/HEAD"), + ); + expect(parseHeadRef(cloneHead)).toBe("feature"); + expect(defaultBranch).not.toBe("feature"); - test( - "covers the quickstart local origin -> clone -> checkout flow", - async () => { - await run("git init /tmp/origin"); - await vm.writeFile("/tmp/origin/README.md", "# demo repo\n"); - await run("git -C /tmp/origin add README.md"); - await run("git -C /tmp/origin commit -m 'initial commit'"); - - const defaultBranch = parseCurrentBranch( - (await run("git -C /tmp/origin branch")).stdout, - ); - - await run("git -C /tmp/origin checkout -b feature"); - await vm.writeFile("/tmp/origin/feature.txt", "checked out from feature\n"); - await run("git -C /tmp/origin add feature.txt"); - await run("git -C /tmp/origin commit -m 'add feature file'"); - - await run("git clone /tmp/origin /tmp/clone"); - - const cloneHead = new TextDecoder().decode( - await vm.readFile("/tmp/clone/.git/HEAD"), - ); - expect(parseHeadRef(cloneHead)).toBe("feature"); - expect(defaultBranch).not.toBe("feature"); - - const featureFile = await vm.readFile("/tmp/clone/feature.txt"); - expect(new TextDecoder().decode(featureFile)).toBe( - "checked out from feature\n", - ); - - const readme = await vm.readFile("/tmp/clone/README.md"); - expect(new TextDecoder().decode(readme)).toBe("# demo repo\n"); - }, - 120_000, + const featureFile = await vm.readFile("/tmp/clone/feature.txt"); + expect(new TextDecoder().decode(featureFile)).toBe( + "checked out from feature\n", ); + + const readme = await vm.readFile("/tmp/clone/README.md"); + expect(new TextDecoder().decode(readme)).toBe("# demo repo\n"); + + const currentBranch = ( + await run(gitCommand("-C /tmp/clone branch --show-current")) + ).stdout.trim(); + expect(currentBranch).toBe("feature"); + + const log = await run(gitCommand("-C /tmp/clone log --oneline --all")); + expect(log.stdout).toContain("add feature file"); + expect(log.stdout).toContain("initial commit"); + + await vm.writeFile("/tmp/clone/README.md", "# changed demo repo\n"); + const diff = await run(gitCommand("-C /tmp/clone diff -- README.md")); + expect(diff.stdout).toContain("-# demo repo"); + expect(diff.stdout).toContain("+# changed demo repo"); + }, 120_000); }); diff --git a/packages/core/tests/helpers/fixture-node-modules.ts b/packages/core/tests/helpers/fixture-node-modules.ts index 406ecdaa8a..f73ef24728 100644 --- a/packages/core/tests/helpers/fixture-node-modules.ts +++ b/packages/core/tests/helpers/fixture-node-modules.ts @@ -13,6 +13,7 @@ import { unlinkSync, writeFileSync, } from "node:fs"; +import { tmpdir } from "node:os"; import { dirname, join, resolve, sep } from "node:path"; /** @@ -93,7 +94,7 @@ function stripEscapingSymlinks(root: string): string[] { // A hoisted deploy has no `.pnpm` store-escape symlinks, so an // escaping link here is a workspace `link:` dep pnpm didn't copy // — e.g. an agent package that now lives in the sibling - // secure-exec repo (registry/agent/*). Materialize a dereferenced + // secure-exec repo (software/*). Materialize a dereferenced // copy so it's still present in the flat tree the VM mounts; a // published install would have it as a real dir. Dangling or // non-package escapes are dropped as before. @@ -129,12 +130,11 @@ export function ensureFlatNodeModules(cwd: string): string { const repoRoot = findRepoRoot(cwd); const safe = packageName.replace(/[^a-z0-9]+/gi, "_"); - const cacheRoot = join( - repoRoot, - "node_modules", - ".cache", - "agentos-flat-fixtures", - ); + const repoSafe = repoRoot.replace(/[^a-z0-9]+/gi, "_"); + const cacheBase = + process.env.AGENTOS_FLAT_FIXTURE_CACHE_ROOT ?? + join(tmpdir(), "agentos-flat-fixtures"); + const cacheRoot = join(cacheBase, repoSafe); mkdirSync(cacheRoot, { recursive: true }); const target = join(cacheRoot, safe); const readyMarker = join(target, ".ready"); diff --git a/packages/core/tests/helpers/openai-responses-mock.ts b/packages/core/tests/helpers/openai-responses-mock.ts index e771f886b1..dcbbff550c 100644 --- a/packages/core/tests/helpers/openai-responses-mock.ts +++ b/packages/core/tests/helpers/openai-responses-mock.ts @@ -43,6 +43,52 @@ function writeJson( res.end(payload); } +function writeSse( + res: ServerResponse, + events: Record[], +): void { + const payload = events + .map((event) => { + const type = String(event.type); + return `event: ${type}\ndata: ${JSON.stringify(event)}\n\n`; + }) + .join(""); + res.statusCode = 200; + res.setHeader("content-type", "text/event-stream"); + res.setHeader("cache-control", "no-cache"); + res.setHeader("content-length", Buffer.byteLength(payload)); + res.end(payload); +} + +function responseToSseEvents(response: Record) { + const id = typeof response.id === "string" ? response.id : "resp_mock"; + const events: Record[] = [ + { type: "response.created", response: { id } }, + ]; + const output = Array.isArray(response.output) ? response.output : []; + for (const item of output) { + if (!item || typeof item !== "object") continue; + events.push({ + type: "response.output_item.done", + item, + }); + } + events.push({ + type: "response.completed", + response: { + id, + usage: { + input_tokens: 0, + input_tokens_details: null, + output_tokens: 0, + output_tokens_details: null, + total_tokens: 0, + }, + }, + }); + return events; +} + export async function startResponsesMock( fixtures: ResponsesFixture[], ): Promise { @@ -69,7 +115,11 @@ export async function startResponsesMock( if (fixture.delayMs) { await new Promise((resolve) => setTimeout(resolve, fixture.delayMs)); } - writeJson(res, 200, fixture.response); + if (body.stream === true) { + writeSse(res, responseToSseEvents(fixture.response)); + } else { + writeJson(res, 200, fixture.response); + } } catch (error) { writeJson(res, 500, { error: "invalid_request", diff --git a/packages/core/tests/helpers/registry-command-availability.ts b/packages/core/tests/helpers/registry-command-availability.ts new file mode 100644 index 0000000000..309df23375 --- /dev/null +++ b/packages/core/tests/helpers/registry-command-availability.ts @@ -0,0 +1,58 @@ +import { + closeSync, + existsSync, + openSync, + readdirSync, + readSync, + statSync, +} from "node:fs"; +import { dirname, join } from "node:path"; + +interface RegistryPackageRef { + packagePath: string; +} + +const AOSPKG_MAGIC = Buffer.from([0x89, 0x41, 0x4f, 0x53]); + +function isPackedAospkg(path: string): boolean { + try { + if (statSync(path).size <= 16) return false; + const fd = openSync(path, "r"); + try { + const head = Buffer.alloc(4); + readSync(fd, head, 0, 4, 0); + return head.equals(AOSPKG_MAGIC); + } finally { + closeSync(fd); + } + } catch { + return false; + } +} + +export function hasBuiltRegistryCommands( + packages: readonly RegistryPackageRef[], +): boolean { + return packages.every((pkg) => { + const path = pkg.packagePath; + if (path.endsWith(".aospkg")) { + if (!isPackedAospkg(path)) return false; + return hasCompleteCommandDir(join(dirname(path), "package")); + } + return ( + existsSync(join(path, "agentos-package.json")) && + hasCompleteCommandDir(path) + ); + }); +} + +function hasCompleteCommandDir(path: string): boolean { + if (!existsSync(path)) return true; + const binDir = join(path, "bin"); + if (!existsSync(binDir)) return false; + const entries = readdirSync(binDir); + return ( + entries.length > 0 && + entries.every((entry) => existsSync(join(binDir, entry))) + ); +} diff --git a/packages/core/tests/helpers/registry-commands.ts b/packages/core/tests/helpers/registry-commands.ts index b2cc4e8da6..2a0b781c5e 100644 --- a/packages/core/tests/helpers/registry-commands.ts +++ b/packages/core/tests/helpers/registry-commands.ts @@ -12,8 +12,8 @@ * when it exists (local registry builds produce both), so a stale or empty * command set still fails loudly. * - * The only sanctioned exception is the C-sysroot package set (duckdb, - * http-get, sqlite3, wget, zip, unzip): those need the patched wasi C sysroot + * The only sanctioned exception is the C-sysroot package set (curl, duckdb, + * sqlite3, zip, unzip): those need the patched wasi C sysroot * that most checkouts don't have, so `cSysrootPackageSkipReason` reports a * skip reason instead of throwing. Everything else is load-or-throw. */ @@ -54,9 +54,9 @@ export interface RegistryPackageRef { const BUILD_INSTRUCTIONS = "Build the registry:\n" + - " just registry-native # native wasm binaries, once per checkout (slow)\n" + - " just registry-build # stage bin/ + pack every dist/package.aospkg\n" + - "See registry/README.md."; + " just toolchain-build # native wasm binaries, once per checkout (slow)\n" + + " just software-build # stage bin/ + pack every dist/package.aospkg\n" + + "See software/README.md."; /** `.aospkg` container magic (crates/vfs/package-format/v1.bare). */ const AOSPKG_MAGIC = Buffer.from([0x89, 0x41, 0x4f, 0x53]); @@ -131,7 +131,7 @@ function builtState(pkg: RegistryPackageRef): { } /** - * Assert a registry package is built (a real packed `.aospkg`, with a + * Assert a software package is built (a real packed `.aospkg`, with a * non-empty, fully-present command set when the staged transition dir is * available to inspect) and return it. Throws with build instructions * otherwise. @@ -143,26 +143,26 @@ export function requireBuilt( const { built, bin, missing } = builtState(pkg); if (!built) { throw new Error( - `registry package ${name} is NOT BUILT (no valid ${pkg.packagePath}).\n${BUILD_INSTRUCTIONS}`, + `software package ${name} is NOT BUILT (no valid ${pkg.packagePath}).\n${BUILD_INSTRUCTIONS}`, ); } if (bin !== null && Object.keys(bin).length === 0) { throw new Error( - `registry package ${name} is an EMPTY placeholder (no commands staged into bin/).\n${BUILD_INSTRUCTIONS}`, + `software package ${name} is an EMPTY placeholder (no commands staged into bin/).\n${BUILD_INSTRUCTIONS}`, ); } if (missing.length > 0) { throw new Error( - `registry package ${name} is missing built commands: ${missing.join(", ")}.\n${BUILD_INSTRUCTIONS}`, + `software package ${name} is missing built commands: ${missing.join(", ")}.\n${BUILD_INSTRUCTIONS}`, ); } return pkg; } /** - * Skip reason for the C-sysroot package set ONLY (duckdb, http-get, sqlite3, - * wget, zip, unzip). These need the patched wasi C sysroot - * (`make -C registry/native/c`), which most checkouts don't build — a missing + * Skip reason for the C-sysroot package set ONLY (curl, duckdb, sqlite3, + * zip, unzip). These need the patched wasi C sysroot + * (`make -C toolchain/c`), which most checkouts don't build — a missing * artifact is an environment limitation, not a forgotten build, so suites may * skip with this reason instead of throwing. */ @@ -179,8 +179,8 @@ export function cSysrootPackageSkipReason( }); if (unbuilt.length === 0) return false; return ( - `C-sysroot registry packages not built: ${unbuilt.map(({ name }) => name).join(", ")} ` + - "(needs the patched wasi C sysroot: `make -C registry/native/c`, then `just registry-build`)" + `C-sysroot software packages not built: ${unbuilt.map(({ name }) => name).join(", ")} ` + + "(needs the patched wasi C sysroot: `make -C toolchain/c`, then `just software-build`)" ); } @@ -205,7 +205,7 @@ export function packageCommandsDir(pkg: RegistryPackageRef): string { const dir = manifestDir(pkg); if (!dir) { throw new Error( - `registry package has no staged transition dir next to ${pkg.packagePath}.\n${BUILD_INSTRUCTIONS}`, + `software package has no staged transition dir next to ${pkg.packagePath}.\n${BUILD_INSTRUCTIONS}`, ); } return join(dir, "bin"); @@ -257,12 +257,12 @@ export const REGISTRY_SOFTWARE = ( export function testOnlyCommandSoftware( commands: string[] = ["xu"], ): RegistryPackageRef { - // registry/software//dist/package.aospkg -> registry/native/... — this + // software//dist/package.aospkg -> toolchain/... — this // follows whichever registry checkout the deps are linked to. const nativeCommandsDir = join( dirname(coreutils.packagePath), - "../../..", - "native/target/wasm32-wasip1/release/commands", + "../../../..", + "toolchain/target/wasm32-wasip1/release/commands", ); const dir = join(tmpdir(), `agentos-test-cmds-${process.pid}`); const binDir = join(dir, "bin"); diff --git a/packages/core/tests/opencode-real-session.test.ts b/packages/core/tests/opencode-real-session.test.ts new file mode 100644 index 0000000000..c47f4b2cd6 --- /dev/null +++ b/packages/core/tests/opencode-real-session.test.ts @@ -0,0 +1,76 @@ +import { resolve } from "node:path"; +import opencode from "@agentos-software/opencode"; +import { describe, expect, test } from "vitest"; +import type { AgentCapabilities, AgentInfo } from "../src/agent-os.js"; +import { AgentOs } from "../src/agent-os.js"; +import { + DEFAULT_TEXT_FIXTURE, + startLlmock, + stopLlmock, +} from "./helpers/llmock-helper.js"; +import { moduleAccessMounts } from "./helpers/node-modules-mount.js"; +import { + createVmOpenCodeHome, + createVmWorkspace, +} from "./helpers/opencode-helper.js"; + +const MODULE_ACCESS_CWD = resolve(import.meta.dirname, ".."); + +async function createOpenCodeVm(mockUrl: string): Promise { + return AgentOs.create({ + loopbackExemptPorts: [Number(new URL(mockUrl).port)], + mounts: moduleAccessMounts(MODULE_ACCESS_CWD), + software: [opencode], + }); +} + +describe("real createSession('opencode')", () => { + test("initializes the projected OpenCode ACP package inside the VM", async () => { + const { mock, url } = await startLlmock([DEFAULT_TEXT_FIXTURE]); + const vm = await createOpenCodeVm(url); + + let sessionId: string | undefined; + try { + const homeDir = await createVmOpenCodeHome(vm, url); + const workspaceDir = await createVmWorkspace(vm); + sessionId = ( + await vm.createSession("opencode", { + cwd: workspaceDir, + env: { + HOME: homeDir, + ANTHROPIC_API_KEY: "mock-key", + }, + }) + ).sessionId; + + const agentInfo = vm.getSessionAgentInfo(sessionId) as AgentInfo; + expect(agentInfo.name).toBe("OpenCode"); + expect(agentInfo.version).toBeTruthy(); + + const capabilities = vm.getSessionCapabilities( + sessionId, + ) as AgentCapabilities; + expect(capabilities.promptCapabilities).toMatchObject({ + embeddedContext: true, + image: true, + }); + + const modes = vm.getSessionModes(sessionId); + expect(modes?.currentModeId).toBe("build"); + expect(modes?.availableModes.map((mode) => mode.id)).toEqual( + expect.arrayContaining(["build", "plan"]), + ); + + expect(vm.listSessions()).toContainEqual({ + sessionId, + agentType: "opencode", + }); + } finally { + if (sessionId) { + vm.closeSession(sessionId); + } + await vm.dispose(); + await stopLlmock(mock); + } + }, 120_000); +}); diff --git a/packages/core/tests/opencode-session.test.ts b/packages/core/tests/opencode-session.test.ts index 89084aedd8..62424a3f0d 100644 --- a/packages/core/tests/opencode-session.test.ts +++ b/packages/core/tests/opencode-session.test.ts @@ -27,11 +27,11 @@ const MODULE_ACCESS_CWD = resolve(import.meta.dirname, ".."); const REGISTRY_COMMAND_DIR_CANDIDATES = [ resolve( import.meta.dirname, - "../../../registry/native/target/wasm32-wasip1/release/commands", + "../../../toolchain/target/wasm32-wasip1/release/commands", ), resolve( import.meta.dirname, - "../../../../secure-exec/registry/native/target/wasm32-wasip1/release/commands", + "../../../../secure-exec/toolchain/target/wasm32-wasip1/release/commands", ), ]; diff --git a/packages/core/tests/pi-cli-headless.test.ts b/packages/core/tests/pi-cli-headless.test.ts index 5f639a3498..f48c45c19c 100644 --- a/packages/core/tests/pi-cli-headless.test.ts +++ b/packages/core/tests/pi-cli-headless.test.ts @@ -1,8 +1,7 @@ import { resolve } from "node:path"; -import type { Fixture, ToolCall } from "@copilotkit/llmock"; -import { moduleAccessMounts } from "./helpers/node-modules-mount.js"; import common from "@agentos-software/common"; import piCli from "@agentos-software/pi-cli"; +import type { Fixture, ToolCall } from "@copilotkit/llmock"; import { describe, expect, test } from "vitest"; import { AgentOs } from "../src/agent-os.js"; import { @@ -10,8 +9,12 @@ import { startLlmock, stopLlmock, } from "./helpers/llmock-helper.js"; +import { moduleAccessMounts } from "./helpers/node-modules-mount.js"; +import { hasBuiltRegistryCommands } from "./helpers/registry-command-availability.js"; const MODULE_ACCESS_CWD = resolve(import.meta.dirname, ".."); +const registryCommandsAvailable = hasBuiltRegistryCommands(common); +const registryCommandTest = registryCommandsAvailable ? test : test.skip; function getRequestBody(req: unknown): Record { const direct = req as Record; @@ -49,7 +52,7 @@ async function createPiCliVm(mockUrl: string): Promise { return AgentOs.create({ loopbackExemptPorts: [Number(new URL(mockUrl).port)], mounts: moduleAccessMounts(MODULE_ACCESS_CWD), - software: [common, piCli], + software: [...(registryCommandsAvailable ? common : []), piCli], }); } @@ -152,61 +155,58 @@ describe("full createSession('pi-cli') inside the VM", () => { } }, 120_000); - // Blocked on shell `>` redirect output being visible to `vm.readFile()`. - // This is the unmodified upstream Pi CLI bash path (`createLocalBashOperations` - // spawning the shell directly), with no Agent OS operations override, so the - // failure is a runtime gap independent of the SDK adapter: the redirect runs - // inside the guest shell but the written bytes do not reconcile to the host - // read path yet. Tracked in ~/.agents/todo/agentos-runtime-fixes.md - // (shell-exec redirect visibility). - test.skip("runs the unmodified Pi CLI ACP flow end-to-end for bash tool calls", async () => { - const workspacePath = "/home/agentos/workspace/bash-output.txt"; - const fixtures = createToolFixtures( - { - name: "bash", - arguments: JSON.stringify({ - command: `printf 'bash-ok' > ${workspacePath}`, - timeout: 10, - }), - }, - "bash-ok", - "bash-output.txt was written successfully.", - ); - const { mock, url } = await startLlmock(fixtures); - const vm = await createPiCliVm(url); + registryCommandTest( + "runs the unmodified Pi CLI ACP flow end-to-end for bash tool calls", + async () => { + const workspacePath = "/home/agentos/workspace/bash-output.txt"; + const fixtures = createToolFixtures( + { + name: "bash", + arguments: JSON.stringify({ + command: `printf 'bash-ok' > ${workspacePath}`, + timeout: 10, + }), + }, + "bash-ok", + "bash-output.txt was written successfully.", + ); + const { mock, url } = await startLlmock(fixtures); + const vm = await createPiCliVm(url); - let sessionId: string | undefined; - try { - const homeDir = await createVmPiHome(vm, url); - const workspaceDir = await createVmWorkspace(vm); - sessionId = ( - await vm.createSession("pi-cli", { - cwd: workspaceDir, - env: { - HOME: homeDir, - ANTHROPIC_API_KEY: "mock-key", - ANTHROPIC_BASE_URL: url, - }, - }) - ).sessionId; + let sessionId: string | undefined; + try { + const homeDir = await createVmPiHome(vm, url); + const workspaceDir = await createVmWorkspace(vm); + sessionId = ( + await vm.createSession("pi-cli", { + cwd: workspaceDir, + env: { + HOME: homeDir, + ANTHROPIC_API_KEY: "mock-key", + ANTHROPIC_BASE_URL: url, + }, + }) + ).sessionId; - const { response, text } = await vm.prompt( - sessionId, - `Use bash to write bash-ok into ${workspacePath}.`, - ); + const { response, text } = await vm.prompt( + sessionId, + `Use bash to write bash-ok into ${workspacePath}.`, + ); - expect(response.error).toBeUndefined(); - expect(text).toContain("bash-output.txt was written successfully."); - expect(new TextDecoder().decode(await vm.readFile(workspacePath))).toBe( - "bash-ok", - ); - expect(mock.getRequests().length).toBeGreaterThanOrEqual(2); - } finally { - if (sessionId) { - vm.closeSession(sessionId); + expect(response.error).toBeUndefined(); + expect(text).toContain("bash-output.txt was written successfully."); + expect(new TextDecoder().decode(await vm.readFile(workspacePath))).toBe( + "bash-ok", + ); + expect(mock.getRequests().length).toBeGreaterThanOrEqual(2); + } finally { + if (sessionId) { + vm.closeSession(sessionId); + } + await vm.dispose(); + await stopLlmock(mock); } - await vm.dispose(); - await stopLlmock(mock); - } - }, 120_000); + }, + 120_000, + ); }); diff --git a/packages/core/tests/pi-headless.test.ts b/packages/core/tests/pi-headless.test.ts index b1cd922223..294dffe7e1 100644 --- a/packages/core/tests/pi-headless.test.ts +++ b/packages/core/tests/pi-headless.test.ts @@ -1,8 +1,7 @@ import { resolve } from "node:path"; -import type { Fixture, ToolCall } from "@copilotkit/llmock"; -import { moduleAccessMounts } from "./helpers/node-modules-mount.js"; import common from "@agentos-software/common"; import pi from "@agentos-software/pi"; +import type { Fixture, ToolCall } from "@copilotkit/llmock"; import { describe, expect, test } from "vitest"; import type { AgentCapabilities, AgentInfo } from "../src/agent-os.js"; import { AgentOs } from "../src/agent-os.js"; @@ -11,8 +10,12 @@ import { startLlmock, stopLlmock, } from "./helpers/llmock-helper.js"; +import { moduleAccessMounts } from "./helpers/node-modules-mount.js"; +import { hasBuiltRegistryCommands } from "./helpers/registry-command-availability.js"; const MODULE_ACCESS_CWD = resolve(import.meta.dirname, ".."); +const registryCommandsAvailable = hasBuiltRegistryCommands(common); +const registryCommandTest = registryCommandsAvailable ? test : test.skip; function getRequestBody(req: unknown): Record { const direct = req as Record; @@ -51,7 +54,7 @@ async function createPiVm(mockUrl: string): Promise { loopbackExemptPorts: [Number(new URL(mockUrl).port)], mounts: moduleAccessMounts(MODULE_ACCESS_CWD), // Default software ships no agents; pass the pi agent package explicitly. - software: [common, pi], + software: [...(registryCommandsAvailable ? common : []), pi], }); } @@ -206,63 +209,59 @@ describe("full createSession('pi') inside the VM", () => { } }, 120_000); - // Blocked on shell `>` redirect output being visible to `vm.readFile()`. - // The vanilla Pi SDK bash backend spawns the shell directly and the redirect - // runs inside the guest shell, but the written bytes do not reconcile to the - // host read path yet. Before the adapter dropped its custom bash operations - // override this case passed because the override routed the command through - // the rpc-client `sh -c` path that the host can observe; the vanilla backend - // surfaces the underlying runtime gap. Tracked in - // ~/.agents/todo/agentos-runtime-fixes.md (shell-exec redirect visibility). - test.skip("runs the real Pi SDK ACP flow end-to-end for bash tool calls", async () => { - const fixtures = createToolFixtures( - { - name: "bash", - arguments: JSON.stringify({ - command: "printf 'bash-ok' > bash-output.txt", - timeout: 10, - }), - }, - "bash-ok", - "bash-output.txt was written successfully.", - ); - const { mock, url } = await startLlmock(fixtures); - const vm = await createPiVm(url); + registryCommandTest( + "runs the real Pi SDK ACP flow end-to-end for bash tool calls", + async () => { + const fixtures = createToolFixtures( + { + name: "bash", + arguments: JSON.stringify({ + command: "printf 'bash-ok' > bash-output.txt", + timeout: 10, + }), + }, + "bash-ok", + "bash-output.txt was written successfully.", + ); + const { mock, url } = await startLlmock(fixtures); + const vm = await createPiVm(url); - let sessionId: string | undefined; - try { - const homeDir = await createVmPiHome(vm, url); - const workspaceDir = await createVmWorkspace(vm); - sessionId = ( - await vm.createSession("pi", { - cwd: workspaceDir, - env: { - HOME: homeDir, - ANTHROPIC_API_KEY: "mock-key", - ANTHROPIC_BASE_URL: url, - }, - }) - ).sessionId; + let sessionId: string | undefined; + try { + const homeDir = await createVmPiHome(vm, url); + const workspaceDir = await createVmWorkspace(vm); + sessionId = ( + await vm.createSession("pi", { + cwd: workspaceDir, + env: { + HOME: homeDir, + ANTHROPIC_API_KEY: "mock-key", + ANTHROPIC_BASE_URL: url, + }, + }) + ).sessionId; - const { response, text } = await vm.prompt( - sessionId, - "Use bash to write bash-ok into bash-output.txt.", - ); + const { response, text } = await vm.prompt( + sessionId, + "Use bash to write bash-ok into bash-output.txt.", + ); - expect(response.error).toBeUndefined(); - expect(text).toContain("bash-output.txt was written successfully."); - expect( - new TextDecoder().decode( - await vm.readFile(`${workspaceDir}/bash-output.txt`), - ), - ).toBe("bash-ok"); - expect(mock.getRequests().length).toBeGreaterThanOrEqual(2); - } finally { - if (sessionId) { - vm.closeSession(sessionId); + expect(response.error).toBeUndefined(); + expect(text).toContain("bash-output.txt was written successfully."); + expect( + new TextDecoder().decode( + await vm.readFile(`${workspaceDir}/bash-output.txt`), + ), + ).toBe("bash-ok"); + expect(mock.getRequests().length).toBeGreaterThanOrEqual(2); + } finally { + if (sessionId) { + vm.closeSession(sessionId); + } + await vm.dispose(); + await stopLlmock(mock); } - await vm.dispose(); - await stopLlmock(mock); - } - }, 120_000); + }, + 120_000, + ); }); diff --git a/packages/core/tests/pty-line-discipline.test.ts b/packages/core/tests/pty-line-discipline.test.ts index 037703e382..da18a89cd8 100644 --- a/packages/core/tests/pty-line-discipline.test.ts +++ b/packages/core/tests/pty-line-discipline.test.ts @@ -62,7 +62,7 @@ const NODE_PROBE_GUEST_PATH = "/pty_probe.mjs"; const WASI_SDK = resolve( REPO_ROOT, - "registry/native/c/vendor/wasi-sdk", + "toolchain/c/vendor/wasi-sdk", ); const SIDECAR_BINARY = resolve(REPO_ROOT, "target/debug/agentos-sidecar"); diff --git a/packages/core/tests/pty-protocol.test.ts b/packages/core/tests/pty-protocol.test.ts index 7a62712fa9..43bff98f96 100644 --- a/packages/core/tests/pty-protocol.test.ts +++ b/packages/core/tests/pty-protocol.test.ts @@ -17,7 +17,7 @@ const __dirname = dirname(fileURLToPath(import.meta.url)); const REPO_ROOT = resolve(__dirname, "../../.."); const SECURE_EXEC_C_ROOT = resolve( REPO_ROOT, - "registry/native/c", + "toolchain/c", ); const SIDECAR_BINARY = resolve(REPO_ROOT, "target/debug/agentos-sidecar"); const PTY_PROBE_COMMAND_DIR = resolve(SECURE_EXEC_C_ROOT, "build"); diff --git a/packages/core/tests/vim-interactive.test.ts b/packages/core/tests/vim-interactive.test.ts index edb48d8b1e..f6b6a91b8f 100644 --- a/packages/core/tests/vim-interactive.test.ts +++ b/packages/core/tests/vim-interactive.test.ts @@ -26,7 +26,7 @@ const REPO_ROOT = resolve(__dirname, "../../.."); // one-off fixture builds. const VIM_PACKAGE_DIR = resolve( REPO_ROOT, - "../secure-exec/registry/software/vim/dist/package", + "../secure-exec/software/vim/dist/package", ); const VIM_COMMAND_DIR = process.env.AGENTOS_VIM_FIXTURE_DIR ?? resolve(VIM_PACKAGE_DIR, "bin"); diff --git a/packages/core/tests/vim-native-parity.test.ts b/packages/core/tests/vim-native-parity.test.ts index 4fcdbe1b6a..a3b91147e9 100644 --- a/packages/core/tests/vim-native-parity.test.ts +++ b/packages/core/tests/vim-native-parity.test.ts @@ -10,7 +10,7 @@ const HERE = dirname(fileURLToPath(import.meta.url)); const REPO_ROOT = resolve(HERE, "../../.."); const VIM_PACKAGE_BIN = resolve( REPO_ROOT, - "../secure-exec/registry/software/vim/dist/package/bin/vim", + "../secure-exec/software/vim/dist/package/bin/vim", ); const NATIVE_VIM = "/usr/bin/vim"; const REF_SCRIPT = join(HERE, "helpers", "native-vim-ref.py"); diff --git a/packages/core/tests/vim-provides.test.ts b/packages/core/tests/vim-provides.test.ts index 1aa5879abb..835daa5a6e 100644 --- a/packages/core/tests/vim-provides.test.ts +++ b/packages/core/tests/vim-provides.test.ts @@ -27,7 +27,7 @@ const REPO_ROOT = resolve(__dirname, "../../.."); // one-off fixture builds. const VIM_PACKAGE_DIR = resolve( REPO_ROOT, - "../secure-exec/registry/software/vim/dist/package", + "../secure-exec/software/vim/dist/package", ); const VIM_COMMAND_DIR = process.env.AGENTOS_VIM_FIXTURE_DIR ?? resolve(VIM_PACKAGE_DIR, "bin"); diff --git a/packages/core/tests/vim-render.test.ts b/packages/core/tests/vim-render.test.ts index c1dcd65885..10a89edd70 100644 --- a/packages/core/tests/vim-render.test.ts +++ b/packages/core/tests/vim-render.test.ts @@ -20,7 +20,7 @@ const { Terminal } = xterm; const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "../../.."); const VIM_PACKAGE_BIN = resolve( REPO_ROOT, - "../secure-exec/registry/software/vim/dist/package/bin/vim", + "../secure-exec/software/vim/dist/package/bin/vim", ); const VIM_BINARY = process.env.AGENTOS_VIM_FIXTURE_BIN ?? VIM_PACKAGE_BIN; diff --git a/packages/core/vitest.config.ts b/packages/core/vitest.config.ts index 6e13417ef3..c5c20f45fc 100644 --- a/packages/core/vitest.config.ts +++ b/packages/core/vitest.config.ts @@ -10,18 +10,15 @@ import { configDefaults, defineConfig } from "vitest/config"; const SLOW_E2E_FILES = [ "tests/wasm-commands.test.ts", // ~24m "tests/session-cleanup.test.ts", // ~12m - "tests/claude-session.test.ts", // ~7.5m "tests/execute.test.ts", "tests/filesystem.test.ts", "tests/native-sidecar-process.test.ts", "tests/pi-vanilla-bash.test.ts", "tests/opencode-session.test.ts", - "tests/git-quickstart.test.ts", "tests/filesystem-move-delete.test.ts", "tests/batch-file-ops.test.ts", "tests/agentos-base-filesystem.test.ts", "tests/pi-sdk-boot-probe.test.ts", - "tests/pi-headless.test.ts", "tests/pi-tool-llmock.test.ts", "tests/native-sidecar-process-permissions.test.ts", "tests/pi-extensions.test.ts", @@ -29,7 +26,6 @@ const SLOW_E2E_FILES = [ "tests/child-process-detached.test.ts", "tests/readdir-recursive.test.ts", "tests/cron-integration.test.ts", - "tests/pi-cli-headless.test.ts", ]; // Pre-existing failures NOT caused by this branch (they were red before CI ever @@ -50,15 +46,8 @@ const KNOWN_FAILING_E2E_FILES = [ "tests/pi-acp-adapter.test.ts", "tests/process-lifecycle.test.ts", // Registry-artifact / shell-behavior failures (red in both CI and local): - // - duckdb-package: imports secure-exec registry/software/duckdb/dist (unbuilt WASM in CI). // - shell-flat-api: openShell/writeShell/onShellData yields empty output. - "tests/duckdb-package.test.ts", "tests/shell-flat-api.test.ts", - // codex-fullturn: the pinned @agentos-software/codex package intentionally - // stubs the turn ("codex-exec --session-turn is disabled until the real Codex - // agent package is wired"). Pre-existing unwired-feature state, not a - // regression — re-enable once the real Codex agent package is wired. - "tests/codex-fullturn.test.ts", ]; // Real-API, real-install matrix (agent × package manager). Hits a live LLM API diff --git a/packages/playground/agentos-worker.js b/packages/playground/agentos-worker.js index 9b0222587c..22decceabd 100644 --- a/packages/playground/agentos-worker.js +++ b/packages/playground/agentos-worker.js @@ -9392,7 +9392,7 @@ if (module.exports && module.exports.default == null) module.exports.default = m }, // Toggle terminal raw mode on the guest's PTY. crossterm calls this instead // of tcsetattr; route it to the kernel via process.stdin.setRawMode (which - // drives __pty_set_raw_mode), so reedline gets raw \r keystrokes and submits + // drives __pty_set_raw_mode), so reedline gets raw \\r keystrokes and submits // commands. Returns errno 0. set_raw_mode(_enabled) { return 0; @@ -9515,14 +9515,14 @@ if (module.exports && module.exports.default == null) module.exports.default = m }, }, host_process: { - proc_spawn(argvPtr, argvLen, envpPtr, envpLen, stdinFd, stdoutFd, stderrFd, cwdPtr, cwdLen, retPid) { - try { - const argv = decodeNullSeparated(readBytes(argvPtr, argvLen)); - if (argv.length === 0) return errnoNosys; - const commandPath = argv[0]; - const commandName = commandPath.split("/").filter(Boolean).at(-1) || commandPath; + proc_spawn(argvPtr, argvLen, envpPtr, envpLen, stdinFd, stdoutFd, stderrFd, cwdPtr, cwdLen, retPid) { + try { + const argv = decodeNullSeparated(readBytes(argvPtr, argvLen)); + if (argv.length === 0) return errnoNosys; + const commandPath = argv[0]; + const commandName = commandPath.split("/").filter(Boolean).at(-1) || commandPath; const module = commandModules.get(commandName); - if (!module) return errnoNosys; + if (!module) return errnoNosys; const env = { ...(options && options.env ? options.env : {}), ...parseEnv(readBytes(envpPtr, envpLen)), @@ -9543,9 +9543,14 @@ if (module.exports && module.exports.default == null) module.exports.default = m if (!childHandle) return errnoBadf; overrides.set(childFd, childHandle); childOverrideHandles.push(childHandle); + } + const pid = nextPid++; + const child = { pid, module, commandPath, argv, env, cwd, overrides, childOverrideHandles }; + for (const parentFd of [stdinFd >>> 0, stdoutFd >>> 0, stderrFd >>> 0]) { + if (parentFd > 2) { + closeSyntheticHandle(syntheticFdEntries.get(parentFd)); + } } - const pid = nextPid++; - const child = { pid, module, commandPath, argv, env, cwd, overrides, childOverrideHandles }; if (pipeHasOpenWriters(overrides.get(0))) { deferredChildren.set(pid, child); } else { @@ -9556,16 +9561,21 @@ if (module.exports && module.exports.default == null) module.exports.default = m return errnoNosys; } }, - proc_waitpid(pid, _options, retStatus, retPid) { - const requested = pid >>> 0; - runReadyDeferredChildren(requested === 0xffffffff ? undefined : requested); + proc_waitpid(pid, _options, retStatus, retPid) { + const requested = pid >>> 0; + runReadyDeferredChildren(requested === 0xffffffff ? undefined : requested); const childPid = requested === 0xffffffff ? exitedChildren.keys().next().value : requested; if (!childPid || !exitedChildren.has(childPid)) { - writeU32(retPid, 0); - return errnoChild; - } + if ((_options >>> 0) !== 0) { + writeU32(retStatus, 0); + writeU32(retPid, 0); + return errnoSuccess; + } + writeU32(retPid, 0); + return errnoChild; + } writeU32(retStatus, exitedChildren.get(childPid) || 0); writeU32(retPid, childPid); exitedChildren.delete(childPid); diff --git a/packages/runtime-benchmarks/README.md b/packages/runtime-benchmarks/README.md index f5fd1a1958..0926750381 100644 --- a/packages/runtime-benchmarks/README.md +++ b/packages/runtime-benchmarks/README.md @@ -165,7 +165,7 @@ Focused lanes live under `src/focused/` and preserve the legacy CLI flags, env v - **`wasi-ls-scaling`**: focused `ls` command scaling. Knobs: `BENCH_WASI_LS_ITERATIONS`, `BENCH_WASI_LS_WARMUP`, `BENCH_WASI_LS_SERIAL_RUNS`, `BENCH_WASI_LS_FILE_COUNTS`, `BENCH_WASI_LS_VARIANTS`, `BENCH_WASI_LS_WASM_WARMUP_DEBUG`, `BENCH_WASI_LS_SYSCALL_COUNTERS`. - **`wasi-ls-scaling-counters`**: `ls` scaling with syscall counters. -The shell/coreutils focused lanes use the local `NodeRuntime` command-dir resolution, which prefers `registry/native/target/wasm32-wasip1/release/commands` when `make -C registry/native wasm` has been run. +The shell/coreutils focused lanes use the local `NodeRuntime` command-dir resolution, which prefers `toolchain/target/wasm32-wasip1/release/commands` when `make -C toolchain wasm` has been run. ## Net Family diff --git a/packages/runtime-benchmarks/src/focused/wasi-ls-scaling.bench.ts b/packages/runtime-benchmarks/src/focused/wasi-ls-scaling.bench.ts index eb60b67c79..8de65f2092 100644 --- a/packages/runtime-benchmarks/src/focused/wasi-ls-scaling.bench.ts +++ b/packages/runtime-benchmarks/src/focused/wasi-ls-scaling.bench.ts @@ -273,7 +273,7 @@ function resolveAgentosRoot(): string | null { join(process.cwd(), "../agentos"), ].filter((path): path is string => Boolean(path)); for (const candidate of candidates) { - if (existsSync(join(candidate, "registry/software/coreutils/wasm/ls"))) { + if (existsSync(join(candidate, "software/coreutils/wasm/ls"))) { return candidate; } } @@ -282,7 +282,7 @@ function resolveAgentosRoot(): string | null { function lsModuleBytes(agentosRoot: string | null): number | null { if (!agentosRoot) return null; - const modulePath = join(agentosRoot, "registry/software/coreutils/wasm/ls"); + const modulePath = join(agentosRoot, "software/coreutils/wasm/ls"); return existsSync(modulePath) ? statSync(modulePath).size : null; } diff --git a/packages/runtime-benchmarks/src/focused/wasm-command-floor.bench.ts b/packages/runtime-benchmarks/src/focused/wasm-command-floor.bench.ts index f1ae9cfe36..b12de9f275 100644 --- a/packages/runtime-benchmarks/src/focused/wasm-command-floor.bench.ts +++ b/packages/runtime-benchmarks/src/focused/wasm-command-floor.bench.ts @@ -92,7 +92,7 @@ function resolveAgentosRoot(): string | null { ].filter((path): path is string => Boolean(path)); for (const candidate of candidates) { - if (existsSync(join(candidate, "registry/software/coreutils/wasm"))) { + if (existsSync(join(candidate, "software/coreutils/wasm"))) { return candidate; } } @@ -103,7 +103,7 @@ function commandModuleBytes(command: string, agentosRoot: string | null): number if (!agentosRoot) { return null; } - const modulePath = join(agentosRoot, "registry/software/coreutils/wasm", command); + const modulePath = join(agentosRoot, "software/coreutils/wasm", command); return existsSync(modulePath) ? statSync(modulePath).size : null; } diff --git a/packages/runtime-browser/AGENTS.md b/packages/runtime-browser/AGENTS.md deleted file mode 120000 index 681311eb9c..0000000000 --- a/packages/runtime-browser/AGENTS.md +++ /dev/null @@ -1 +0,0 @@ -CLAUDE.md \ No newline at end of file diff --git a/packages/runtime-browser/AGENTS.md b/packages/runtime-browser/AGENTS.md new file mode 100644 index 0000000000..22ea843adf --- /dev/null +++ b/packages/runtime-browser/AGENTS.md @@ -0,0 +1,5 @@ +# Browser Support + +- Browser support is untested after the secure-exec split; only build-level validation is required during migration. +- Provenance: moved from rivet-dev/agentos@87ed8e21e454. +- Keep this package generic to secure-exec. Agent OS browser glue belongs in Agent OS. diff --git a/packages/runtime-browser/scripts/check-bridge-contract.mjs b/packages/runtime-browser/scripts/check-bridge-contract.mjs index c1524001a5..bb913662fb 100644 --- a/packages/runtime-browser/scripts/check-bridge-contract.mjs +++ b/packages/runtime-browser/scripts/check-bridge-contract.mjs @@ -179,6 +179,7 @@ const browserUnsupportedContractGlobals = new Set([ "_networkHttpServerRespondRaw", "_networkHttpServerWaitRaw", "_processKill", + "_processTakeSignal", "_ptySetRawMode", "_pythonRpc", "_pythonStdinRead", diff --git a/packages/runtime-browser/src/runtime.ts b/packages/runtime-browser/src/runtime.ts index 33738bd364..8447b69ac3 100644 --- a/packages/runtime-browser/src/runtime.ts +++ b/packages/runtime-browser/src/runtime.ts @@ -1,13 +1,13 @@ +import { guestEncodingBootstrapCode } from "./encoding.js"; +import { BROWSER_BUFFER_POLYFILL_CODE } from "./generated/buffer-polyfill.js"; +import { BROWSER_PATH_POLYFILL_CODE } from "./generated/path-polyfill.js"; +import { BROWSER_UTIL_POLYFILL_CODE } from "./generated/util-polyfill.js"; import { createInMemoryFileSystem, InMemoryFileSystem, } from "./os-filesystem.js"; -import { guestEncodingBootstrapCode } from "./encoding.js"; -import { BROWSER_WASI_POLYFILL_CODE } from "./wasi-polyfill.js"; import { PROCESS_SIGNAL_NUMBERS } from "./signals.js"; -import { BROWSER_BUFFER_POLYFILL_CODE } from "./generated/buffer-polyfill.js"; -import { BROWSER_PATH_POLYFILL_CODE } from "./generated/path-polyfill.js"; -import { BROWSER_UTIL_POLYFILL_CODE } from "./generated/util-polyfill.js"; +import { BROWSER_WASI_POLYFILL_CODE } from "./wasi-polyfill.js"; export type StdioChannel = "stdout" | "stderr"; export type TimingMitigation = "off" | "freeze"; @@ -295,7 +295,11 @@ export interface NodeRuntimeDriver { /** End stdin for a running `streamingStdin` execution. */ endStdin?(executionId: string): void; /** Write bytes to a PTY fd owned by the execution. */ - writePty?(executionId: string, fd: number, data: string | Uint8Array): Promise; + writePty?( + executionId: string, + fd: number, + data: string | Uint8Array, + ): Promise; /** Read bytes from a PTY fd owned by the execution. */ readPty?( executionId: string, @@ -2957,9 +2961,15 @@ export const POLYFILL_CODE_MAP: Record = { const waitBuffer = new SharedArrayBuffer(4); const wait = new Int32Array(waitBuffer); const errnoSuccess = 0; + const errnoAcces = 2; const errnoBadf = 8; const errnoChild = 10; + const errnoInval = 28; + const errnoIo = 29; + const errnoNoent = 44; const errnoNosys = 52; + const errnoPerm = 63; + const errnoRofs = 69; let nextSyntheticFd = 1000; const syntheticFdEntries = new Map(); let activeFdOverrides = null; @@ -2993,6 +3003,16 @@ export const POLYFILL_CODE_MAP: Record = { if (stat && typeof stat.isSymbolicLink === "function" && stat.isSymbolicLink()) return 0o120777; return fallback >>> 0; }; + const hostFsErrno = (error) => { + switch (error && typeof error === "object" ? error.code : undefined) { + case "EACCES": return errnoAcces; + case "EPERM": return errnoPerm; + case "ENOENT": return errnoNoent; + case "EINVAL": return errnoInval; + case "EROFS": return errnoRofs; + default: return errnoIo; + } + }; const currentGuestCwd = () => { const cwd = typeof activeChildCwd === "string" && activeChildCwd.startsWith("/") ? activeChildCwd @@ -3084,7 +3104,7 @@ export const POLYFILL_CODE_MAP: Record = { try { const childWasi = new WASI({ returnOnExit: true, - args: [child.commandPath, ...child.argv.slice(1)], + args: child.argv.length > 0 ? child.argv : [child.commandPath], env: child.env, preopens: { "/": child.cwd || "/" }, }); @@ -3097,9 +3117,9 @@ export const POLYFILL_CODE_MAP: Record = { activeFdOverrides = child.overrides; activeChildCwd = child.cwd || "/"; const exitCode = childWasi.start(childInstance); - exitedChildren.set(child.pid, exitCode << 8); + exitedChildren.set(child.pid, exitCode); } catch { - exitedChildren.set(child.pid, 127 << 8); + exitedChildren.set(child.pid, 127); } finally { for (const handle of child.childOverrideHandles) closeSyntheticHandle(handle); activeFdOverrides = previousActiveFdOverrides; @@ -3264,15 +3284,14 @@ export const POLYFILL_CODE_MAP: Record = { return 0; } }, - // Matches node runner host_fs.chmod(fd, pathPtr, pathLen, mode): - // 0 on success, 1 on failure. + // Match node's host_fs chmod errno contract exactly. chmod(_fd, pathPtr, pathLen, mode) { try { const guestPath = resolveGuestPath(readString(pathPtr, pathLen)); fs().chmodSync(guestPath, Number(mode) >>> 0); - return 0; - } catch { - return 1; + return errnoSuccess; + } catch (error) { + return hostFsErrno(error); } }, // The node runner exports 7 host_fs symbols; mirror the full @@ -3316,23 +3335,37 @@ export const POLYFILL_CODE_MAP: Record = { return (1n << 64n) - 1n; } }, - // Browser fs() has no fd-based fchmod/ftruncate; provide the - // symbols (best-effort failure) so imports resolve. Rust guest - // binaries bypass wasi-libc stat/chmod/truncate, so these paths - // are unreached in practice. - fchmod(_fd, _mode) { - return 1; + fchmod(fd, mode) { + const handle = lookupSyntheticFd(fd >>> 0); + if (!handle || handle.kind !== "guest-file" || typeof handle.targetFd !== "number") { + return errnoBadf; + } + try { + fs().fchmodSync(handle.targetFd, Number(mode) >>> 0); + return errnoSuccess; + } catch (error) { + return hostFsErrno(error); + } }, - ftruncate(_fd, _length) { - return 1; + ftruncate(fd, length) { + const handle = lookupSyntheticFd(fd >>> 0); + if (!handle || handle.kind !== "guest-file" || typeof handle.targetFd !== "number") { + return errnoBadf; + } + try { + fs().ftruncateSync(handle.targetFd, Number(length)); + return errnoSuccess; + } catch (error) { + return hostFsErrno(error); + } }, }, host_process: { - proc_spawn(argvPtr, argvLen, envpPtr, envpLen, stdinFd, stdoutFd, stderrFd, cwdPtr, cwdLen, retPid) { + proc_spawn(execPathPtr, execPathLen, argvPtr, argvLen, envpPtr, envpLen, stdinFd, stdoutFd, stderrFd, cwdPtr, cwdLen, retPid) { try { + const commandPath = readString(execPathPtr, execPathLen); + if (!commandPath) return errnoNosys; const argv = decodeNullSeparated(readBytes(argvPtr, argvLen)); - if (argv.length === 0) return errnoNosys; - const commandPath = argv[0]; const commandName = commandPath.split("/").filter(Boolean).at(-1) || commandPath; const module = commandModules.get(commandName); if (!module) return errnoNosys; @@ -3374,22 +3407,24 @@ export const POLYFILL_CODE_MAP: Record = { return errnoNosys; } }, - proc_waitpid(pid, _options, retStatus, retPid) { + proc_waitpid(pid, _options, retExitCode, retSignal, retPid) { const requested = pid >>> 0; runReadyDeferredChildren(requested === 0xffffffff ? undefined : requested); const childPid = requested === 0xffffffff ? exitedChildren.keys().next().value : requested; if (!childPid || !exitedChildren.has(childPid)) { - if ((_options >>> 0) !== 0) { - writeU32(retStatus, 0); - writeU32(retPid, 0); + if ((_options >>> 0) !== 0) { + writeU32(retExitCode, 0); + writeU32(retSignal, 0); + writeU32(retPid, 0); return errnoSuccess; } writeU32(retPid, 0); return errnoChild; } - writeU32(retStatus, exitedChildren.get(childPid) || 0); + writeU32(retExitCode, exitedChildren.get(childPid) || 0); + writeU32(retSignal, 0); writeU32(retPid, childPid); exitedChildren.delete(childPid); return errnoSuccess; @@ -3413,6 +3448,15 @@ export const POLYFILL_CODE_MAP: Record = { replaceSyntheticFd(newFd >>> 0, cloned); return errnoSuccess; }, + proc_closefrom(lowFd) { + const minimumFd = lowFd >>> 0; + for (const [fd, handle] of Array.from(syntheticFdEntries.entries())) { + if (fd < minimumFd) continue; + closeSyntheticHandle(handle); + syntheticFdEntries.delete(fd); + } + return errnoSuccess; + }, fd_pipe(retReadFd, retWriteFd) { const pipe = { chunks: [], @@ -3431,7 +3475,11 @@ export const POLYFILL_CODE_MAP: Record = { proc_getppid(retPid) { return writeU32(retPid, 0); }, proc_kill() { return errnoNosys; }, sleep_ms(milliseconds) { - Atomics.wait(wait, 0, 0, milliseconds >>> 0); + const deadline = Date.now() + (milliseconds >>> 0); + while (Date.now() < deadline) { + // Keep guest sleeps interruptible by V8 termination during kill/dispose. + Atomics.wait(wait, 0, 0, Math.max(1, Math.min(10, deadline - Date.now()))); + } return errnoSuccess; }, pty_open() { return errnoNosys; }, diff --git a/packages/runtime-browser/src/wasi-polyfill.ts b/packages/runtime-browser/src/wasi-polyfill.ts index 40ee39ab9f..f62158a5b4 100644 --- a/packages/runtime-browser/src/wasi-polyfill.ts +++ b/packages/runtime-browser/src/wasi-polyfill.ts @@ -148,6 +148,7 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = const __agentOSWasiErrnoNoent = 44; const __agentOSWasiErrnoNosys = 52; const __agentOSWasiErrnoNotdir = 54; + const __agentOSWasiErrnoNotempty = 55; const __agentOSWasiErrnoPipe = 64; const __agentOSWasiErrnoRofs = 69; const __agentOSWasiErrnoNotcapable = 76; @@ -162,6 +163,7 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = const __agentOSWasiOpenDirectory = 2; const __agentOSWasiOpenExclusive = 4; const __agentOSWasiOpenTruncate = 8; + const __agentOSWasiFdflagsAppend = 1; const __agentOSWasiRightFdRead = 1n << 1n; const __agentOSWasiRightFdWrite = 1n << 6n; const __agentOSWasiDefaultRightsBase = 0xffffffffffffffffn; @@ -504,6 +506,14 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = } } + _hasReadRights(rights) { + try { + return (BigInt(rights) & __agentOSWasiRightFdRead) !== 0n; + } catch { + return true; + } + } + _writeUint32(ptr, value) { try { this._memoryView().setUint32(Number(ptr) >>> 0, Number(value) >>> 0, true); @@ -887,6 +897,9 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = } _mapFsError(error) { + __agentOSWasiDebug( + \`fs error code=\${String(error?.code ?? "")} message=\${String(error?.message ?? error)}\`, + ); switch (error?.code) { case "EACCES": case "EPERM": @@ -895,6 +908,8 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = return __agentOSWasiErrnoNoent; case "ENOTDIR": return __agentOSWasiErrnoNotdir; + case "ENOTEMPTY": + return __agentOSWasiErrnoNotempty; case "EEXIST": return __agentOSWasiErrnoExist; case "EINVAL": @@ -935,6 +950,7 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = refCount: 1, open: true, readOnly: entry.readOnly === true, + append: entry.append === true, }; } @@ -1488,19 +1504,44 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = return this._measureWasiPhase("writeResultPtr", () => this._writeUint32(nwrittenPtr, written)); } } + const entry = this._descriptorEntry(descriptor); + const localHostPassthrough = + handle.kind === "host-passthrough" && + entry?.kind === "file" && + entry.realFd === handle.targetFd; + const position = handle.append + ? this._measureWasiPhase("appendFstat", () => + Number(__agentOSFs().fstatSync(handle.targetFd).size ?? 0) + ) + : localHostPassthrough + ? (entry.offset ?? 0) + : null; const written = this._measureWasiPhase("writeSync", () => __agentOSFs().writeSync( handle.targetFd, bytes, 0, bytes.length, - null, + position, ) ); + if (localHostPassthrough) { + if (handle.append) { + entry.offset = this._measureWasiPhase("appendFstat", () => + Number(__agentOSFs().fstatSync(handle.targetFd).size ?? 0) + ); + } else { + entry.offset = (entry.offset ?? 0) + written; + } + } return this._measureWasiPhase("writeResultPtr", () => this._writeUint32(nwrittenPtr, written)); } if (handle?.kind === "guest-file" && typeof handle.targetFd === "number") { - const position = handle.append ? null : (handle.position ?? 0); + const position = handle.append + ? this._measureWasiPhase("appendFstat", () => + Number(__agentOSFs().fstatSync(handle.targetFd).size ?? 0) + ) + : (handle.position ?? 0); const written = this._measureWasiPhase("writeSync", () => __agentOSFs().writeSync( handle.targetFd, @@ -1577,7 +1618,11 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = } if (entry.kind === "file") { this._clearStatCache(); - const position = typeof entry.offset === "number" ? entry.offset : null; + const position = entry.append + ? this._measureWasiPhase("appendFstat", () => + Number(__agentOSFs().fstatSync(entry.realFd).size ?? 0) + ) + : (typeof entry.offset === "number" ? entry.offset : null); const written = this._measureWasiPhase("writeSync", () => __agentOSFs().writeSync( entry.realFd, @@ -1587,7 +1632,11 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = position, ) ); - if (typeof entry.offset === "number") { + if (entry.append) { + entry.offset = this._measureWasiPhase("appendFstat", () => + Number(__agentOSFs().fstatSync(entry.realFd).size ?? 0) + ); + } else if (typeof entry.offset === "number") { entry.offset += written; } return this._measureWasiPhase("writeResultPtr", () => this._writeUint32(nwrittenPtr, written)); @@ -1823,6 +1872,11 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = (handle?.kind === "passthrough" || handle?.kind === "host-passthrough") && typeof handle.targetFd === "number" ) { + const localEntry = this._descriptorEntry(descriptor); + const localHostPassthrough = + handle.kind === "host-passthrough" && + localEntry?.kind === "file" && + localEntry.realFd === handle.targetFd; const totalLength = this._boundedReadLength(iovs, iovsLen); const buffer = Buffer.alloc(totalLength); const bytesRead = __agentOSFs().readSync( @@ -1830,8 +1884,11 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = buffer, 0, totalLength, - null, + localHostPassthrough ? (localEntry.offset ?? 0) : null, ); + if (localHostPassthrough) { + localEntry.offset = (localEntry.offset ?? 0) + bytesRead; + } const written = this._writeToIovs(iovs, iovsLen, buffer.subarray(0, bytesRead)); return this._writeUint32(nreadPtr, written); } @@ -2314,8 +2371,12 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = this._clearStatCache(); } const fsConstants = __agentOSFs().constants ?? {}; + const requestedFdFlags = Number(_fdflags) >>> 0; + const append = (requestedFdFlags & __agentOSWasiFdflagsAppend) !== 0; let openFlags = requestedWriteAccess - ? fsConstants.O_RDWR ?? 2 + ? (this._hasReadRights(requestedRightsBase) + ? fsConstants.O_RDWR ?? 2 + : fsConstants.O_WRONLY ?? 1) : fsConstants.O_RDONLY ?? 0; if ((requestedFlags & __agentOSWasiOpenCreate) !== 0) { openFlags |= fsConstants.O_CREAT ?? 64; @@ -2326,6 +2387,9 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = if ((requestedFlags & __agentOSWasiOpenTruncate) !== 0) { openFlags |= fsConstants.O_TRUNC ?? 512; } + if (append) { + openFlags |= fsConstants.O_APPEND ?? 1024; + } if (openDirectory) { openFlags |= fsConstants.O_DIRECTORY ?? 0; } @@ -2343,10 +2407,13 @@ if (typeof globalThis !== "undefined" && typeof globalThis.__agentOSWasiModule = hostPath: fsPath, readOnly: resolved.readOnly === true, realFd, - offset: 0, + offset: append + ? Number(__agentOSFs().fstatSync(realFd).size ?? 0) + : 0, + append, rightsBase: requestedRightsBase & allowedRightsInheriting, rightsInheriting: requestedRightsInheriting & allowedRightsInheriting, - fdFlags: (Number(_fdflags) >>> 0) & 0xffff, + fdFlags: requestedFdFlags & 0xffff, }); }); return this._measureWasiPhase("writeOpenedFd", () => this._writeUint32(openedFdPtr, openedFd)); diff --git a/packages/runtime-browser/src/worker.ts b/packages/runtime-browser/src/worker.ts index bf77c5aba1..de5cde3343 100644 --- a/packages/runtime-browser/src/worker.ts +++ b/packages/runtime-browser/src/worker.ts @@ -1,13 +1,13 @@ -import { hmac as nobleHmac } from "@noble/hashes/hmac.js"; import { cbc as nobleAesCbc, ctr as nobleAesCtr, gcm as nobleAesGcm, } from "@noble/ciphers/aes.js"; +import { hmac as nobleHmac } from "@noble/hashes/hmac.js"; import { md5, sha1 } from "@noble/hashes/legacy.js"; import { pbkdf2 as noblePbkdf2 } from "@noble/hashes/pbkdf2.js"; -import { sha224, sha256, sha384, sha512 } from "@noble/hashes/sha2.js"; import { scrypt as nobleScrypt } from "@noble/hashes/scrypt.js"; +import { sha224, sha256, sha384, sha512 } from "@noble/hashes/sha2.js"; import { transform } from "sucrase"; import type { BrowserChildProcessPollEvent, @@ -16,11 +16,6 @@ import type { import { createBrowserNetworkAdapter } from "./driver.js"; import { base64ToBytes, toUint8Array } from "./encoding.js"; import { posixErrno } from "./errno.js"; -import { - PROCESS_SIGNAL_NUMBERS, - defaultSignalExitCode, - signalNumberForEvent, -} from "./signals.js"; import type { ExecResult, NetworkAdapter, @@ -40,6 +35,11 @@ import { POLYFILL_CODE_MAP, transformDynamicImport, } from "./runtime.js"; +import { + defaultSignalExitCode, + PROCESS_SIGNAL_NUMBERS, + signalNumberForEvent, +} from "./signals.js"; import { assertBrowserSyncBridgeSupport, type BrowserSyncBridgeErrorPayload, @@ -94,8 +94,9 @@ const MAX_STDIO_MESSAGE_CHARS = 8192; function eventForSignalNumber(signal: number): string { return ( - Object.entries(PROCESS_SIGNAL_NUMBERS).find(([, value]) => value === signal)?.[0] ?? - `SIG${signal}` + Object.entries(PROCESS_SIGNAL_NUMBERS).find( + ([, value]) => value === signal, + )?.[0] ?? `SIG${signal}` ); } @@ -1626,6 +1627,12 @@ function createFsModule(syncBridge: ReturnType) { fstatSync(fd: number) { return statSync(requireOpenFd(fd, "fstat").path); }, + fchmodSync(fd: number, mode: number) { + syncBridge.requestVoid("fs.chmod", [ + requireOpenFd(fd, "fchmod").path, + Number(mode) || 0, + ]); + }, ftruncateSync(fd: number, length = 0) { syncBridge.requestVoid("fs.truncate", [ requireOpenFd(fd, "ftruncate").path, @@ -2711,7 +2718,11 @@ function createBrowserProcess(): Record { slaveFd = pair.slaveFd; path = typeof pair.path === "string" ? pair.path : undefined; ptySyncBridge().requestVoid("pty.resize", [ - { fd: slaveFd as number, cols: Math.max(1, columns), rows: Math.max(1, rows) }, + { + fd: slaveFd as number, + cols: Math.max(1, columns), + rows: Math.max(1, rows), + }, ]); if (activeExecutionId && activeProcessRequestId !== null) { postPtyOpened(activeExecutionId, activeProcessRequestId, { @@ -3265,7 +3276,9 @@ function createBrowserProcess(): Record { }, kill(pid: number, signal: string | number = "SIGTERM") { if (pid !== processBridge.pid) { - throw new Error(`process.kill only supports the current browser process pid (${processBridge.pid})`); + throw new Error( + `process.kill only supports the current browser process pid (${processBridge.pid})`, + ); } const event = typeof signal === "number" diff --git a/packages/runtime-core/commands/[ b/packages/runtime-core/commands/[ deleted file mode 100644 index 321231e0c4..0000000000 Binary files a/packages/runtime-core/commands/[ and /dev/null differ diff --git a/packages/runtime-core/commands/_stubs b/packages/runtime-core/commands/_stubs deleted file mode 100644 index abc6255662..0000000000 Binary files a/packages/runtime-core/commands/_stubs and /dev/null differ diff --git a/packages/runtime-core/commands/arch b/packages/runtime-core/commands/arch deleted file mode 100644 index 414eecc3c7..0000000000 Binary files a/packages/runtime-core/commands/arch and /dev/null differ diff --git a/packages/runtime-core/commands/awk b/packages/runtime-core/commands/awk deleted file mode 100644 index c4048f65c4..0000000000 Binary files a/packages/runtime-core/commands/awk and /dev/null differ diff --git a/packages/runtime-core/commands/b2sum b/packages/runtime-core/commands/b2sum deleted file mode 100644 index cb078b1959..0000000000 Binary files a/packages/runtime-core/commands/b2sum and /dev/null differ diff --git a/packages/runtime-core/commands/base32 b/packages/runtime-core/commands/base32 deleted file mode 100644 index 662f4584d8..0000000000 Binary files a/packages/runtime-core/commands/base32 and /dev/null differ diff --git a/packages/runtime-core/commands/base64 b/packages/runtime-core/commands/base64 deleted file mode 100644 index e315357a92..0000000000 Binary files a/packages/runtime-core/commands/base64 and /dev/null differ diff --git a/packages/runtime-core/commands/basename b/packages/runtime-core/commands/basename deleted file mode 100644 index 5a5547db31..0000000000 Binary files a/packages/runtime-core/commands/basename and /dev/null differ diff --git a/packages/runtime-core/commands/basenc b/packages/runtime-core/commands/basenc deleted file mode 100644 index faed8dc645..0000000000 Binary files a/packages/runtime-core/commands/basenc and /dev/null differ diff --git a/packages/runtime-core/commands/bash b/packages/runtime-core/commands/bash deleted file mode 100644 index 9e80dda943..0000000000 Binary files a/packages/runtime-core/commands/bash and /dev/null differ diff --git a/packages/runtime-core/commands/cat b/packages/runtime-core/commands/cat deleted file mode 100644 index d48dcaa8cd..0000000000 Binary files a/packages/runtime-core/commands/cat and /dev/null differ diff --git a/packages/runtime-core/commands/chcon b/packages/runtime-core/commands/chcon deleted file mode 100644 index abc6255662..0000000000 Binary files a/packages/runtime-core/commands/chcon and /dev/null differ diff --git a/packages/runtime-core/commands/chgrp b/packages/runtime-core/commands/chgrp deleted file mode 100644 index abc6255662..0000000000 Binary files a/packages/runtime-core/commands/chgrp and /dev/null differ diff --git a/packages/runtime-core/commands/chmod b/packages/runtime-core/commands/chmod deleted file mode 100644 index 05e1293098..0000000000 Binary files a/packages/runtime-core/commands/chmod and /dev/null differ diff --git a/packages/runtime-core/commands/chown b/packages/runtime-core/commands/chown deleted file mode 100644 index abc6255662..0000000000 Binary files a/packages/runtime-core/commands/chown and /dev/null differ diff --git a/packages/runtime-core/commands/chroot b/packages/runtime-core/commands/chroot deleted file mode 100644 index abc6255662..0000000000 Binary files a/packages/runtime-core/commands/chroot and /dev/null differ diff --git a/packages/runtime-core/commands/cksum b/packages/runtime-core/commands/cksum deleted file mode 100644 index 45bb83c1f8..0000000000 Binary files a/packages/runtime-core/commands/cksum and /dev/null differ diff --git a/packages/runtime-core/commands/codex b/packages/runtime-core/commands/codex deleted file mode 100644 index 381586fa9d..0000000000 Binary files a/packages/runtime-core/commands/codex and /dev/null differ diff --git a/packages/runtime-core/commands/codex-exec b/packages/runtime-core/commands/codex-exec deleted file mode 100644 index a151f153a8..0000000000 Binary files a/packages/runtime-core/commands/codex-exec and /dev/null differ diff --git a/packages/runtime-core/commands/column b/packages/runtime-core/commands/column deleted file mode 100644 index 1580c3b660..0000000000 Binary files a/packages/runtime-core/commands/column and /dev/null differ diff --git a/packages/runtime-core/commands/comm b/packages/runtime-core/commands/comm deleted file mode 100644 index 63c7779a2d..0000000000 Binary files a/packages/runtime-core/commands/comm and /dev/null differ diff --git a/packages/runtime-core/commands/cp b/packages/runtime-core/commands/cp deleted file mode 100644 index d2af9d05b3..0000000000 Binary files a/packages/runtime-core/commands/cp and /dev/null differ diff --git a/packages/runtime-core/commands/curl b/packages/runtime-core/commands/curl deleted file mode 100644 index 5e85f8a87d..0000000000 Binary files a/packages/runtime-core/commands/curl and /dev/null differ diff --git a/packages/runtime-core/commands/cut b/packages/runtime-core/commands/cut deleted file mode 100644 index 2c58944d96..0000000000 Binary files a/packages/runtime-core/commands/cut and /dev/null differ diff --git a/packages/runtime-core/commands/date b/packages/runtime-core/commands/date deleted file mode 100644 index 16c8308fc5..0000000000 Binary files a/packages/runtime-core/commands/date and /dev/null differ diff --git a/packages/runtime-core/commands/dd b/packages/runtime-core/commands/dd deleted file mode 100644 index e255315326..0000000000 Binary files a/packages/runtime-core/commands/dd and /dev/null differ diff --git a/packages/runtime-core/commands/df b/packages/runtime-core/commands/df deleted file mode 100644 index abc6255662..0000000000 Binary files a/packages/runtime-core/commands/df and /dev/null differ diff --git a/packages/runtime-core/commands/diff b/packages/runtime-core/commands/diff deleted file mode 100644 index 003d074f99..0000000000 Binary files a/packages/runtime-core/commands/diff and /dev/null differ diff --git a/packages/runtime-core/commands/dir b/packages/runtime-core/commands/dir deleted file mode 100644 index 3dc3e5295e..0000000000 Binary files a/packages/runtime-core/commands/dir and /dev/null differ diff --git a/packages/runtime-core/commands/dircolors b/packages/runtime-core/commands/dircolors deleted file mode 100644 index f230724e29..0000000000 Binary files a/packages/runtime-core/commands/dircolors and /dev/null differ diff --git a/packages/runtime-core/commands/dirname b/packages/runtime-core/commands/dirname deleted file mode 100644 index 6f21608159..0000000000 Binary files a/packages/runtime-core/commands/dirname and /dev/null differ diff --git a/packages/runtime-core/commands/du b/packages/runtime-core/commands/du deleted file mode 100644 index bd087ccddf..0000000000 Binary files a/packages/runtime-core/commands/du and /dev/null differ diff --git a/packages/runtime-core/commands/echo b/packages/runtime-core/commands/echo deleted file mode 100644 index dbe1cb5b22..0000000000 Binary files a/packages/runtime-core/commands/echo and /dev/null differ diff --git a/packages/runtime-core/commands/egrep b/packages/runtime-core/commands/egrep deleted file mode 100644 index c78e121f14..0000000000 Binary files a/packages/runtime-core/commands/egrep and /dev/null differ diff --git a/packages/runtime-core/commands/env b/packages/runtime-core/commands/env deleted file mode 100644 index d9860269d1..0000000000 Binary files a/packages/runtime-core/commands/env and /dev/null differ diff --git a/packages/runtime-core/commands/envsubst b/packages/runtime-core/commands/envsubst deleted file mode 100644 index 46b6f4f8c7..0000000000 Binary files a/packages/runtime-core/commands/envsubst and /dev/null differ diff --git a/packages/runtime-core/commands/expand b/packages/runtime-core/commands/expand deleted file mode 100644 index acf0b7e03d..0000000000 Binary files a/packages/runtime-core/commands/expand and /dev/null differ diff --git a/packages/runtime-core/commands/expr b/packages/runtime-core/commands/expr deleted file mode 100644 index b2825b7557..0000000000 Binary files a/packages/runtime-core/commands/expr and /dev/null differ diff --git a/packages/runtime-core/commands/factor b/packages/runtime-core/commands/factor deleted file mode 100644 index 4c1d451774..0000000000 Binary files a/packages/runtime-core/commands/factor and /dev/null differ diff --git a/packages/runtime-core/commands/false b/packages/runtime-core/commands/false deleted file mode 100644 index 0913e06b95..0000000000 Binary files a/packages/runtime-core/commands/false and /dev/null differ diff --git a/packages/runtime-core/commands/fd b/packages/runtime-core/commands/fd deleted file mode 100644 index b724ec24b1..0000000000 Binary files a/packages/runtime-core/commands/fd and /dev/null differ diff --git a/packages/runtime-core/commands/fgrep b/packages/runtime-core/commands/fgrep deleted file mode 100644 index c78e121f14..0000000000 Binary files a/packages/runtime-core/commands/fgrep and /dev/null differ diff --git a/packages/runtime-core/commands/file b/packages/runtime-core/commands/file deleted file mode 100644 index 1e32fe6064..0000000000 Binary files a/packages/runtime-core/commands/file and /dev/null differ diff --git a/packages/runtime-core/commands/find b/packages/runtime-core/commands/find deleted file mode 100644 index e08e8b58a1..0000000000 Binary files a/packages/runtime-core/commands/find and /dev/null differ diff --git a/packages/runtime-core/commands/fmt b/packages/runtime-core/commands/fmt deleted file mode 100644 index 8f7f5b020a..0000000000 Binary files a/packages/runtime-core/commands/fmt and /dev/null differ diff --git a/packages/runtime-core/commands/fold b/packages/runtime-core/commands/fold deleted file mode 100644 index 66893046f6..0000000000 Binary files a/packages/runtime-core/commands/fold and /dev/null differ diff --git a/packages/runtime-core/commands/git b/packages/runtime-core/commands/git deleted file mode 100644 index 1865311237..0000000000 Binary files a/packages/runtime-core/commands/git and /dev/null differ diff --git a/packages/runtime-core/commands/git-remote-http b/packages/runtime-core/commands/git-remote-http deleted file mode 100644 index 1865311237..0000000000 Binary files a/packages/runtime-core/commands/git-remote-http and /dev/null differ diff --git a/packages/runtime-core/commands/git-remote-https b/packages/runtime-core/commands/git-remote-https deleted file mode 100644 index 1865311237..0000000000 Binary files a/packages/runtime-core/commands/git-remote-https and /dev/null differ diff --git a/packages/runtime-core/commands/grep b/packages/runtime-core/commands/grep deleted file mode 100644 index c78e121f14..0000000000 Binary files a/packages/runtime-core/commands/grep and /dev/null differ diff --git a/packages/runtime-core/commands/groups b/packages/runtime-core/commands/groups deleted file mode 100644 index abc6255662..0000000000 Binary files a/packages/runtime-core/commands/groups and /dev/null differ diff --git a/packages/runtime-core/commands/gunzip b/packages/runtime-core/commands/gunzip deleted file mode 100644 index e60ebe5317..0000000000 Binary files a/packages/runtime-core/commands/gunzip and /dev/null differ diff --git a/packages/runtime-core/commands/gzip b/packages/runtime-core/commands/gzip deleted file mode 100644 index e60ebe5317..0000000000 Binary files a/packages/runtime-core/commands/gzip and /dev/null differ diff --git a/packages/runtime-core/commands/head b/packages/runtime-core/commands/head deleted file mode 100644 index 28a97356d7..0000000000 Binary files a/packages/runtime-core/commands/head and /dev/null differ diff --git a/packages/runtime-core/commands/hostid b/packages/runtime-core/commands/hostid deleted file mode 100644 index abc6255662..0000000000 Binary files a/packages/runtime-core/commands/hostid and /dev/null differ diff --git a/packages/runtime-core/commands/hostname b/packages/runtime-core/commands/hostname deleted file mode 100644 index abc6255662..0000000000 Binary files a/packages/runtime-core/commands/hostname and /dev/null differ diff --git a/packages/runtime-core/commands/http-test b/packages/runtime-core/commands/http-test deleted file mode 100644 index a36c814d00..0000000000 Binary files a/packages/runtime-core/commands/http-test and /dev/null differ diff --git a/packages/runtime-core/commands/http_get b/packages/runtime-core/commands/http_get deleted file mode 100644 index 52a1fb8342..0000000000 Binary files a/packages/runtime-core/commands/http_get and /dev/null differ diff --git a/packages/runtime-core/commands/id b/packages/runtime-core/commands/id deleted file mode 100644 index abc6255662..0000000000 Binary files a/packages/runtime-core/commands/id and /dev/null differ diff --git a/packages/runtime-core/commands/install b/packages/runtime-core/commands/install deleted file mode 100644 index abc6255662..0000000000 Binary files a/packages/runtime-core/commands/install and /dev/null differ diff --git a/packages/runtime-core/commands/join b/packages/runtime-core/commands/join deleted file mode 100644 index 60b7cad297..0000000000 Binary files a/packages/runtime-core/commands/join and /dev/null differ diff --git a/packages/runtime-core/commands/jq b/packages/runtime-core/commands/jq deleted file mode 100644 index 22f0359722..0000000000 Binary files a/packages/runtime-core/commands/jq and /dev/null differ diff --git a/packages/runtime-core/commands/kill b/packages/runtime-core/commands/kill deleted file mode 100644 index abc6255662..0000000000 Binary files a/packages/runtime-core/commands/kill and /dev/null differ diff --git a/packages/runtime-core/commands/link b/packages/runtime-core/commands/link deleted file mode 100644 index 609677d3fe..0000000000 Binary files a/packages/runtime-core/commands/link and /dev/null differ diff --git a/packages/runtime-core/commands/ln b/packages/runtime-core/commands/ln deleted file mode 100644 index 7930e6370e..0000000000 Binary files a/packages/runtime-core/commands/ln and /dev/null differ diff --git a/packages/runtime-core/commands/logname b/packages/runtime-core/commands/logname deleted file mode 100644 index f0fad8d365..0000000000 Binary files a/packages/runtime-core/commands/logname and /dev/null differ diff --git a/packages/runtime-core/commands/ls b/packages/runtime-core/commands/ls deleted file mode 100644 index 3dc3e5295e..0000000000 Binary files a/packages/runtime-core/commands/ls and /dev/null differ diff --git a/packages/runtime-core/commands/md5sum b/packages/runtime-core/commands/md5sum deleted file mode 100644 index 38d5041ca5..0000000000 Binary files a/packages/runtime-core/commands/md5sum and /dev/null differ diff --git a/packages/runtime-core/commands/mkdir b/packages/runtime-core/commands/mkdir deleted file mode 100644 index ff1921b459..0000000000 Binary files a/packages/runtime-core/commands/mkdir and /dev/null differ diff --git a/packages/runtime-core/commands/mkfifo b/packages/runtime-core/commands/mkfifo deleted file mode 100644 index abc6255662..0000000000 Binary files a/packages/runtime-core/commands/mkfifo and /dev/null differ diff --git a/packages/runtime-core/commands/mknod b/packages/runtime-core/commands/mknod deleted file mode 100644 index abc6255662..0000000000 Binary files a/packages/runtime-core/commands/mknod and /dev/null differ diff --git a/packages/runtime-core/commands/mktemp b/packages/runtime-core/commands/mktemp deleted file mode 100644 index a67bb8b610..0000000000 Binary files a/packages/runtime-core/commands/mktemp and /dev/null differ diff --git a/packages/runtime-core/commands/more b/packages/runtime-core/commands/more deleted file mode 100644 index d48dcaa8cd..0000000000 Binary files a/packages/runtime-core/commands/more and /dev/null differ diff --git a/packages/runtime-core/commands/mv b/packages/runtime-core/commands/mv deleted file mode 100644 index db5e849abc..0000000000 Binary files a/packages/runtime-core/commands/mv and /dev/null differ diff --git a/packages/runtime-core/commands/nice b/packages/runtime-core/commands/nice deleted file mode 100644 index f0d9cb44fb..0000000000 Binary files a/packages/runtime-core/commands/nice and /dev/null differ diff --git a/packages/runtime-core/commands/nl b/packages/runtime-core/commands/nl deleted file mode 100644 index 53abc2d686..0000000000 Binary files a/packages/runtime-core/commands/nl and /dev/null differ diff --git a/packages/runtime-core/commands/nohup b/packages/runtime-core/commands/nohup deleted file mode 100644 index 5b91d7ca8a..0000000000 Binary files a/packages/runtime-core/commands/nohup and /dev/null differ diff --git a/packages/runtime-core/commands/nproc b/packages/runtime-core/commands/nproc deleted file mode 100644 index 30671ed9d6..0000000000 Binary files a/packages/runtime-core/commands/nproc and /dev/null differ diff --git a/packages/runtime-core/commands/numfmt b/packages/runtime-core/commands/numfmt deleted file mode 100644 index ed496d7db7..0000000000 Binary files a/packages/runtime-core/commands/numfmt and /dev/null differ diff --git a/packages/runtime-core/commands/od b/packages/runtime-core/commands/od deleted file mode 100644 index 29edaafc0c..0000000000 Binary files a/packages/runtime-core/commands/od and /dev/null differ diff --git a/packages/runtime-core/commands/paste b/packages/runtime-core/commands/paste deleted file mode 100644 index 2ffd02940f..0000000000 Binary files a/packages/runtime-core/commands/paste and /dev/null differ diff --git a/packages/runtime-core/commands/pathchk b/packages/runtime-core/commands/pathchk deleted file mode 100644 index 785f1f7af3..0000000000 Binary files a/packages/runtime-core/commands/pathchk and /dev/null differ diff --git a/packages/runtime-core/commands/pinky b/packages/runtime-core/commands/pinky deleted file mode 100644 index abc6255662..0000000000 Binary files a/packages/runtime-core/commands/pinky and /dev/null differ diff --git a/packages/runtime-core/commands/printenv b/packages/runtime-core/commands/printenv deleted file mode 100644 index cbb8bc32cd..0000000000 Binary files a/packages/runtime-core/commands/printenv and /dev/null differ diff --git a/packages/runtime-core/commands/printf b/packages/runtime-core/commands/printf deleted file mode 100644 index 244d2d9ae2..0000000000 Binary files a/packages/runtime-core/commands/printf and /dev/null differ diff --git a/packages/runtime-core/commands/ptx b/packages/runtime-core/commands/ptx deleted file mode 100644 index 881c9e8850..0000000000 Binary files a/packages/runtime-core/commands/ptx and /dev/null differ diff --git a/packages/runtime-core/commands/pwd b/packages/runtime-core/commands/pwd deleted file mode 100644 index e173b2b9b2..0000000000 Binary files a/packages/runtime-core/commands/pwd and /dev/null differ diff --git a/packages/runtime-core/commands/readlink b/packages/runtime-core/commands/readlink deleted file mode 100644 index 5c4dcb2009..0000000000 Binary files a/packages/runtime-core/commands/readlink and /dev/null differ diff --git a/packages/runtime-core/commands/realpath b/packages/runtime-core/commands/realpath deleted file mode 100644 index 6df9bfa73f..0000000000 Binary files a/packages/runtime-core/commands/realpath and /dev/null differ diff --git a/packages/runtime-core/commands/rev b/packages/runtime-core/commands/rev deleted file mode 100644 index 864b62afd1..0000000000 Binary files a/packages/runtime-core/commands/rev and /dev/null differ diff --git a/packages/runtime-core/commands/rg b/packages/runtime-core/commands/rg deleted file mode 100644 index 76a63b75ea..0000000000 Binary files a/packages/runtime-core/commands/rg and /dev/null differ diff --git a/packages/runtime-core/commands/rm b/packages/runtime-core/commands/rm deleted file mode 100644 index e00af18f8e..0000000000 Binary files a/packages/runtime-core/commands/rm and /dev/null differ diff --git a/packages/runtime-core/commands/rmdir b/packages/runtime-core/commands/rmdir deleted file mode 100644 index 3ec6f0f036..0000000000 Binary files a/packages/runtime-core/commands/rmdir and /dev/null differ diff --git a/packages/runtime-core/commands/runcon b/packages/runtime-core/commands/runcon deleted file mode 100644 index abc6255662..0000000000 Binary files a/packages/runtime-core/commands/runcon and /dev/null differ diff --git a/packages/runtime-core/commands/sed b/packages/runtime-core/commands/sed deleted file mode 100644 index cf0fd26325..0000000000 Binary files a/packages/runtime-core/commands/sed and /dev/null differ diff --git a/packages/runtime-core/commands/seq b/packages/runtime-core/commands/seq deleted file mode 100644 index 45a46fd826..0000000000 Binary files a/packages/runtime-core/commands/seq and /dev/null differ diff --git a/packages/runtime-core/commands/sh b/packages/runtime-core/commands/sh deleted file mode 100644 index 9e80dda943..0000000000 Binary files a/packages/runtime-core/commands/sh and /dev/null differ diff --git a/packages/runtime-core/commands/sha1sum b/packages/runtime-core/commands/sha1sum deleted file mode 100644 index a45e8a03a1..0000000000 Binary files a/packages/runtime-core/commands/sha1sum and /dev/null differ diff --git a/packages/runtime-core/commands/sha224sum b/packages/runtime-core/commands/sha224sum deleted file mode 100644 index f4d681dbc0..0000000000 Binary files a/packages/runtime-core/commands/sha224sum and /dev/null differ diff --git a/packages/runtime-core/commands/sha256sum b/packages/runtime-core/commands/sha256sum deleted file mode 100644 index 9a1d01135d..0000000000 Binary files a/packages/runtime-core/commands/sha256sum and /dev/null differ diff --git a/packages/runtime-core/commands/sha384sum b/packages/runtime-core/commands/sha384sum deleted file mode 100644 index a5f63b13dd..0000000000 Binary files a/packages/runtime-core/commands/sha384sum and /dev/null differ diff --git a/packages/runtime-core/commands/sha512sum b/packages/runtime-core/commands/sha512sum deleted file mode 100644 index cda448b94f..0000000000 Binary files a/packages/runtime-core/commands/sha512sum and /dev/null differ diff --git a/packages/runtime-core/commands/shred b/packages/runtime-core/commands/shred deleted file mode 100644 index a0d21cd818..0000000000 Binary files a/packages/runtime-core/commands/shred and /dev/null differ diff --git a/packages/runtime-core/commands/shuf b/packages/runtime-core/commands/shuf deleted file mode 100644 index 974b618c1d..0000000000 Binary files a/packages/runtime-core/commands/shuf and /dev/null differ diff --git a/packages/runtime-core/commands/sleep b/packages/runtime-core/commands/sleep deleted file mode 100644 index af6efb965e..0000000000 Binary files a/packages/runtime-core/commands/sleep and /dev/null differ diff --git a/packages/runtime-core/commands/sort b/packages/runtime-core/commands/sort deleted file mode 100644 index 1d6dc77ac3..0000000000 Binary files a/packages/runtime-core/commands/sort and /dev/null differ diff --git a/packages/runtime-core/commands/spawn-test-host b/packages/runtime-core/commands/spawn-test-host deleted file mode 100644 index 6c412c79c9..0000000000 Binary files a/packages/runtime-core/commands/spawn-test-host and /dev/null differ diff --git a/packages/runtime-core/commands/split b/packages/runtime-core/commands/split deleted file mode 100644 index a5821574ca..0000000000 Binary files a/packages/runtime-core/commands/split and /dev/null differ diff --git a/packages/runtime-core/commands/sqlite3 b/packages/runtime-core/commands/sqlite3 deleted file mode 100755 index a9ee1f206f..0000000000 Binary files a/packages/runtime-core/commands/sqlite3 and /dev/null differ diff --git a/packages/runtime-core/commands/stat b/packages/runtime-core/commands/stat deleted file mode 100644 index af4a973161..0000000000 Binary files a/packages/runtime-core/commands/stat and /dev/null differ diff --git a/packages/runtime-core/commands/stdbuf b/packages/runtime-core/commands/stdbuf deleted file mode 100644 index d5884b404a..0000000000 Binary files a/packages/runtime-core/commands/stdbuf and /dev/null differ diff --git a/packages/runtime-core/commands/strings b/packages/runtime-core/commands/strings deleted file mode 100644 index 9b501f8ec3..0000000000 Binary files a/packages/runtime-core/commands/strings and /dev/null differ diff --git a/packages/runtime-core/commands/stty b/packages/runtime-core/commands/stty deleted file mode 100644 index abc6255662..0000000000 Binary files a/packages/runtime-core/commands/stty and /dev/null differ diff --git a/packages/runtime-core/commands/sum b/packages/runtime-core/commands/sum deleted file mode 100644 index 5dd3eaf7a5..0000000000 Binary files a/packages/runtime-core/commands/sum and /dev/null differ diff --git a/packages/runtime-core/commands/sync b/packages/runtime-core/commands/sync deleted file mode 100644 index abc6255662..0000000000 Binary files a/packages/runtime-core/commands/sync and /dev/null differ diff --git a/packages/runtime-core/commands/tac b/packages/runtime-core/commands/tac deleted file mode 100644 index 35179d8163..0000000000 Binary files a/packages/runtime-core/commands/tac and /dev/null differ diff --git a/packages/runtime-core/commands/tail b/packages/runtime-core/commands/tail deleted file mode 100644 index 0838f8e1ca..0000000000 Binary files a/packages/runtime-core/commands/tail and /dev/null differ diff --git a/packages/runtime-core/commands/tar b/packages/runtime-core/commands/tar deleted file mode 100644 index 5d813b99d9..0000000000 Binary files a/packages/runtime-core/commands/tar and /dev/null differ diff --git a/packages/runtime-core/commands/tee b/packages/runtime-core/commands/tee deleted file mode 100644 index b23a82ccbd..0000000000 Binary files a/packages/runtime-core/commands/tee and /dev/null differ diff --git a/packages/runtime-core/commands/test b/packages/runtime-core/commands/test deleted file mode 100644 index 321231e0c4..0000000000 Binary files a/packages/runtime-core/commands/test and /dev/null differ diff --git a/packages/runtime-core/commands/timeout b/packages/runtime-core/commands/timeout deleted file mode 100644 index 2638c09dc5..0000000000 Binary files a/packages/runtime-core/commands/timeout and /dev/null differ diff --git a/packages/runtime-core/commands/touch b/packages/runtime-core/commands/touch deleted file mode 100644 index fd1ed0710f..0000000000 Binary files a/packages/runtime-core/commands/touch and /dev/null differ diff --git a/packages/runtime-core/commands/tr b/packages/runtime-core/commands/tr deleted file mode 100644 index 536e993b9a..0000000000 Binary files a/packages/runtime-core/commands/tr and /dev/null differ diff --git a/packages/runtime-core/commands/tree b/packages/runtime-core/commands/tree deleted file mode 100644 index 49e9c7c18d..0000000000 Binary files a/packages/runtime-core/commands/tree and /dev/null differ diff --git a/packages/runtime-core/commands/true b/packages/runtime-core/commands/true deleted file mode 100644 index e08e0be528..0000000000 Binary files a/packages/runtime-core/commands/true and /dev/null differ diff --git a/packages/runtime-core/commands/truncate b/packages/runtime-core/commands/truncate deleted file mode 100644 index c693df7acb..0000000000 Binary files a/packages/runtime-core/commands/truncate and /dev/null differ diff --git a/packages/runtime-core/commands/tsort b/packages/runtime-core/commands/tsort deleted file mode 100644 index 15609b02d8..0000000000 Binary files a/packages/runtime-core/commands/tsort and /dev/null differ diff --git a/packages/runtime-core/commands/tty b/packages/runtime-core/commands/tty deleted file mode 100644 index abc6255662..0000000000 Binary files a/packages/runtime-core/commands/tty and /dev/null differ diff --git a/packages/runtime-core/commands/uname b/packages/runtime-core/commands/uname deleted file mode 100644 index 7c9268ff8c..0000000000 Binary files a/packages/runtime-core/commands/uname and /dev/null differ diff --git a/packages/runtime-core/commands/unexpand b/packages/runtime-core/commands/unexpand deleted file mode 100644 index 50728dccb0..0000000000 Binary files a/packages/runtime-core/commands/unexpand and /dev/null differ diff --git a/packages/runtime-core/commands/uniq b/packages/runtime-core/commands/uniq deleted file mode 100644 index e243c67dd6..0000000000 Binary files a/packages/runtime-core/commands/uniq and /dev/null differ diff --git a/packages/runtime-core/commands/unlink b/packages/runtime-core/commands/unlink deleted file mode 100644 index de09163081..0000000000 Binary files a/packages/runtime-core/commands/unlink and /dev/null differ diff --git a/packages/runtime-core/commands/unzip b/packages/runtime-core/commands/unzip deleted file mode 100644 index 07f40041e1..0000000000 Binary files a/packages/runtime-core/commands/unzip and /dev/null differ diff --git a/packages/runtime-core/commands/uptime b/packages/runtime-core/commands/uptime deleted file mode 100644 index abc6255662..0000000000 Binary files a/packages/runtime-core/commands/uptime and /dev/null differ diff --git a/packages/runtime-core/commands/users b/packages/runtime-core/commands/users deleted file mode 100644 index abc6255662..0000000000 Binary files a/packages/runtime-core/commands/users and /dev/null differ diff --git a/packages/runtime-core/commands/vdir b/packages/runtime-core/commands/vdir deleted file mode 100644 index 3dc3e5295e..0000000000 Binary files a/packages/runtime-core/commands/vdir and /dev/null differ diff --git a/packages/runtime-core/commands/wc b/packages/runtime-core/commands/wc deleted file mode 100644 index 3b7e2fdf39..0000000000 Binary files a/packages/runtime-core/commands/wc and /dev/null differ diff --git a/packages/runtime-core/commands/which b/packages/runtime-core/commands/which deleted file mode 100644 index a27bc06037..0000000000 Binary files a/packages/runtime-core/commands/which and /dev/null differ diff --git a/packages/runtime-core/commands/who b/packages/runtime-core/commands/who deleted file mode 100644 index abc6255662..0000000000 Binary files a/packages/runtime-core/commands/who and /dev/null differ diff --git a/packages/runtime-core/commands/whoami b/packages/runtime-core/commands/whoami deleted file mode 100644 index c4b00993ed..0000000000 Binary files a/packages/runtime-core/commands/whoami and /dev/null differ diff --git a/packages/runtime-core/commands/xargs b/packages/runtime-core/commands/xargs deleted file mode 100644 index 613a7bb3ea..0000000000 Binary files a/packages/runtime-core/commands/xargs and /dev/null differ diff --git a/packages/runtime-core/commands/xu b/packages/runtime-core/commands/xu deleted file mode 100644 index e37012c89f..0000000000 Binary files a/packages/runtime-core/commands/xu and /dev/null differ diff --git a/packages/runtime-core/commands/yes b/packages/runtime-core/commands/yes deleted file mode 100644 index 3a0ebb9b7a..0000000000 Binary files a/packages/runtime-core/commands/yes and /dev/null differ diff --git a/packages/runtime-core/commands/yq b/packages/runtime-core/commands/yq deleted file mode 100644 index 98a2de41ee..0000000000 Binary files a/packages/runtime-core/commands/yq and /dev/null differ diff --git a/packages/runtime-core/commands/zcat b/packages/runtime-core/commands/zcat deleted file mode 100644 index e60ebe5317..0000000000 Binary files a/packages/runtime-core/commands/zcat and /dev/null differ diff --git a/packages/runtime-core/commands/zip b/packages/runtime-core/commands/zip deleted file mode 100644 index 8324bdc9fe..0000000000 Binary files a/packages/runtime-core/commands/zip and /dev/null differ diff --git a/packages/runtime-core/scripts/copy-wasm-commands.mjs b/packages/runtime-core/scripts/copy-wasm-commands.mjs index 1a095fc10c..48d699b0bf 100644 --- a/packages/runtime-core/scripts/copy-wasm-commands.mjs +++ b/packages/runtime-core/scripts/copy-wasm-commands.mjs @@ -7,7 +7,7 @@ * The kernel needs a guest `sh` (plus coreutils) to spawn any process — without * these binaries `NodeRuntime.create()` cannot boot. The binaries are produced * by the in-repo Rust command build at - * `registry/native/target/wasm32-wasip1/release/commands/`. That path only + * `toolchain/target/wasm32-wasip1/release/commands/`. That path only * exists in a developer checkout, so we copy the whole command set (symlinks * included, the way `bash -> sh` and the stub aliases are laid out) into * `packages/core/commands/`, which is listed in `files` and resolved at runtime @@ -35,7 +35,7 @@ const REPO_ROOT = fileURLToPath(new URL("../../..", import.meta.url)); const SOURCE_DIR = path.join( REPO_ROOT, - "registry/native/target/wasm32-wasip1/release/commands", + "toolchain/target/wasm32-wasip1/release/commands", ); const DEST_DIR = path.join(PACKAGE_ROOT, "commands"); @@ -66,7 +66,7 @@ function main() { // never built without the commands. const message = `WASM commands not found at ${SOURCE_DIR} and none vendored at ${DEST_DIR}. ` + - "Build them with `make -C registry/native wasm` (or drop a prebuilt " + + "Build them with `make -C toolchain wasm` (or drop a prebuilt " + "commands artifact into the package) before packing so they ship in the tarball."; if (process.argv.includes("--require")) { console.error(`error: ${message}`); diff --git a/packages/runtime-core/src/kernel-proxy.ts b/packages/runtime-core/src/kernel-proxy.ts index 4c4f964bde..9f2c07f571 100644 --- a/packages/runtime-core/src/kernel-proxy.ts +++ b/packages/runtime-core/src/kernel-proxy.ts @@ -118,6 +118,15 @@ const PREFERRED_SIGNAL_NAMES = [ "SIGEMT", "SIGINFO", ] as const; +const NON_TERMINATING_SIGNALS = new Set([ + "0", + "SIGCHLD", + "SIGCONT", + "SIGSTOP", + "SIGURG", + "SIGWINCH", +]); +const LOCALLY_TERMINATING_SIGNALS = new Set(["SIGKILL", "SIGTERM"]); const NON_CANONICAL_SIGNAL_NAMES = new Set([ "SIGCLD", "SIGIOT", @@ -418,7 +427,10 @@ export class NativeSidecarKernelProxy { liveProcesses.map((entry) => this.signalProcess(entry, 15)), ); - await this.client.disposeVm(this.session, this.vm).catch(() => {}); + await Promise.race([ + this.client.disposeVm(this.session, this.vm), + new Promise((resolve) => setTimeout(resolve, 1000)), + ]).catch(() => {}); for (const entry of liveProcesses) { if (entry.exitCode === null) { // The sidecar dispose path already performs TERM/KILL escalation for any @@ -679,13 +691,19 @@ export class NativeSidecarKernelProxy { } entry.pendingKillSignal = signal; void entry.startPromise.then(async () => { - if (entry.exitCode !== null || entry.pendingKillSignal === null) { + if (entry.pendingKillSignal === null) { return; } const pendingSignal = entry.pendingKillSignal; entry.pendingKillSignal = null; await this.signalProcess(entry, pendingSignal); }); + if ( + (signal === 9 || signal === 15) && + entry.exitCode === null + ) { + this.finishProcess(entry, 128 + signal); + } }, wait: async () => { const exitCode = await this.waitForTrackedProcess(entry); @@ -1467,16 +1485,6 @@ export class NativeSidecarKernelProxy { return; } - if (entry.exitViaEvent) { - // The sidecar drains process output before it emits `process_exited`, - // the stdio frame stream is FIFO, and this pump dispatches events in - // order. Once the exit event reaches this entry, no trailing output can - // follow it; one zero-delay turn is cheap insurance for same-tick - // listener scheduling. - await drainTrailingProcessOutputTurn(0); - return; - } - let observedGeneration = entry.outputGeneration; let quietTurns = 0; let delayMs = 0; @@ -1681,14 +1689,37 @@ export class NativeSidecarKernelProxy { entry: TrackedProcessEntry, signal: number, ): Promise { - await this.signalRefreshes.get(entry.pid); + const sidecarSignal = toSidecarSignalName(signal); + let timedOut = false; + const killPromise = this.client.killProcess( + this.session, + this.vm, + entry.processId, + sidecarSignal, + ); try { - await this.client.killProcess( - this.session, - this.vm, - entry.processId, - toSidecarSignalName(signal), - ); + await Promise.race([ + killPromise, + new Promise((resolve) => + setTimeout(() => { + timedOut = true; + resolve(); + }, 1000), + ), + ]); + if (timedOut) { + void killPromise.catch(() => {}); + } + const hasGuestHandler = + this.signalStates.get(entry.pid)?.handlers.has(signal) ?? false; + if ( + entry.exitCode === null && + !hasGuestHandler && + LOCALLY_TERMINATING_SIGNALS.has(sidecarSignal) && + (entry.driver === "wasmvm" || timedOut) + ) { + this.finishProcess(entry, 128 + signal); + } } catch (error) { if (isNoSuchProcessError(error) || isUnknownVmError(error)) { return; @@ -2009,16 +2040,12 @@ export class NativeSidecarKernelProxy { return this.client.pread(this.session, this.vm, path, offset, length); }, pwrite: async (path, offset, data) => { - const bytes = - await this.createFilesystemView(includeLocalMounts).readFile(path); - const nextSize = Math.max(bytes.length, offset + data.length); - const updated = new Uint8Array(nextSize); - updated.set(bytes); - updated.set(data, offset); - await this.createFilesystemView(includeLocalMounts).writeFile( - path, - updated, - ); + const local = includeLocalMounts ? this.resolveLocalMount(path) : null; + if (local) { + this.assertLocalWritable(local.mount); + return local.mount.fs.pwrite(local.relativePath, offset, data); + } + return this.client.pwrite(this.session, this.vm, path, offset, data); }, }; } diff --git a/packages/runtime-core/src/node-runtime.ts b/packages/runtime-core/src/node-runtime.ts index 7bbd4a6822..832d740a6c 100644 --- a/packages/runtime-core/src/node-runtime.ts +++ b/packages/runtime-core/src/node-runtime.ts @@ -63,13 +63,13 @@ const REPO_ROOT = fileURLToPath(new URL("../../..", import.meta.url)); /** * In-repo build output for the WASM coreutils/shell command binaries, produced - * by the Rust command build (`make -C registry/native wasm`). Only present in a + * by the Rust command build (`make -C toolchain wasm`). Only present in a * developer checkout; preferred when it exists so local edits are picked up * without re-vendoring. */ const REPO_COMMANDS_DIR = path.join( REPO_ROOT, - "registry/native/target/wasm32-wasip1/release/commands", + "toolchain/target/wasm32-wasip1/release/commands", ); /** diff --git a/packages/runtime-core/src/sidecar-process.ts b/packages/runtime-core/src/sidecar-process.ts index 9a5e926945..cd60c8d635 100644 --- a/packages/runtime-core/src/sidecar-process.ts +++ b/packages/runtime-core/src/sidecar-process.ts @@ -196,6 +196,8 @@ export interface SidecarSpawnOptions { command?: string; args?: string[]; eventBufferCapacity?: number; + gracefulExitMs?: number; + forceExitMs?: number; // Migration-only compatibility path for pre-BARE test fixtures. payloadCodec?: NativeTransportPayloadCodec; /** @@ -368,8 +370,8 @@ export class SidecarProcess { silenceTimeoutMs: options.silenceTimeoutMs, eventBufferCapacity: options.eventBufferCapacity ?? DEFAULT_SIDECAR_EVENT_BUFFER_CAPACITY, - gracefulExitMs: DEFAULT_SIDECAR_GRACEFUL_EXIT_MS, - forceExitMs: DEFAULT_SIDECAR_FORCE_EXIT_MS, + gracefulExitMs: options.gracefulExitMs ?? DEFAULT_SIDECAR_GRACEFUL_EXIT_MS, + forceExitMs: options.forceExitMs ?? DEFAULT_SIDECAR_FORCE_EXIT_MS, disposedErrorMessage: "native sidecar disposed", payloadCodec: options.payloadCodec ?? "bare", }); @@ -890,6 +892,23 @@ export class SidecarProcess { }); } + async pwrite( + session: AuthenticatedSession, + vm: CreatedVm, + path: string, + offset: number, + content: Uint8Array, + ): Promise { + const encoded = encodeGuestFilesystemContent(content); + await this.guestFilesystemCall(session, vm, { + operation: "pwrite", + path, + offset, + content: encoded.content, + encoding: encoded.encoding, + }); + } + async mkdir( session: AuthenticatedSession, vm: CreatedVm, diff --git a/packages/runtime-core/src/test-runtime.ts b/packages/runtime-core/src/test-runtime.ts index 776bd8bb25..914d16a917 100644 --- a/packages/runtime-core/src/test-runtime.ts +++ b/packages/runtime-core/src/test-runtime.ts @@ -477,6 +477,7 @@ export interface Kernel extends KernelInterface { removeFile(path: string): Promise; removeDir(path: string): Promise; rename(oldPath: string, newPath: string): Promise; + pwrite(path: string, offset: number, data: Uint8Array): Promise; vmFetch(request: { port: number; method: string; @@ -1215,7 +1216,10 @@ export class NodeFileSystem implements VirtualFileSystem { offset: number, data: Uint8Array, ): Promise { - const handle = await fs.open(this.normalizeTarget(targetPath), "r+"); + const handle = await fs.open( + this.normalizeTarget(targetPath), + fsSync.constants.O_WRONLY, + ); try { await handle.write(data, 0, data.length, offset); } finally { @@ -1585,7 +1589,6 @@ export const WASMVM_COMMANDS = Object.freeze([ "unzip", "sqlite3", "curl", - "wget", "git", "git-remote-http", "git-remote-https", @@ -1754,7 +1757,6 @@ export const DEFAULT_FIRST_PARTY_TIERS: Readonly< tac: "read-only", tsort: "read-only", curl: "full", - wget: "full", sqlite3: "read-write", }); @@ -2984,6 +2986,15 @@ class NativeKernel implements Kernel { return this.proxy!.rename(oldPath, newPath); } + async pwrite( + targetPath: string, + offset: number, + data: Uint8Array, + ): Promise { + await this.ensureReady(); + return this.proxy!.vfs.pwrite(targetPath, offset, data); + } + private tryResolveMountedCommand(command: string): boolean { const normalized = normalizeCommandLookup(command); for (const driver of this.mountedRuntimeDrivers) { @@ -3071,6 +3082,8 @@ class NativeKernel implements Kernel { cwd: REPO_ROOT, command: ensureNativeSidecarBinary(), args: [], + gracefulExitMs: 100, + forceExitMs: 100, }), ); const session = await this.measureBoot("session_open", () => diff --git a/registry/tests/kernel/bridge-child-process.test.ts b/packages/runtime-core/tests/integration/bridge-child-process.test.ts similarity index 97% rename from registry/tests/kernel/bridge-child-process.test.ts rename to packages/runtime-core/tests/integration/bridge-child-process.test.ts index 7cf5032c62..930256fccf 100644 --- a/registry/tests/kernel/bridge-child-process.test.ts +++ b/packages/runtime-core/tests/integration/bridge-child-process.test.ts @@ -21,8 +21,8 @@ import { describeIf, createIntegrationKernel, NodeFileSystem, -} from './helpers.ts'; -import type { IntegrationKernelResult } from './helpers.ts'; +} from '@rivet-dev/agentos-vm-test-harness'; +import type { IntegrationKernelResult } from '@rivet-dev/agentos-vm-test-harness'; const __dirname = dirname(fileURLToPath(import.meta.url)); const PACKAGED_COREUTILS_COMMANDS_DIR = resolve( @@ -319,7 +319,17 @@ describeIf(!skipReason, 'bridge child_process → kernel routing', () => { fs.chmodSync('/tmp/write-only.txt', 0o200); // A real shell opens the append target write-only, so a 0o200 file is // appendable even though it cannot be read back until the chmod below. - execSync("printf changed >> /tmp/write-only.txt", { encoding: 'utf-8' }); + try { + execSync("printf changed >> /tmp/write-only.txt", { encoding: 'utf-8' }); + } catch (error) { + console.error(JSON.stringify({ + message: error instanceof Error ? error.message : String(error), + status: error && typeof error === 'object' && 'status' in error ? error.status : null, + stdout: error && typeof error === 'object' && 'stdout' in error ? String(error.stdout ?? '') : '', + stderr: error && typeof error === 'object' && 'stderr' in error ? String(error.stderr ?? '') : '' + })); + process.exit(99); + } fs.chmodSync('/tmp/write-only.txt', 0o600); console.log(JSON.stringify({ mode: 'loaded', diff --git a/registry/tests/kernel/cross-runtime-network.test.ts b/packages/runtime-core/tests/integration/cross-runtime-network.test.ts similarity index 92% rename from registry/tests/kernel/cross-runtime-network.test.ts rename to packages/runtime-core/tests/integration/cross-runtime-network.test.ts index 98519e2b24..9336b2320c 100644 --- a/registry/tests/kernel/cross-runtime-network.test.ts +++ b/packages/runtime-core/tests/integration/cross-runtime-network.test.ts @@ -16,10 +16,10 @@ import { createIntegrationKernel, itIf, skipUnlessWasmBuilt, -} from './helpers.ts'; -import type { IntegrationKernelResult, Kernel } from './helpers.ts'; +} from '@rivet-dev/agentos-vm-test-harness'; +import type { IntegrationKernelResult, Kernel } from '@rivet-dev/agentos-vm-test-harness'; -const WASM_HTTP_GET = resolve(C_BUILD_DIR, 'http_get'); +const WASM_CURL = resolve(C_BUILD_DIR, 'curl'); const WASM_HTTP_SERVER = resolve(C_BUILD_DIR, 'http_server'); const WASM_TCP_ECHO = resolve(C_BUILD_DIR, 'tcp_echo'); const WASM_TCP_SERVER = resolve(C_BUILD_DIR, 'tcp_server'); @@ -28,7 +28,7 @@ function skipReasonWasmNetwork(): string | false { const wasmSkipReason = skipUnlessWasmBuilt(); if (wasmSkipReason) return wasmSkipReason; for (const [name, path] of [ - ['http_get', WASM_HTTP_GET], + ['curl', WASM_CURL], ['http_server', WASM_HTTP_SERVER], ['tcp_echo', WASM_TCP_ECHO], ['tcp_server', WASM_TCP_SERVER], @@ -231,7 +231,7 @@ describe('cross-runtime network integration', { timeout: 90_000 }, () => { expect(client.stdout.trim()).toBe('js-pong:ping'); }); - itIf(!wasmNetworkSkipReason, 'W1 WASM http_get -> JS node:http server over VM loopback', async () => { + itIf(!wasmNetworkSkipReason, 'W1 WASM curl -> JS node:http server over VM loopback', async () => { ctx = await createIntegrationKernel({ runtimes: ['wasmvm', 'node'], commandDirs: [C_BUILD_DIR, COMMANDS_DIR], @@ -239,13 +239,13 @@ describe('cross-runtime network integration', { timeout: 90_000 }, () => { const server = spawnGuestNodeProgram(ctx.kernel, guestJsHttpServer(3102)); await waitForOutput(server, 'js http listening 3102', 'JS HTTP server'); - const wasm = await ctx.kernel.exec('http_get 3102 /from-wasm'); + const wasm = await ctx.kernel.exec('curl -fsS http://127.0.0.1:3102/from-wasm'); server.process.kill(15); await server.process.wait().catch(() => {}); expect(wasm.exitCode).toBe(0); - expect(wasm.stderr).not.toContain('socket error'); - expect(wasm.stdout).toContain('body: js:GET:/from-wasm'); + expect(wasm.stderr).toBe(''); + expect(wasm.stdout.trim()).toBe('js:GET:/from-wasm'); }); itIf(!wasmNetworkSkipReason, 'J3 JS fetch -> WASM HTTP server over VM loopback', async () => { @@ -341,7 +341,7 @@ describe('cross-runtime network integration', { timeout: 90_000 }, () => { expect(wasm.stdout).toContain('received: js-pong:hello'); }); - itIf(!wasmNetworkSkipReason, 'W3 WASM http_get -> WASM HTTP server over VM loopback', async () => { + itIf(!wasmNetworkSkipReason, 'W3 WASM curl -> WASM HTTP server over VM loopback', async () => { ctx = await createIntegrationKernel({ runtimes: ['wasmvm', 'node'], commandDirs: [C_BUILD_DIR, COMMANDS_DIR], @@ -349,12 +349,12 @@ describe('cross-runtime network integration', { timeout: 90_000 }, () => { const server = spawnGuestProgram(ctx.kernel, 'http_server', ['3108']); await waitForListener(ctx.kernel, 3108, 'WASM HTTP server'); - const wasm = await ctx.kernel.exec('http_get 3108 /from-wasm'); + const wasm = await ctx.kernel.exec('curl -fsS http://127.0.0.1:3108/from-wasm'); const serverExit = await server.process.wait(); expect(wasm.exitCode).toBe(0); - expect(wasm.stderr).not.toContain('socket error'); - expect(wasm.stdout).toContain('body: wasm:GET:/from-wasm'); + expect(wasm.stderr).toBe(''); + expect(wasm.stdout.trim()).toBe('wasm:GET:/from-wasm'); expect(serverExit).toBe(0); }); @@ -426,7 +426,7 @@ describe('cross-runtime network integration', { timeout: 90_000 }, () => { } }); - itIf(!wasmNetworkSkipReason, 'O2 WASM http_get -> host loopback requires loopback exemption', async () => { + itIf(!wasmNetworkSkipReason, 'O2 WASM curl -> host loopback requires loopback exemption', async () => { const seenRequests: string[] = []; const hostServer = createHttpServer((req, res) => { seenRequests.push(req.url ?? ''); @@ -443,9 +443,9 @@ describe('cross-runtime network integration', { timeout: 90_000 }, () => { runtimes: ['wasmvm', 'node'], commandDirs: [C_BUILD_DIR, COMMANDS_DIR], }); - const noExemption = await ctx.kernel.exec(`http_get ${port} /blocked`); + const noExemption = await ctx.kernel.exec(`curl -fsS http://127.0.0.1:${port}/blocked`); expect(noExemption.exitCode).not.toBe(0); - expect(noExemption.stderr).toMatch(/EACCES|Bad address|Connection refused|connect/); + expect(noExemption.stderr).toMatch(/EACCES|Bad address|Connection refused|connect|Failed to connect|Invalid argument/); expect(seenRequests).toEqual([]); await ctx.dispose(); @@ -454,10 +454,10 @@ describe('cross-runtime network integration', { timeout: 90_000 }, () => { commandDirs: [C_BUILD_DIR, COMMANDS_DIR], loopbackExemptPorts: [port], }); - const allowed = await ctx.kernel.exec(`http_get ${port} /allowed`); + const allowed = await ctx.kernel.exec(`curl -fsS http://127.0.0.1:${port}/allowed`); expect(allowed.exitCode).toBe(0); - expect(allowed.stderr).not.toContain('socket error'); - expect(allowed.stdout).toContain('body: host:/allowed'); + expect(allowed.stderr).toBe(''); + expect(allowed.stdout.trim()).toBe('host:/allowed'); expect(seenRequests).toEqual(['/allowed']); } finally { await new Promise((resolveClose) => hostServer.close(() => resolveClose())); diff --git a/registry/tests/kernel/cross-runtime-pipes.test.ts b/packages/runtime-core/tests/integration/cross-runtime-pipes.test.ts similarity index 95% rename from registry/tests/kernel/cross-runtime-pipes.test.ts rename to packages/runtime-core/tests/integration/cross-runtime-pipes.test.ts index 4db8328b22..8c19d7e198 100644 --- a/registry/tests/kernel/cross-runtime-pipes.test.ts +++ b/packages/runtime-core/tests/integration/cross-runtime-pipes.test.ts @@ -17,8 +17,8 @@ import { describeIf, createIntegrationKernel, skipUnlessWasmBuilt, -} from './helpers.ts'; -import type { Kernel } from './helpers.ts'; +} from '@rivet-dev/agentos-vm-test-harness'; +import type { Kernel } from '@rivet-dev/agentos-vm-test-harness'; // --------------------------------------------------------------------------- // Integration tests with real WasmVM + Node (skipped if WASM not built) diff --git a/registry/tests/kernel/cross-runtime-terminal.test.ts b/packages/runtime-core/tests/integration/cross-runtime-terminal.test.ts similarity index 98% rename from registry/tests/kernel/cross-runtime-terminal.test.ts rename to packages/runtime-core/tests/integration/cross-runtime-terminal.test.ts index 5a4f14b749..3fd5039d1f 100644 --- a/registry/tests/kernel/cross-runtime-terminal.test.ts +++ b/packages/runtime-core/tests/integration/cross-runtime-terminal.test.ts @@ -15,8 +15,8 @@ import { createIntegrationKernel, skipUnlessWasmBuilt, TerminalHarness, -} from './helpers.ts'; -import type { IntegrationKernelResult } from './helpers.ts'; +} from '@rivet-dev/agentos-vm-test-harness'; +import type { IntegrationKernelResult } from '@rivet-dev/agentos-vm-test-harness'; /** brush-shell interactive prompt. */ const PROMPT = 'sh-0.4$ '; diff --git a/registry/tests/kernel/ctrl-c-shell-behavior.test.ts b/packages/runtime-core/tests/integration/ctrl-c-shell-behavior.test.ts similarity index 95% rename from registry/tests/kernel/ctrl-c-shell-behavior.test.ts rename to packages/runtime-core/tests/integration/ctrl-c-shell-behavior.test.ts index 08af122ec2..3788f6fc1c 100644 --- a/registry/tests/kernel/ctrl-c-shell-behavior.test.ts +++ b/packages/runtime-core/tests/integration/ctrl-c-shell-behavior.test.ts @@ -16,8 +16,8 @@ import { createIntegrationKernel, skipUnlessWasmBuilt, TerminalHarness, -} from './helpers.ts'; -import type { IntegrationKernelResult } from './helpers.ts'; +} from '@rivet-dev/agentos-vm-test-harness'; +import type { IntegrationKernelResult } from '@rivet-dev/agentos-vm-test-harness'; const PROMPT = 'sh-0.4$ '; const wasmSkip = skipUnlessWasmBuilt(); diff --git a/registry/tests/kernel/dispose-behavior.test.ts b/packages/runtime-core/tests/integration/dispose-behavior.test.ts similarity index 92% rename from registry/tests/kernel/dispose-behavior.test.ts rename to packages/runtime-core/tests/integration/dispose-behavior.test.ts index 299b08e3fc..d77a5cbcd5 100644 --- a/registry/tests/kernel/dispose-behavior.test.ts +++ b/packages/runtime-core/tests/integration/dispose-behavior.test.ts @@ -17,9 +17,9 @@ import { createIntegrationKernel, skipUnlessWasmBuilt, createInMemoryFileSystem, -} from './helpers.ts'; -import type { Kernel } from './helpers.ts'; -import type { IntegrationKernelResult } from './helpers.ts'; +} from '@rivet-dev/agentos-vm-test-harness'; +import type { Kernel } from '@rivet-dev/agentos-vm-test-harness'; +import type { IntegrationKernelResult } from '@rivet-dev/agentos-vm-test-harness'; const skipReason = skipUnlessWasmBuilt(); diff --git a/registry/tests/wasmvm/dynamic-module-integration.test.ts b/packages/runtime-core/tests/integration/dynamic-module-integration.test.ts similarity index 98% rename from registry/tests/wasmvm/dynamic-module-integration.test.ts rename to packages/runtime-core/tests/integration/dynamic-module-integration.test.ts index a884adee9c..afc8bd00e9 100644 --- a/registry/tests/wasmvm/dynamic-module-integration.test.ts +++ b/packages/runtime-core/tests/integration/dynamic-module-integration.test.ts @@ -10,16 +10,16 @@ */ import { describe, it, expect, afterEach, vi } from 'vitest'; -import { createWasmVmRuntime, WASMVM_COMMANDS } from '../helpers.js'; -import type { WasmVmRuntimeOptions } from '../helpers.js'; -import { COMMANDS_DIR, createKernel, describeIf, hasWasmBinaries } from '../helpers.js'; +import { createWasmVmRuntime, WASMVM_COMMANDS } from '@rivet-dev/agentos-vm-test-harness'; +import type { WasmVmRuntimeOptions } from '@rivet-dev/agentos-vm-test-harness'; +import { COMMANDS_DIR, createKernel, describeIf, hasWasmBinaries } from '@rivet-dev/agentos-vm-test-harness'; import type { DriverProcess, Kernel, KernelInterface, KernelRuntimeDriver as RuntimeDriver, ProcessContext, -} from '../helpers.js'; +} from '@rivet-dev/agentos-vm-test-harness'; import { writeFile, mkdir, rm, symlink } from 'node:fs/promises'; import { existsSync } from 'node:fs'; import { join } from 'node:path'; diff --git a/registry/tests/kernel/e2e-concurrently.test.ts b/packages/runtime-core/tests/integration/e2e-concurrently.test.ts similarity index 99% rename from registry/tests/kernel/e2e-concurrently.test.ts rename to packages/runtime-core/tests/integration/e2e-concurrently.test.ts index 95ef7cf24d..bdc77ed515 100644 --- a/registry/tests/kernel/e2e-concurrently.test.ts +++ b/packages/runtime-core/tests/integration/e2e-concurrently.test.ts @@ -25,7 +25,7 @@ import { createWasmVmRuntime, createNodeRuntime, skipUnlessWasmBuilt, -} from './helpers.ts'; +} from '@rivet-dev/agentos-vm-test-harness'; const wasmSkip = skipUnlessWasmBuilt(); diff --git a/registry/tests/kernel/e2e-nextjs-build.test.ts b/packages/runtime-core/tests/integration/e2e-nextjs-build.test.ts similarity index 98% rename from registry/tests/kernel/e2e-nextjs-build.test.ts rename to packages/runtime-core/tests/integration/e2e-nextjs-build.test.ts index f5ade032a4..3ebf9e7192 100644 --- a/registry/tests/kernel/e2e-nextjs-build.test.ts +++ b/packages/runtime-core/tests/integration/e2e-nextjs-build.test.ts @@ -29,7 +29,7 @@ import { createWasmVmRuntime, createNodeRuntime, skipUnlessWasmBuilt, -} from './helpers.ts'; +} from '@rivet-dev/agentos-vm-test-harness'; const wasmSkip = skipUnlessWasmBuilt(); const __dirname = path.dirname(fileURLToPath(import.meta.url)); diff --git a/registry/tests/kernel/e2e-npm-install.test.ts b/packages/runtime-core/tests/integration/e2e-npm-install.test.ts similarity index 98% rename from registry/tests/kernel/e2e-npm-install.test.ts rename to packages/runtime-core/tests/integration/e2e-npm-install.test.ts index 42c508be53..d7cd270212 100644 --- a/registry/tests/kernel/e2e-npm-install.test.ts +++ b/packages/runtime-core/tests/integration/e2e-npm-install.test.ts @@ -22,7 +22,7 @@ import { createWasmVmRuntime, createNodeRuntime, skipUnlessWasmBuilt, -} from './helpers.ts'; +} from '@rivet-dev/agentos-vm-test-harness'; const wasmSkip = skipUnlessWasmBuilt(); diff --git a/registry/tests/kernel/e2e-npm-lifecycle.test.ts b/packages/runtime-core/tests/integration/e2e-npm-lifecycle.test.ts similarity index 99% rename from registry/tests/kernel/e2e-npm-lifecycle.test.ts rename to packages/runtime-core/tests/integration/e2e-npm-lifecycle.test.ts index 69f3f46e12..cf1b7b3ff0 100644 --- a/registry/tests/kernel/e2e-npm-lifecycle.test.ts +++ b/packages/runtime-core/tests/integration/e2e-npm-lifecycle.test.ts @@ -24,7 +24,7 @@ import { createWasmVmRuntime, createNodeRuntime, skipUnlessWasmBuilt, -} from './helpers.ts'; +} from '@rivet-dev/agentos-vm-test-harness'; const wasmSkip = skipUnlessWasmBuilt(); diff --git a/registry/tests/kernel/e2e-npm-scripts.test.ts b/packages/runtime-core/tests/integration/e2e-npm-scripts.test.ts similarity index 96% rename from registry/tests/kernel/e2e-npm-scripts.test.ts rename to packages/runtime-core/tests/integration/e2e-npm-scripts.test.ts index ff16dc5c0b..3950bf97a6 100644 --- a/registry/tests/kernel/e2e-npm-scripts.test.ts +++ b/packages/runtime-core/tests/integration/e2e-npm-scripts.test.ts @@ -9,7 +9,7 @@ */ import { describe, expect, it } from 'vitest'; -import { createIntegrationKernel, skipUnlessWasmBuilt } from './helpers.ts'; +import { createIntegrationKernel, skipUnlessWasmBuilt } from '@rivet-dev/agentos-vm-test-harness'; const skipReason = skipUnlessWasmBuilt(); void skipReason; diff --git a/registry/tests/kernel/e2e-npm-suite.test.ts b/packages/runtime-core/tests/integration/e2e-npm-suite.test.ts similarity index 99% rename from registry/tests/kernel/e2e-npm-suite.test.ts rename to packages/runtime-core/tests/integration/e2e-npm-suite.test.ts index 6d5faffe2a..276ce773a7 100644 --- a/registry/tests/kernel/e2e-npm-suite.test.ts +++ b/packages/runtime-core/tests/integration/e2e-npm-suite.test.ts @@ -26,7 +26,7 @@ import { createIntegrationKernel, NodeFileSystem, skipUnlessWasmBuilt, -} from './helpers.ts'; +} from '@rivet-dev/agentos-vm-test-harness'; const wasmSkip = skipUnlessWasmBuilt(); diff --git a/registry/tests/kernel/e2e-npm-version-init.test.ts b/packages/runtime-core/tests/integration/e2e-npm-version-init.test.ts similarity index 95% rename from registry/tests/kernel/e2e-npm-version-init.test.ts rename to packages/runtime-core/tests/integration/e2e-npm-version-init.test.ts index 3f529ae057..9debd4d285 100644 --- a/registry/tests/kernel/e2e-npm-version-init.test.ts +++ b/packages/runtime-core/tests/integration/e2e-npm-version-init.test.ts @@ -13,7 +13,7 @@ */ import { describe, expect, it } from 'vitest'; -import { createIntegrationKernel, skipUnlessWasmBuilt } from './helpers.ts'; +import { createIntegrationKernel, skipUnlessWasmBuilt } from '@rivet-dev/agentos-vm-test-harness'; const skipReason = skipUnlessWasmBuilt(); void skipReason; diff --git a/registry/tests/kernel/e2e-npx-and-pipes.test.ts b/packages/runtime-core/tests/integration/e2e-npx-and-pipes.test.ts similarity index 98% rename from registry/tests/kernel/e2e-npx-and-pipes.test.ts rename to packages/runtime-core/tests/integration/e2e-npx-and-pipes.test.ts index a4d49092ee..c6c3ba84ab 100644 --- a/registry/tests/kernel/e2e-npx-and-pipes.test.ts +++ b/packages/runtime-core/tests/integration/e2e-npx-and-pipes.test.ts @@ -11,7 +11,7 @@ import { createIntegrationKernel, itIf, skipUnlessWasmBuilt, -} from './helpers.ts'; +} from '@rivet-dev/agentos-vm-test-harness'; const skipReason = skipUnlessWasmBuilt(); void skipReason; diff --git a/registry/tests/kernel/e2e-project-matrix.test.ts b/packages/runtime-core/tests/integration/e2e-project-matrix.test.ts similarity index 99% rename from registry/tests/kernel/e2e-project-matrix.test.ts rename to packages/runtime-core/tests/integration/e2e-project-matrix.test.ts index cc9eaabb78..27f5297a20 100644 --- a/registry/tests/kernel/e2e-project-matrix.test.ts +++ b/packages/runtime-core/tests/integration/e2e-project-matrix.test.ts @@ -26,7 +26,7 @@ import { createWasmVmRuntime, createNodeRuntime, skipUnlessWasmBuilt, -} from './helpers.ts'; +} from '@rivet-dev/agentos-vm-test-harness'; const execFileAsync = promisify(execFile); const TEST_TIMEOUT_MS = 55_000; diff --git a/registry/tests/kernel/error-propagation.test.ts b/packages/runtime-core/tests/integration/error-propagation.test.ts similarity index 95% rename from registry/tests/kernel/error-propagation.test.ts rename to packages/runtime-core/tests/integration/error-propagation.test.ts index c71c3c580d..cdb233c52d 100644 --- a/registry/tests/kernel/error-propagation.test.ts +++ b/packages/runtime-core/tests/integration/error-propagation.test.ts @@ -13,8 +13,8 @@ import { describeIf, createIntegrationKernel, skipUnlessWasmBuilt, -} from './helpers.ts'; -import type { IntegrationKernelResult } from './helpers.ts'; +} from '@rivet-dev/agentos-vm-test-harness'; +import type { IntegrationKernelResult } from '@rivet-dev/agentos-vm-test-harness'; const skipReason = skipUnlessWasmBuilt(); diff --git a/registry/tests/kernel/exec-integration.test.ts b/packages/runtime-core/tests/integration/exec-integration.test.ts similarity index 97% rename from registry/tests/kernel/exec-integration.test.ts rename to packages/runtime-core/tests/integration/exec-integration.test.ts index a02216d71b..9cd0184869 100644 --- a/registry/tests/kernel/exec-integration.test.ts +++ b/packages/runtime-core/tests/integration/exec-integration.test.ts @@ -13,8 +13,8 @@ import { describeIf, createIntegrationKernel, skipUnlessWasmBuilt, -} from './helpers.ts'; -import type { IntegrationKernelResult } from './helpers.ts'; +} from '@rivet-dev/agentos-vm-test-harness'; +import type { IntegrationKernelResult } from '@rivet-dev/agentos-vm-test-harness'; const skipReason = skipUnlessWasmBuilt(); diff --git a/registry/tests/kernel/fd-inheritance.test.ts b/packages/runtime-core/tests/integration/fd-inheritance.test.ts similarity index 95% rename from registry/tests/kernel/fd-inheritance.test.ts rename to packages/runtime-core/tests/integration/fd-inheritance.test.ts index b99412009d..7892e5e75a 100644 --- a/registry/tests/kernel/fd-inheritance.test.ts +++ b/packages/runtime-core/tests/integration/fd-inheritance.test.ts @@ -12,8 +12,8 @@ import { describeIf, createIntegrationKernel, skipUnlessWasmBuilt, -} from './helpers.ts'; -import type { IntegrationKernelResult } from './helpers.ts'; +} from '@rivet-dev/agentos-vm-test-harness'; +import type { IntegrationKernelResult } from '@rivet-dev/agentos-vm-test-harness'; const skipReason = skipUnlessWasmBuilt(); diff --git a/registry/tests/kernel/module-resolution.test.ts b/packages/runtime-core/tests/integration/module-resolution.test.ts similarity index 96% rename from registry/tests/kernel/module-resolution.test.ts rename to packages/runtime-core/tests/integration/module-resolution.test.ts index 829f757144..424bbb5a69 100644 --- a/registry/tests/kernel/module-resolution.test.ts +++ b/packages/runtime-core/tests/integration/module-resolution.test.ts @@ -12,8 +12,8 @@ import { describeIf, createIntegrationKernel, skipUnlessWasmBuilt, -} from './helpers.ts'; -import type { IntegrationKernelResult } from './helpers.ts'; +} from '@rivet-dev/agentos-vm-test-harness'; +import type { IntegrationKernelResult } from '@rivet-dev/agentos-vm-test-harness'; const skipReason = skipUnlessWasmBuilt(); diff --git a/packages/runtime-core/tests/integration/net-server.test.ts b/packages/runtime-core/tests/integration/net-server.test.ts new file mode 100644 index 0000000000..b25935c237 --- /dev/null +++ b/packages/runtime-core/tests/integration/net-server.test.ts @@ -0,0 +1,137 @@ +/** + * Integration test for WasmVM TCP server sockets. + * + * Spawns the tcp_server C program as WASM and connects to it from a guest Node + * process over VM loopback, proving bytes cross the kernel socket table. + */ + +import { afterEach, describe, expect, it } from "vitest"; +import { existsSync } from "node:fs"; +import { resolve } from "node:path"; +import { + C_BUILD_DIR, + COMMANDS_DIR, + createIntegrationKernel, + describeIf, + skipUnlessWasmBuilt, +} from "@rivet-dev/agentos-vm-test-harness"; +import type { + IntegrationKernelResult, + Kernel, +} from "@rivet-dev/agentos-vm-test-harness"; + +const WASM_TCP_SERVER = resolve(C_BUILD_DIR, "tcp_server"); + +function skipReason(): string | false { + const wasmSkipReason = skipUnlessWasmBuilt(); + if (wasmSkipReason) return wasmSkipReason; + if (!existsSync(WASM_TCP_SERVER)) { + return `tcp_server WASM binary not found at ${WASM_TCP_SERVER}`; + } + return false; +} + +interface RunningGuestProgram { + process: ReturnType; + stdoutChunks: Uint8Array[]; + stderrChunks: Uint8Array[]; + getExitCode: () => number | null; +} + +function decodeChunks(chunks: Uint8Array[]): string { + return chunks.map((chunk) => new TextDecoder().decode(chunk)).join(""); +} + +function spawnGuestProgram( + kernel: Kernel, + command: string, + args: string[], +): RunningGuestProgram { + const stdoutChunks: Uint8Array[] = []; + const stderrChunks: Uint8Array[] = []; + let exitCode: number | null = null; + const process = kernel.spawn(command, args, { + onStdout: (chunk) => stdoutChunks.push(chunk), + onStderr: (chunk) => stderrChunks.push(chunk), + }); + void process.wait().then((code) => { + exitCode = code; + }); + return { + process, + stdoutChunks, + stderrChunks, + getExitCode: () => exitCode, + }; +} + +async function runGuestNodeProgram( + kernel: Kernel, + code: string, +): Promise<{ exitCode: number; stdout: string; stderr: string }> { + const program = spawnGuestProgram(kernel, "node", ["-e", code]); + const exitCode = await program.process.wait(); + return { + exitCode, + stdout: decodeChunks(program.stdoutChunks), + stderr: decodeChunks(program.stderrChunks), + }; +} + +async function waitForListener( + kernel: Kernel, + port: number, + label: string, +): Promise { + const deadline = Date.now() + 20_000; + while (Date.now() < deadline) { + if (kernel.socketTable.findListener({ host: "0.0.0.0", port })) { + return; + } + await new Promise((resolveWait) => setTimeout(resolveWait, 20)); + } + throw new Error(`Timed out waiting for ${label} listener on port ${port}`); +} + +const TEST_PORT = 9876; + +describeIf(!skipReason(), "WasmVM TCP server integration", { timeout: 30_000 }, () => { + let ctx: IntegrationKernelResult | undefined; + + afterEach(async () => { + await ctx?.dispose(); + ctx = undefined; + }); + + it("tcp_server: accept connection, recv data, send pong", async () => { + ctx = await createIntegrationKernel({ + runtimes: ["wasmvm", "node"], + commandDirs: [C_BUILD_DIR, COMMANDS_DIR], + }); + const server = spawnGuestProgram(ctx.kernel, "tcp_server", [ + String(TEST_PORT), + ]); + await waitForListener(ctx.kernel, TEST_PORT, "WASM TCP server"); + + const client = await runGuestNodeProgram( + ctx.kernel, + [ + "const net = require('net');", + `const client = net.connect({ host: '127.0.0.1', port: ${TEST_PORT} }, () => client.write('ping'));`, + "client.on('data', (chunk) => { console.log(chunk.toString()); client.end(); });", + "client.on('error', (error) => { console.error(error); process.exit(1); });", + ].join("\n"), + ); + const serverExit = await server.process.wait(); + + expect(client.exitCode).toBe(0); + expect(client.stderr).toBe(""); + expect(client.stdout.trim()).toBe("pong"); + expect(serverExit).toBe(0); + expect(decodeChunks(server.stdoutChunks)).toContain( + "listening on port 9876", + ); + expect(decodeChunks(server.stdoutChunks)).toContain("received: ping"); + expect(decodeChunks(server.stdoutChunks)).toContain("sent: 4"); + }); +}); diff --git a/packages/runtime-core/tests/integration/net-udp.test.ts b/packages/runtime-core/tests/integration/net-udp.test.ts new file mode 100644 index 0000000000..5a98e42b84 --- /dev/null +++ b/packages/runtime-core/tests/integration/net-udp.test.ts @@ -0,0 +1,161 @@ +/** + * Integration test for WasmVM UDP sockets. + * + * Spawns the udp_echo C program as WASM and sends datagrams from a guest Node + * process over VM loopback. + */ + +import { afterEach, describe, expect, it } from "vitest"; +import { existsSync } from "node:fs"; +import { resolve } from "node:path"; +import { + C_BUILD_DIR, + COMMANDS_DIR, + createIntegrationKernel, + describeIf, + skipUnlessWasmBuilt, +} from "@rivet-dev/agentos-vm-test-harness"; +import type { + IntegrationKernelResult, + Kernel, +} from "@rivet-dev/agentos-vm-test-harness"; + +const WASM_UDP_ECHO = resolve(C_BUILD_DIR, "udp_echo"); + +function skipReason(): string | false { + const wasmSkipReason = skipUnlessWasmBuilt(); + if (wasmSkipReason) return wasmSkipReason; + if (!existsSync(WASM_UDP_ECHO)) { + return `udp_echo WASM binary not found at ${WASM_UDP_ECHO}`; + } + return false; +} + +interface RunningGuestProgram { + process: ReturnType; + stdoutChunks: Uint8Array[]; + stderrChunks: Uint8Array[]; +} + +function decodeChunks(chunks: Uint8Array[]): string { + return chunks.map((chunk) => new TextDecoder().decode(chunk)).join(""); +} + +function spawnGuestProgram( + kernel: Kernel, + command: string, + args: string[], +): RunningGuestProgram { + const stdoutChunks: Uint8Array[] = []; + const stderrChunks: Uint8Array[] = []; + const process = kernel.spawn(command, args, { + onStdout: (chunk) => stdoutChunks.push(chunk), + onStderr: (chunk) => stderrChunks.push(chunk), + }); + return { process, stdoutChunks, stderrChunks }; +} + +async function runGuestNodeProgram( + kernel: Kernel, + code: string, +): Promise<{ exitCode: number; stdout: string; stderr: string }> { + const program = spawnGuestProgram(kernel, "node", ["-e", code]); + const exitCode = await program.process.wait(); + return { + exitCode, + stdout: decodeChunks(program.stdoutChunks), + stderr: decodeChunks(program.stderrChunks), + }; +} + +async function waitForUdpBinding(kernel: Kernel, port: number): Promise { + const deadline = Date.now() + 20_000; + while (Date.now() < deadline) { + if (kernel.socketTable.findBoundUdp({ host: "0.0.0.0", port })) { + return; + } + await new Promise((resolveWait) => setTimeout(resolveWait, 20)); + } + throw new Error(`Timed out waiting for UDP binding on port ${port}`); +} + +async function runUdpEchoCase( + kernel: Kernel, + port: number, + message: string, +): Promise<{ client: { exitCode: number; stdout: string; stderr: string }; server: RunningGuestProgram }> { + const server = spawnGuestProgram(kernel, "udp_echo", [String(port)]); + await waitForUdpBinding(kernel, port); + + const client = await runGuestNodeProgram( + kernel, + [ + "const dgram = require('dgram');", + "const client = dgram.createSocket('udp4');", + `const payload = Buffer.from(${JSON.stringify(message)});`, + "const timer = setTimeout(() => { console.error('timeout'); client.close(); process.exit(1); }, 5000);", + "client.on('message', (msg) => { clearTimeout(timer); console.log(msg.toString()); client.close(); });", + "client.on('error', (error) => { clearTimeout(timer); console.error(error); client.close(); process.exit(1); });", + `client.bind(0, '127.0.0.1', () => client.send(payload, ${port}, '127.0.0.1'));`, + ].join("\n"), + ); + + return { client, server }; +} + +const TEST_PORT = 9877; + +describeIf(!skipReason(), "WasmVM UDP integration", { timeout: 30_000 }, () => { + let ctx: IntegrationKernelResult | undefined; + + afterEach(async () => { + await ctx?.dispose(); + ctx = undefined; + }); + + it("udp_echo: recv datagram and echo it back", async () => { + ctx = await createIntegrationKernel({ + runtimes: ["wasmvm", "node"], + commandDirs: [C_BUILD_DIR, COMMANDS_DIR], + }); + + const { client, server } = await runUdpEchoCase( + ctx.kernel, + TEST_PORT, + "hello", + ); + const serverExit = await server.process.wait(); + + expect(client.exitCode).toBe(0); + expect(client.stderr).toBe(""); + expect(client.stdout.trim()).toBe("hello"); + expect(serverExit).toBe(0); + expect(decodeChunks(server.stdoutChunks)).toContain( + "listening on port 9877", + ); + expect(decodeChunks(server.stdoutChunks)).toContain("received: hello"); + expect(decodeChunks(server.stdoutChunks)).toContain("echoed: 5"); + }); + + it("udp_echo: message boundaries are preserved", async () => { + ctx = await createIntegrationKernel({ + runtimes: ["wasmvm", "node"], + commandDirs: [C_BUILD_DIR, COMMANDS_DIR], + }); + + const { client, server } = await runUdpEchoCase( + ctx.kernel, + TEST_PORT + 1, + "boundary-test-message", + ); + const serverExit = await server.process.wait(); + + expect(client.exitCode).toBe(0); + expect(client.stderr).toBe(""); + expect(client.stdout.trim()).toBe("boundary-test-message"); + expect(serverExit).toBe(0); + expect(decodeChunks(server.stdoutChunks)).toContain( + "received: boundary-test-message", + ); + }); +}); diff --git a/packages/runtime-core/tests/integration/net-unix.test.ts b/packages/runtime-core/tests/integration/net-unix.test.ts new file mode 100644 index 0000000000..b0de6e93b0 --- /dev/null +++ b/packages/runtime-core/tests/integration/net-unix.test.ts @@ -0,0 +1,125 @@ +/** + * Integration test for WasmVM Unix domain sockets. + * + * Spawns the unix_socket C program as WASM and connects to it from a guest Node + * process through an AF_UNIX path. + */ + +import { afterEach, describe, expect, it } from "vitest"; +import { existsSync } from "node:fs"; +import { resolve } from "node:path"; +import { + C_BUILD_DIR, + COMMANDS_DIR, + createIntegrationKernel, + describeIf, + skipUnlessWasmBuilt, +} from "@rivet-dev/agentos-vm-test-harness"; +import type { + IntegrationKernelResult, + Kernel, +} from "@rivet-dev/agentos-vm-test-harness"; + +const WASM_UNIX_SOCKET = resolve(C_BUILD_DIR, "unix_socket"); + +function skipReason(): string | false { + const wasmSkipReason = skipUnlessWasmBuilt(); + if (wasmSkipReason) return wasmSkipReason; + if (!existsSync(WASM_UNIX_SOCKET)) { + return `unix_socket WASM binary not found at ${WASM_UNIX_SOCKET}`; + } + return false; +} + +interface RunningGuestProgram { + process: ReturnType; + stdoutChunks: Uint8Array[]; + stderrChunks: Uint8Array[]; +} + +function decodeChunks(chunks: Uint8Array[]): string { + return chunks.map((chunk) => new TextDecoder().decode(chunk)).join(""); +} + +function spawnGuestProgram( + kernel: Kernel, + command: string, + args: string[], +): RunningGuestProgram { + const stdoutChunks: Uint8Array[] = []; + const stderrChunks: Uint8Array[] = []; + const process = kernel.spawn(command, args, { + onStdout: (chunk) => stdoutChunks.push(chunk), + onStderr: (chunk) => stderrChunks.push(chunk), + }); + return { process, stdoutChunks, stderrChunks }; +} + +async function runGuestNodeProgram( + kernel: Kernel, + code: string, +): Promise<{ exitCode: number; stdout: string; stderr: string }> { + const program = spawnGuestProgram(kernel, "node", ["-e", code]); + const exitCode = await program.process.wait(); + return { + exitCode, + stdout: decodeChunks(program.stdoutChunks), + stderr: decodeChunks(program.stderrChunks), + }; +} + +async function waitForUnixListener( + kernel: Kernel, + path: string, +): Promise { + const deadline = Date.now() + 20_000; + while (Date.now() < deadline) { + if (kernel.socketTable.findListener({ path })) { + return; + } + await new Promise((resolveWait) => setTimeout(resolveWait, 20)); + } + throw new Error(`Timed out waiting for Unix listener on ${path}`); +} + +const SOCK_PATH = "/tmp/test.sock"; + +describeIf(!skipReason(), "WasmVM Unix domain socket integration", { timeout: 30_000 }, () => { + let ctx: IntegrationKernelResult | undefined; + + afterEach(async () => { + await ctx?.dispose(); + ctx = undefined; + }); + + it("unix_socket: accept connection, recv data, send pong", async () => { + ctx = await createIntegrationKernel({ + runtimes: ["wasmvm", "node"], + commandDirs: [C_BUILD_DIR, COMMANDS_DIR], + }); + await ctx.kernel.mkdir("/tmp"); + const server = spawnGuestProgram(ctx.kernel, "unix_socket", [SOCK_PATH]); + await waitForUnixListener(ctx.kernel, SOCK_PATH); + + const client = await runGuestNodeProgram( + ctx.kernel, + [ + "const net = require('net');", + `const client = net.connect({ path: ${JSON.stringify(SOCK_PATH)} }, () => client.write('ping'));`, + "client.on('data', (chunk) => { console.log(chunk.toString()); client.end(); });", + "client.on('error', (error) => { console.error(error); process.exit(1); });", + ].join("\n"), + ); + const serverExit = await server.process.wait(); + + expect(client.exitCode).toBe(0); + expect(client.stderr).toBe(""); + expect(client.stdout.trim()).toBe("pong"); + expect(serverExit).toBe(0); + expect(decodeChunks(server.stdoutChunks)).toContain( + `listening on ${SOCK_PATH}`, + ); + expect(decodeChunks(server.stdoutChunks)).toContain("received: ping"); + expect(decodeChunks(server.stdoutChunks)).toContain("sent: 4"); + }); +}); diff --git a/registry/tests/kernel/node-binary-behavior.test.ts b/packages/runtime-core/tests/integration/node-binary-behavior.test.ts similarity index 99% rename from registry/tests/kernel/node-binary-behavior.test.ts rename to packages/runtime-core/tests/integration/node-binary-behavior.test.ts index 0f71bd648b..6464eb3f44 100644 --- a/registry/tests/kernel/node-binary-behavior.test.ts +++ b/packages/runtime-core/tests/integration/node-binary-behavior.test.ts @@ -26,8 +26,8 @@ import { createWasmVmRuntime, NodeFileSystem, COMMANDS_DIR, -} from './helpers.ts'; -import type { IntegrationKernelResult } from './helpers.ts'; +} from '@rivet-dev/agentos-vm-test-harness'; +import type { IntegrationKernelResult } from '@rivet-dev/agentos-vm-test-harness'; const skipReason = skipUnlessWasmBuilt(); diff --git a/registry/tests/projects/astro-pass/astro.config.mjs b/packages/runtime-core/tests/integration/projects/astro-pass/astro.config.mjs similarity index 100% rename from registry/tests/projects/astro-pass/astro.config.mjs rename to packages/runtime-core/tests/integration/projects/astro-pass/astro.config.mjs diff --git a/registry/tests/projects/astro-pass/fixture.json b/packages/runtime-core/tests/integration/projects/astro-pass/fixture.json similarity index 100% rename from registry/tests/projects/astro-pass/fixture.json rename to packages/runtime-core/tests/integration/projects/astro-pass/fixture.json diff --git a/registry/tests/projects/astro-pass/package.json b/packages/runtime-core/tests/integration/projects/astro-pass/package.json similarity index 100% rename from registry/tests/projects/astro-pass/package.json rename to packages/runtime-core/tests/integration/projects/astro-pass/package.json diff --git a/registry/tests/projects/astro-pass/src/components/Counter.jsx b/packages/runtime-core/tests/integration/projects/astro-pass/src/components/Counter.jsx similarity index 100% rename from registry/tests/projects/astro-pass/src/components/Counter.jsx rename to packages/runtime-core/tests/integration/projects/astro-pass/src/components/Counter.jsx diff --git a/registry/tests/projects/astro-pass/src/index.js b/packages/runtime-core/tests/integration/projects/astro-pass/src/index.js similarity index 100% rename from registry/tests/projects/astro-pass/src/index.js rename to packages/runtime-core/tests/integration/projects/astro-pass/src/index.js diff --git a/registry/tests/projects/astro-pass/src/pages/index.astro b/packages/runtime-core/tests/integration/projects/astro-pass/src/pages/index.astro similarity index 100% rename from registry/tests/projects/astro-pass/src/pages/index.astro rename to packages/runtime-core/tests/integration/projects/astro-pass/src/pages/index.astro diff --git a/registry/tests/projects/axios-pass/fixture.json b/packages/runtime-core/tests/integration/projects/axios-pass/fixture.json similarity index 100% rename from registry/tests/projects/axios-pass/fixture.json rename to packages/runtime-core/tests/integration/projects/axios-pass/fixture.json diff --git a/registry/tests/projects/axios-pass/package.json b/packages/runtime-core/tests/integration/projects/axios-pass/package.json similarity index 100% rename from registry/tests/projects/axios-pass/package.json rename to packages/runtime-core/tests/integration/projects/axios-pass/package.json diff --git a/registry/tests/projects/axios-pass/src/index.js b/packages/runtime-core/tests/integration/projects/axios-pass/src/index.js similarity index 100% rename from registry/tests/projects/axios-pass/src/index.js rename to packages/runtime-core/tests/integration/projects/axios-pass/src/index.js diff --git a/registry/tests/projects/bcryptjs-pass/fixture.json b/packages/runtime-core/tests/integration/projects/bcryptjs-pass/fixture.json similarity index 100% rename from registry/tests/projects/bcryptjs-pass/fixture.json rename to packages/runtime-core/tests/integration/projects/bcryptjs-pass/fixture.json diff --git a/registry/tests/projects/bcryptjs-pass/package.json b/packages/runtime-core/tests/integration/projects/bcryptjs-pass/package.json similarity index 100% rename from registry/tests/projects/bcryptjs-pass/package.json rename to packages/runtime-core/tests/integration/projects/bcryptjs-pass/package.json diff --git a/registry/tests/projects/bcryptjs-pass/pnpm-lock.yaml b/packages/runtime-core/tests/integration/projects/bcryptjs-pass/pnpm-lock.yaml similarity index 100% rename from registry/tests/projects/bcryptjs-pass/pnpm-lock.yaml rename to packages/runtime-core/tests/integration/projects/bcryptjs-pass/pnpm-lock.yaml diff --git a/registry/tests/projects/bcryptjs-pass/src/index.js b/packages/runtime-core/tests/integration/projects/bcryptjs-pass/src/index.js similarity index 100% rename from registry/tests/projects/bcryptjs-pass/src/index.js rename to packages/runtime-core/tests/integration/projects/bcryptjs-pass/src/index.js diff --git a/registry/tests/projects/bun-layout-pass/bun.lock b/packages/runtime-core/tests/integration/projects/bun-layout-pass/bun.lock similarity index 100% rename from registry/tests/projects/bun-layout-pass/bun.lock rename to packages/runtime-core/tests/integration/projects/bun-layout-pass/bun.lock diff --git a/registry/tests/projects/bun-layout-pass/fixture.json b/packages/runtime-core/tests/integration/projects/bun-layout-pass/fixture.json similarity index 100% rename from registry/tests/projects/bun-layout-pass/fixture.json rename to packages/runtime-core/tests/integration/projects/bun-layout-pass/fixture.json diff --git a/registry/tests/projects/bun-layout-pass/package.json b/packages/runtime-core/tests/integration/projects/bun-layout-pass/package.json similarity index 100% rename from registry/tests/projects/bun-layout-pass/package.json rename to packages/runtime-core/tests/integration/projects/bun-layout-pass/package.json diff --git a/registry/tests/projects/bun-layout-pass/src/index.js b/packages/runtime-core/tests/integration/projects/bun-layout-pass/src/index.js similarity index 100% rename from registry/tests/projects/bun-layout-pass/src/index.js rename to packages/runtime-core/tests/integration/projects/bun-layout-pass/src/index.js diff --git a/registry/tests/projects/chalk-pass/fixture.json b/packages/runtime-core/tests/integration/projects/chalk-pass/fixture.json similarity index 100% rename from registry/tests/projects/chalk-pass/fixture.json rename to packages/runtime-core/tests/integration/projects/chalk-pass/fixture.json diff --git a/registry/tests/projects/chalk-pass/package.json b/packages/runtime-core/tests/integration/projects/chalk-pass/package.json similarity index 100% rename from registry/tests/projects/chalk-pass/package.json rename to packages/runtime-core/tests/integration/projects/chalk-pass/package.json diff --git a/registry/tests/projects/chalk-pass/src/index.js b/packages/runtime-core/tests/integration/projects/chalk-pass/src/index.js similarity index 100% rename from registry/tests/projects/chalk-pass/src/index.js rename to packages/runtime-core/tests/integration/projects/chalk-pass/src/index.js diff --git a/registry/tests/projects/conditional-exports-pass/fixture.json b/packages/runtime-core/tests/integration/projects/conditional-exports-pass/fixture.json similarity index 100% rename from registry/tests/projects/conditional-exports-pass/fixture.json rename to packages/runtime-core/tests/integration/projects/conditional-exports-pass/fixture.json diff --git a/registry/tests/projects/conditional-exports-pass/package-lock.json b/packages/runtime-core/tests/integration/projects/conditional-exports-pass/package-lock.json similarity index 100% rename from registry/tests/projects/conditional-exports-pass/package-lock.json rename to packages/runtime-core/tests/integration/projects/conditional-exports-pass/package-lock.json diff --git a/registry/tests/projects/conditional-exports-pass/package.json b/packages/runtime-core/tests/integration/projects/conditional-exports-pass/package.json similarity index 100% rename from registry/tests/projects/conditional-exports-pass/package.json rename to packages/runtime-core/tests/integration/projects/conditional-exports-pass/package.json diff --git a/registry/tests/projects/conditional-exports-pass/packages/cond-exports-lib/lib/feature-cjs.js b/packages/runtime-core/tests/integration/projects/conditional-exports-pass/packages/cond-exports-lib/lib/feature-cjs.js similarity index 100% rename from registry/tests/projects/conditional-exports-pass/packages/cond-exports-lib/lib/feature-cjs.js rename to packages/runtime-core/tests/integration/projects/conditional-exports-pass/packages/cond-exports-lib/lib/feature-cjs.js diff --git a/registry/tests/projects/conditional-exports-pass/packages/cond-exports-lib/lib/feature-default.js b/packages/runtime-core/tests/integration/projects/conditional-exports-pass/packages/cond-exports-lib/lib/feature-default.js similarity index 100% rename from registry/tests/projects/conditional-exports-pass/packages/cond-exports-lib/lib/feature-default.js rename to packages/runtime-core/tests/integration/projects/conditional-exports-pass/packages/cond-exports-lib/lib/feature-default.js diff --git a/registry/tests/projects/conditional-exports-pass/packages/cond-exports-lib/lib/main-cjs.js b/packages/runtime-core/tests/integration/projects/conditional-exports-pass/packages/cond-exports-lib/lib/main-cjs.js similarity index 100% rename from registry/tests/projects/conditional-exports-pass/packages/cond-exports-lib/lib/main-cjs.js rename to packages/runtime-core/tests/integration/projects/conditional-exports-pass/packages/cond-exports-lib/lib/main-cjs.js diff --git a/registry/tests/projects/conditional-exports-pass/packages/cond-exports-lib/lib/main-default.js b/packages/runtime-core/tests/integration/projects/conditional-exports-pass/packages/cond-exports-lib/lib/main-default.js similarity index 100% rename from registry/tests/projects/conditional-exports-pass/packages/cond-exports-lib/lib/main-default.js rename to packages/runtime-core/tests/integration/projects/conditional-exports-pass/packages/cond-exports-lib/lib/main-default.js diff --git a/registry/tests/projects/conditional-exports-pass/packages/cond-exports-lib/package.json b/packages/runtime-core/tests/integration/projects/conditional-exports-pass/packages/cond-exports-lib/package.json similarity index 100% rename from registry/tests/projects/conditional-exports-pass/packages/cond-exports-lib/package.json rename to packages/runtime-core/tests/integration/projects/conditional-exports-pass/packages/cond-exports-lib/package.json diff --git a/registry/tests/projects/conditional-exports-pass/src/index.js b/packages/runtime-core/tests/integration/projects/conditional-exports-pass/src/index.js similarity index 100% rename from registry/tests/projects/conditional-exports-pass/src/index.js rename to packages/runtime-core/tests/integration/projects/conditional-exports-pass/src/index.js diff --git a/registry/tests/projects/crypto-random-pass/fixture.json b/packages/runtime-core/tests/integration/projects/crypto-random-pass/fixture.json similarity index 100% rename from registry/tests/projects/crypto-random-pass/fixture.json rename to packages/runtime-core/tests/integration/projects/crypto-random-pass/fixture.json diff --git a/registry/tests/projects/crypto-random-pass/package.json b/packages/runtime-core/tests/integration/projects/crypto-random-pass/package.json similarity index 100% rename from registry/tests/projects/crypto-random-pass/package.json rename to packages/runtime-core/tests/integration/projects/crypto-random-pass/package.json diff --git a/registry/tests/projects/crypto-random-pass/src/index.js b/packages/runtime-core/tests/integration/projects/crypto-random-pass/src/index.js similarity index 100% rename from registry/tests/projects/crypto-random-pass/src/index.js rename to packages/runtime-core/tests/integration/projects/crypto-random-pass/src/index.js diff --git a/registry/tests/projects/dotenv-pass/.env b/packages/runtime-core/tests/integration/projects/dotenv-pass/.env similarity index 100% rename from registry/tests/projects/dotenv-pass/.env rename to packages/runtime-core/tests/integration/projects/dotenv-pass/.env diff --git a/registry/tests/projects/dotenv-pass/fixture.json b/packages/runtime-core/tests/integration/projects/dotenv-pass/fixture.json similarity index 100% rename from registry/tests/projects/dotenv-pass/fixture.json rename to packages/runtime-core/tests/integration/projects/dotenv-pass/fixture.json diff --git a/registry/tests/projects/dotenv-pass/package.json b/packages/runtime-core/tests/integration/projects/dotenv-pass/package.json similarity index 100% rename from registry/tests/projects/dotenv-pass/package.json rename to packages/runtime-core/tests/integration/projects/dotenv-pass/package.json diff --git a/registry/tests/projects/dotenv-pass/src/index.js b/packages/runtime-core/tests/integration/projects/dotenv-pass/src/index.js similarity index 100% rename from registry/tests/projects/dotenv-pass/src/index.js rename to packages/runtime-core/tests/integration/projects/dotenv-pass/src/index.js diff --git a/registry/tests/projects/drizzle-pass/fixture.json b/packages/runtime-core/tests/integration/projects/drizzle-pass/fixture.json similarity index 100% rename from registry/tests/projects/drizzle-pass/fixture.json rename to packages/runtime-core/tests/integration/projects/drizzle-pass/fixture.json diff --git a/registry/tests/projects/drizzle-pass/package.json b/packages/runtime-core/tests/integration/projects/drizzle-pass/package.json similarity index 100% rename from registry/tests/projects/drizzle-pass/package.json rename to packages/runtime-core/tests/integration/projects/drizzle-pass/package.json diff --git a/registry/tests/projects/drizzle-pass/src/index.js b/packages/runtime-core/tests/integration/projects/drizzle-pass/src/index.js similarity index 100% rename from registry/tests/projects/drizzle-pass/src/index.js rename to packages/runtime-core/tests/integration/projects/drizzle-pass/src/index.js diff --git a/registry/tests/projects/esm-import-pass/fixture.json b/packages/runtime-core/tests/integration/projects/esm-import-pass/fixture.json similarity index 100% rename from registry/tests/projects/esm-import-pass/fixture.json rename to packages/runtime-core/tests/integration/projects/esm-import-pass/fixture.json diff --git a/registry/tests/projects/esm-import-pass/package.json b/packages/runtime-core/tests/integration/projects/esm-import-pass/package.json similarity index 100% rename from registry/tests/projects/esm-import-pass/package.json rename to packages/runtime-core/tests/integration/projects/esm-import-pass/package.json diff --git a/registry/tests/projects/esm-import-pass/src/index.js b/packages/runtime-core/tests/integration/projects/esm-import-pass/src/index.js similarity index 100% rename from registry/tests/projects/esm-import-pass/src/index.js rename to packages/runtime-core/tests/integration/projects/esm-import-pass/src/index.js diff --git a/registry/tests/projects/express-pass/fixture.json b/packages/runtime-core/tests/integration/projects/express-pass/fixture.json similarity index 100% rename from registry/tests/projects/express-pass/fixture.json rename to packages/runtime-core/tests/integration/projects/express-pass/fixture.json diff --git a/registry/tests/projects/express-pass/package.json b/packages/runtime-core/tests/integration/projects/express-pass/package.json similarity index 100% rename from registry/tests/projects/express-pass/package.json rename to packages/runtime-core/tests/integration/projects/express-pass/package.json diff --git a/registry/tests/projects/express-pass/pnpm-lock.yaml b/packages/runtime-core/tests/integration/projects/express-pass/pnpm-lock.yaml similarity index 100% rename from registry/tests/projects/express-pass/pnpm-lock.yaml rename to packages/runtime-core/tests/integration/projects/express-pass/pnpm-lock.yaml diff --git a/registry/tests/projects/express-pass/src/index.js b/packages/runtime-core/tests/integration/projects/express-pass/src/index.js similarity index 100% rename from registry/tests/projects/express-pass/src/index.js rename to packages/runtime-core/tests/integration/projects/express-pass/src/index.js diff --git a/registry/tests/projects/fastify-pass/fixture.json b/packages/runtime-core/tests/integration/projects/fastify-pass/fixture.json similarity index 100% rename from registry/tests/projects/fastify-pass/fixture.json rename to packages/runtime-core/tests/integration/projects/fastify-pass/fixture.json diff --git a/registry/tests/projects/fastify-pass/package.json b/packages/runtime-core/tests/integration/projects/fastify-pass/package.json similarity index 100% rename from registry/tests/projects/fastify-pass/package.json rename to packages/runtime-core/tests/integration/projects/fastify-pass/package.json diff --git a/registry/tests/projects/fastify-pass/pnpm-lock.yaml b/packages/runtime-core/tests/integration/projects/fastify-pass/pnpm-lock.yaml similarity index 100% rename from registry/tests/projects/fastify-pass/pnpm-lock.yaml rename to packages/runtime-core/tests/integration/projects/fastify-pass/pnpm-lock.yaml diff --git a/registry/tests/projects/fastify-pass/src/index.js b/packages/runtime-core/tests/integration/projects/fastify-pass/src/index.js similarity index 100% rename from registry/tests/projects/fastify-pass/src/index.js rename to packages/runtime-core/tests/integration/projects/fastify-pass/src/index.js diff --git a/registry/tests/projects/fs-metadata-rename-pass/fixture.json b/packages/runtime-core/tests/integration/projects/fs-metadata-rename-pass/fixture.json similarity index 100% rename from registry/tests/projects/fs-metadata-rename-pass/fixture.json rename to packages/runtime-core/tests/integration/projects/fs-metadata-rename-pass/fixture.json diff --git a/registry/tests/projects/fs-metadata-rename-pass/package.json b/packages/runtime-core/tests/integration/projects/fs-metadata-rename-pass/package.json similarity index 100% rename from registry/tests/projects/fs-metadata-rename-pass/package.json rename to packages/runtime-core/tests/integration/projects/fs-metadata-rename-pass/package.json diff --git a/registry/tests/projects/fs-metadata-rename-pass/src/index.js b/packages/runtime-core/tests/integration/projects/fs-metadata-rename-pass/src/index.js similarity index 100% rename from registry/tests/projects/fs-metadata-rename-pass/src/index.js rename to packages/runtime-core/tests/integration/projects/fs-metadata-rename-pass/src/index.js diff --git a/registry/tests/projects/ioredis-pass/fixture.json b/packages/runtime-core/tests/integration/projects/ioredis-pass/fixture.json similarity index 100% rename from registry/tests/projects/ioredis-pass/fixture.json rename to packages/runtime-core/tests/integration/projects/ioredis-pass/fixture.json diff --git a/registry/tests/projects/ioredis-pass/package.json b/packages/runtime-core/tests/integration/projects/ioredis-pass/package.json similarity index 100% rename from registry/tests/projects/ioredis-pass/package.json rename to packages/runtime-core/tests/integration/projects/ioredis-pass/package.json diff --git a/registry/tests/projects/ioredis-pass/pnpm-lock.yaml b/packages/runtime-core/tests/integration/projects/ioredis-pass/pnpm-lock.yaml similarity index 100% rename from registry/tests/projects/ioredis-pass/pnpm-lock.yaml rename to packages/runtime-core/tests/integration/projects/ioredis-pass/pnpm-lock.yaml diff --git a/registry/tests/projects/ioredis-pass/src/index.js b/packages/runtime-core/tests/integration/projects/ioredis-pass/src/index.js similarity index 100% rename from registry/tests/projects/ioredis-pass/src/index.js rename to packages/runtime-core/tests/integration/projects/ioredis-pass/src/index.js diff --git a/registry/tests/projects/jsonwebtoken-pass/fixture.json b/packages/runtime-core/tests/integration/projects/jsonwebtoken-pass/fixture.json similarity index 100% rename from registry/tests/projects/jsonwebtoken-pass/fixture.json rename to packages/runtime-core/tests/integration/projects/jsonwebtoken-pass/fixture.json diff --git a/registry/tests/projects/jsonwebtoken-pass/package.json b/packages/runtime-core/tests/integration/projects/jsonwebtoken-pass/package.json similarity index 100% rename from registry/tests/projects/jsonwebtoken-pass/package.json rename to packages/runtime-core/tests/integration/projects/jsonwebtoken-pass/package.json diff --git a/registry/tests/projects/jsonwebtoken-pass/src/index.js b/packages/runtime-core/tests/integration/projects/jsonwebtoken-pass/src/index.js similarity index 100% rename from registry/tests/projects/jsonwebtoken-pass/src/index.js rename to packages/runtime-core/tests/integration/projects/jsonwebtoken-pass/src/index.js diff --git a/registry/tests/projects/lodash-es-pass/fixture.json b/packages/runtime-core/tests/integration/projects/lodash-es-pass/fixture.json similarity index 100% rename from registry/tests/projects/lodash-es-pass/fixture.json rename to packages/runtime-core/tests/integration/projects/lodash-es-pass/fixture.json diff --git a/registry/tests/projects/lodash-es-pass/package.json b/packages/runtime-core/tests/integration/projects/lodash-es-pass/package.json similarity index 100% rename from registry/tests/projects/lodash-es-pass/package.json rename to packages/runtime-core/tests/integration/projects/lodash-es-pass/package.json diff --git a/registry/tests/projects/lodash-es-pass/pnpm-lock.yaml b/packages/runtime-core/tests/integration/projects/lodash-es-pass/pnpm-lock.yaml similarity index 100% rename from registry/tests/projects/lodash-es-pass/pnpm-lock.yaml rename to packages/runtime-core/tests/integration/projects/lodash-es-pass/pnpm-lock.yaml diff --git a/registry/tests/projects/lodash-es-pass/src/index.js b/packages/runtime-core/tests/integration/projects/lodash-es-pass/src/index.js similarity index 100% rename from registry/tests/projects/lodash-es-pass/src/index.js rename to packages/runtime-core/tests/integration/projects/lodash-es-pass/src/index.js diff --git a/registry/tests/projects/module-access-pass/fixture.json b/packages/runtime-core/tests/integration/projects/module-access-pass/fixture.json similarity index 100% rename from registry/tests/projects/module-access-pass/fixture.json rename to packages/runtime-core/tests/integration/projects/module-access-pass/fixture.json diff --git a/registry/tests/projects/module-access-pass/package.json b/packages/runtime-core/tests/integration/projects/module-access-pass/package.json similarity index 100% rename from registry/tests/projects/module-access-pass/package.json rename to packages/runtime-core/tests/integration/projects/module-access-pass/package.json diff --git a/registry/tests/projects/module-access-pass/src/index.js b/packages/runtime-core/tests/integration/projects/module-access-pass/src/index.js similarity index 100% rename from registry/tests/projects/module-access-pass/src/index.js rename to packages/runtime-core/tests/integration/projects/module-access-pass/src/index.js diff --git a/registry/tests/projects/module-access-pass/vendor/entry-lib/index.js b/packages/runtime-core/tests/integration/projects/module-access-pass/vendor/entry-lib/index.js similarity index 100% rename from registry/tests/projects/module-access-pass/vendor/entry-lib/index.js rename to packages/runtime-core/tests/integration/projects/module-access-pass/vendor/entry-lib/index.js diff --git a/registry/tests/projects/module-access-pass/vendor/entry-lib/package.json b/packages/runtime-core/tests/integration/projects/module-access-pass/vendor/entry-lib/package.json similarity index 100% rename from registry/tests/projects/module-access-pass/vendor/entry-lib/package.json rename to packages/runtime-core/tests/integration/projects/module-access-pass/vendor/entry-lib/package.json diff --git a/registry/tests/projects/module-access-pass/vendor/transitive-lib/index.js b/packages/runtime-core/tests/integration/projects/module-access-pass/vendor/transitive-lib/index.js similarity index 100% rename from registry/tests/projects/module-access-pass/vendor/transitive-lib/index.js rename to packages/runtime-core/tests/integration/projects/module-access-pass/vendor/transitive-lib/index.js diff --git a/registry/tests/projects/module-access-pass/vendor/transitive-lib/package.json b/packages/runtime-core/tests/integration/projects/module-access-pass/vendor/transitive-lib/package.json similarity index 100% rename from registry/tests/projects/module-access-pass/vendor/transitive-lib/package.json rename to packages/runtime-core/tests/integration/projects/module-access-pass/vendor/transitive-lib/package.json diff --git a/registry/tests/projects/mysql2-pass/fixture.json b/packages/runtime-core/tests/integration/projects/mysql2-pass/fixture.json similarity index 100% rename from registry/tests/projects/mysql2-pass/fixture.json rename to packages/runtime-core/tests/integration/projects/mysql2-pass/fixture.json diff --git a/registry/tests/projects/mysql2-pass/package.json b/packages/runtime-core/tests/integration/projects/mysql2-pass/package.json similarity index 100% rename from registry/tests/projects/mysql2-pass/package.json rename to packages/runtime-core/tests/integration/projects/mysql2-pass/package.json diff --git a/registry/tests/projects/mysql2-pass/src/index.js b/packages/runtime-core/tests/integration/projects/mysql2-pass/src/index.js similarity index 100% rename from registry/tests/projects/mysql2-pass/src/index.js rename to packages/runtime-core/tests/integration/projects/mysql2-pass/src/index.js diff --git a/registry/tests/projects/net-create-server-pass/fixture.json b/packages/runtime-core/tests/integration/projects/net-create-server-pass/fixture.json similarity index 100% rename from registry/tests/projects/net-create-server-pass/fixture.json rename to packages/runtime-core/tests/integration/projects/net-create-server-pass/fixture.json diff --git a/registry/tests/projects/net-create-server-pass/package.json b/packages/runtime-core/tests/integration/projects/net-create-server-pass/package.json similarity index 100% rename from registry/tests/projects/net-create-server-pass/package.json rename to packages/runtime-core/tests/integration/projects/net-create-server-pass/package.json diff --git a/registry/tests/projects/net-create-server-pass/src/index.js b/packages/runtime-core/tests/integration/projects/net-create-server-pass/src/index.js similarity index 100% rename from registry/tests/projects/net-create-server-pass/src/index.js rename to packages/runtime-core/tests/integration/projects/net-create-server-pass/src/index.js diff --git a/registry/tests/projects/nextjs-pass/.babelrc b/packages/runtime-core/tests/integration/projects/nextjs-pass/.babelrc similarity index 100% rename from registry/tests/projects/nextjs-pass/.babelrc rename to packages/runtime-core/tests/integration/projects/nextjs-pass/.babelrc diff --git a/registry/tests/projects/nextjs-pass/fixture.json b/packages/runtime-core/tests/integration/projects/nextjs-pass/fixture.json similarity index 100% rename from registry/tests/projects/nextjs-pass/fixture.json rename to packages/runtime-core/tests/integration/projects/nextjs-pass/fixture.json diff --git a/registry/tests/projects/nextjs-pass/next-wasm-shim.cjs b/packages/runtime-core/tests/integration/projects/nextjs-pass/next-wasm-shim.cjs similarity index 100% rename from registry/tests/projects/nextjs-pass/next-wasm-shim.cjs rename to packages/runtime-core/tests/integration/projects/nextjs-pass/next-wasm-shim.cjs diff --git a/registry/tests/projects/nextjs-pass/next.config.js b/packages/runtime-core/tests/integration/projects/nextjs-pass/next.config.js similarity index 100% rename from registry/tests/projects/nextjs-pass/next.config.js rename to packages/runtime-core/tests/integration/projects/nextjs-pass/next.config.js diff --git a/registry/tests/projects/nextjs-pass/package.json b/packages/runtime-core/tests/integration/projects/nextjs-pass/package.json similarity index 100% rename from registry/tests/projects/nextjs-pass/package.json rename to packages/runtime-core/tests/integration/projects/nextjs-pass/package.json diff --git a/registry/tests/projects/nextjs-pass/pages/api/hello.js b/packages/runtime-core/tests/integration/projects/nextjs-pass/pages/api/hello.js similarity index 100% rename from registry/tests/projects/nextjs-pass/pages/api/hello.js rename to packages/runtime-core/tests/integration/projects/nextjs-pass/pages/api/hello.js diff --git a/registry/tests/projects/nextjs-pass/pages/index.js b/packages/runtime-core/tests/integration/projects/nextjs-pass/pages/index.js similarity index 100% rename from registry/tests/projects/nextjs-pass/pages/index.js rename to packages/runtime-core/tests/integration/projects/nextjs-pass/pages/index.js diff --git a/registry/tests/projects/nextjs-pass/run-next-build.cjs b/packages/runtime-core/tests/integration/projects/nextjs-pass/run-next-build.cjs similarity index 100% rename from registry/tests/projects/nextjs-pass/run-next-build.cjs rename to packages/runtime-core/tests/integration/projects/nextjs-pass/run-next-build.cjs diff --git a/registry/tests/projects/nextjs-pass/src/index.js b/packages/runtime-core/tests/integration/projects/nextjs-pass/src/index.js similarity index 100% rename from registry/tests/projects/nextjs-pass/src/index.js rename to packages/runtime-core/tests/integration/projects/nextjs-pass/src/index.js diff --git a/registry/tests/projects/node-fetch-pass/fixture.json b/packages/runtime-core/tests/integration/projects/node-fetch-pass/fixture.json similarity index 100% rename from registry/tests/projects/node-fetch-pass/fixture.json rename to packages/runtime-core/tests/integration/projects/node-fetch-pass/fixture.json diff --git a/registry/tests/projects/node-fetch-pass/package.json b/packages/runtime-core/tests/integration/projects/node-fetch-pass/package.json similarity index 100% rename from registry/tests/projects/node-fetch-pass/package.json rename to packages/runtime-core/tests/integration/projects/node-fetch-pass/package.json diff --git a/registry/tests/projects/node-fetch-pass/src/index.js b/packages/runtime-core/tests/integration/projects/node-fetch-pass/src/index.js similarity index 100% rename from registry/tests/projects/node-fetch-pass/src/index.js rename to packages/runtime-core/tests/integration/projects/node-fetch-pass/src/index.js diff --git a/registry/tests/projects/npm-layout-pass/fixture.json b/packages/runtime-core/tests/integration/projects/npm-layout-pass/fixture.json similarity index 100% rename from registry/tests/projects/npm-layout-pass/fixture.json rename to packages/runtime-core/tests/integration/projects/npm-layout-pass/fixture.json diff --git a/registry/tests/projects/npm-layout-pass/package-lock.json b/packages/runtime-core/tests/integration/projects/npm-layout-pass/package-lock.json similarity index 100% rename from registry/tests/projects/npm-layout-pass/package-lock.json rename to packages/runtime-core/tests/integration/projects/npm-layout-pass/package-lock.json diff --git a/registry/tests/projects/npm-layout-pass/package.json b/packages/runtime-core/tests/integration/projects/npm-layout-pass/package.json similarity index 100% rename from registry/tests/projects/npm-layout-pass/package.json rename to packages/runtime-core/tests/integration/projects/npm-layout-pass/package.json diff --git a/registry/tests/projects/npm-layout-pass/src/index.js b/packages/runtime-core/tests/integration/projects/npm-layout-pass/src/index.js similarity index 100% rename from registry/tests/projects/npm-layout-pass/src/index.js rename to packages/runtime-core/tests/integration/projects/npm-layout-pass/src/index.js diff --git a/registry/tests/projects/optional-deps-pass/fixture.json b/packages/runtime-core/tests/integration/projects/optional-deps-pass/fixture.json similarity index 100% rename from registry/tests/projects/optional-deps-pass/fixture.json rename to packages/runtime-core/tests/integration/projects/optional-deps-pass/fixture.json diff --git a/registry/tests/projects/optional-deps-pass/package-lock.json b/packages/runtime-core/tests/integration/projects/optional-deps-pass/package-lock.json similarity index 100% rename from registry/tests/projects/optional-deps-pass/package-lock.json rename to packages/runtime-core/tests/integration/projects/optional-deps-pass/package-lock.json diff --git a/registry/tests/projects/optional-deps-pass/package.json b/packages/runtime-core/tests/integration/projects/optional-deps-pass/package.json similarity index 100% rename from registry/tests/projects/optional-deps-pass/package.json rename to packages/runtime-core/tests/integration/projects/optional-deps-pass/package.json diff --git a/registry/tests/projects/optional-deps-pass/src/index.js b/packages/runtime-core/tests/integration/projects/optional-deps-pass/src/index.js similarity index 100% rename from registry/tests/projects/optional-deps-pass/src/index.js rename to packages/runtime-core/tests/integration/projects/optional-deps-pass/src/index.js diff --git a/registry/tests/projects/peer-deps-pass/fixture.json b/packages/runtime-core/tests/integration/projects/peer-deps-pass/fixture.json similarity index 100% rename from registry/tests/projects/peer-deps-pass/fixture.json rename to packages/runtime-core/tests/integration/projects/peer-deps-pass/fixture.json diff --git a/registry/tests/projects/peer-deps-pass/package-lock.json b/packages/runtime-core/tests/integration/projects/peer-deps-pass/package-lock.json similarity index 100% rename from registry/tests/projects/peer-deps-pass/package-lock.json rename to packages/runtime-core/tests/integration/projects/peer-deps-pass/package-lock.json diff --git a/registry/tests/projects/peer-deps-pass/package.json b/packages/runtime-core/tests/integration/projects/peer-deps-pass/package.json similarity index 100% rename from registry/tests/projects/peer-deps-pass/package.json rename to packages/runtime-core/tests/integration/projects/peer-deps-pass/package.json diff --git a/registry/tests/projects/peer-deps-pass/packages/host/index.js b/packages/runtime-core/tests/integration/projects/peer-deps-pass/packages/host/index.js similarity index 100% rename from registry/tests/projects/peer-deps-pass/packages/host/index.js rename to packages/runtime-core/tests/integration/projects/peer-deps-pass/packages/host/index.js diff --git a/registry/tests/projects/peer-deps-pass/packages/host/package.json b/packages/runtime-core/tests/integration/projects/peer-deps-pass/packages/host/package.json similarity index 100% rename from registry/tests/projects/peer-deps-pass/packages/host/package.json rename to packages/runtime-core/tests/integration/projects/peer-deps-pass/packages/host/package.json diff --git a/registry/tests/projects/peer-deps-pass/packages/plugin/index.js b/packages/runtime-core/tests/integration/projects/peer-deps-pass/packages/plugin/index.js similarity index 100% rename from registry/tests/projects/peer-deps-pass/packages/plugin/index.js rename to packages/runtime-core/tests/integration/projects/peer-deps-pass/packages/plugin/index.js diff --git a/registry/tests/projects/peer-deps-pass/packages/plugin/package.json b/packages/runtime-core/tests/integration/projects/peer-deps-pass/packages/plugin/package.json similarity index 100% rename from registry/tests/projects/peer-deps-pass/packages/plugin/package.json rename to packages/runtime-core/tests/integration/projects/peer-deps-pass/packages/plugin/package.json diff --git a/registry/tests/projects/peer-deps-pass/src/index.js b/packages/runtime-core/tests/integration/projects/peer-deps-pass/src/index.js similarity index 100% rename from registry/tests/projects/peer-deps-pass/src/index.js rename to packages/runtime-core/tests/integration/projects/peer-deps-pass/src/index.js diff --git a/registry/tests/projects/pg-pass/fixture.json b/packages/runtime-core/tests/integration/projects/pg-pass/fixture.json similarity index 100% rename from registry/tests/projects/pg-pass/fixture.json rename to packages/runtime-core/tests/integration/projects/pg-pass/fixture.json diff --git a/registry/tests/projects/pg-pass/package.json b/packages/runtime-core/tests/integration/projects/pg-pass/package.json similarity index 100% rename from registry/tests/projects/pg-pass/package.json rename to packages/runtime-core/tests/integration/projects/pg-pass/package.json diff --git a/registry/tests/projects/pg-pass/pnpm-lock.yaml b/packages/runtime-core/tests/integration/projects/pg-pass/pnpm-lock.yaml similarity index 100% rename from registry/tests/projects/pg-pass/pnpm-lock.yaml rename to packages/runtime-core/tests/integration/projects/pg-pass/pnpm-lock.yaml diff --git a/registry/tests/projects/pg-pass/src/index.js b/packages/runtime-core/tests/integration/projects/pg-pass/src/index.js similarity index 100% rename from registry/tests/projects/pg-pass/src/index.js rename to packages/runtime-core/tests/integration/projects/pg-pass/src/index.js diff --git a/registry/tests/projects/pino-pass/fixture.json b/packages/runtime-core/tests/integration/projects/pino-pass/fixture.json similarity index 100% rename from registry/tests/projects/pino-pass/fixture.json rename to packages/runtime-core/tests/integration/projects/pino-pass/fixture.json diff --git a/registry/tests/projects/pino-pass/package.json b/packages/runtime-core/tests/integration/projects/pino-pass/package.json similarity index 100% rename from registry/tests/projects/pino-pass/package.json rename to packages/runtime-core/tests/integration/projects/pino-pass/package.json diff --git a/registry/tests/projects/pino-pass/pnpm-lock.yaml b/packages/runtime-core/tests/integration/projects/pino-pass/pnpm-lock.yaml similarity index 100% rename from registry/tests/projects/pino-pass/pnpm-lock.yaml rename to packages/runtime-core/tests/integration/projects/pino-pass/pnpm-lock.yaml diff --git a/registry/tests/projects/pino-pass/src/index.js b/packages/runtime-core/tests/integration/projects/pino-pass/src/index.js similarity index 100% rename from registry/tests/projects/pino-pass/src/index.js rename to packages/runtime-core/tests/integration/projects/pino-pass/src/index.js diff --git a/registry/tests/projects/pnpm-layout-pass/fixture.json b/packages/runtime-core/tests/integration/projects/pnpm-layout-pass/fixture.json similarity index 100% rename from registry/tests/projects/pnpm-layout-pass/fixture.json rename to packages/runtime-core/tests/integration/projects/pnpm-layout-pass/fixture.json diff --git a/registry/tests/projects/pnpm-layout-pass/package.json b/packages/runtime-core/tests/integration/projects/pnpm-layout-pass/package.json similarity index 100% rename from registry/tests/projects/pnpm-layout-pass/package.json rename to packages/runtime-core/tests/integration/projects/pnpm-layout-pass/package.json diff --git a/registry/tests/projects/pnpm-layout-pass/pnpm-lock.yaml b/packages/runtime-core/tests/integration/projects/pnpm-layout-pass/pnpm-lock.yaml similarity index 100% rename from registry/tests/projects/pnpm-layout-pass/pnpm-lock.yaml rename to packages/runtime-core/tests/integration/projects/pnpm-layout-pass/pnpm-lock.yaml diff --git a/registry/tests/projects/pnpm-layout-pass/src/index.js b/packages/runtime-core/tests/integration/projects/pnpm-layout-pass/src/index.js similarity index 100% rename from registry/tests/projects/pnpm-layout-pass/src/index.js rename to packages/runtime-core/tests/integration/projects/pnpm-layout-pass/src/index.js diff --git a/registry/tests/projects/rivetkit/fixture.json b/packages/runtime-core/tests/integration/projects/rivetkit/fixture.json similarity index 100% rename from registry/tests/projects/rivetkit/fixture.json rename to packages/runtime-core/tests/integration/projects/rivetkit/fixture.json diff --git a/registry/tests/projects/rivetkit/package.json b/packages/runtime-core/tests/integration/projects/rivetkit/package.json similarity index 100% rename from registry/tests/projects/rivetkit/package.json rename to packages/runtime-core/tests/integration/projects/rivetkit/package.json diff --git a/registry/tests/projects/rivetkit/src/index.js b/packages/runtime-core/tests/integration/projects/rivetkit/src/index.js similarity index 100% rename from registry/tests/projects/rivetkit/src/index.js rename to packages/runtime-core/tests/integration/projects/rivetkit/src/index.js diff --git a/registry/tests/projects/rivetkit/vendor/rivetkit/package.json b/packages/runtime-core/tests/integration/projects/rivetkit/vendor/rivetkit/package.json similarity index 100% rename from registry/tests/projects/rivetkit/vendor/rivetkit/package.json rename to packages/runtime-core/tests/integration/projects/rivetkit/vendor/rivetkit/package.json diff --git a/registry/tests/projects/semver-pass/fixture.json b/packages/runtime-core/tests/integration/projects/semver-pass/fixture.json similarity index 100% rename from registry/tests/projects/semver-pass/fixture.json rename to packages/runtime-core/tests/integration/projects/semver-pass/fixture.json diff --git a/registry/tests/projects/semver-pass/package.json b/packages/runtime-core/tests/integration/projects/semver-pass/package.json similarity index 100% rename from registry/tests/projects/semver-pass/package.json rename to packages/runtime-core/tests/integration/projects/semver-pass/package.json diff --git a/registry/tests/projects/semver-pass/src/index.js b/packages/runtime-core/tests/integration/projects/semver-pass/src/index.js similarity index 100% rename from registry/tests/projects/semver-pass/src/index.js rename to packages/runtime-core/tests/integration/projects/semver-pass/src/index.js diff --git a/registry/tests/projects/sse-streaming-pass/fixture.json b/packages/runtime-core/tests/integration/projects/sse-streaming-pass/fixture.json similarity index 100% rename from registry/tests/projects/sse-streaming-pass/fixture.json rename to packages/runtime-core/tests/integration/projects/sse-streaming-pass/fixture.json diff --git a/registry/tests/projects/sse-streaming-pass/package.json b/packages/runtime-core/tests/integration/projects/sse-streaming-pass/package.json similarity index 100% rename from registry/tests/projects/sse-streaming-pass/package.json rename to packages/runtime-core/tests/integration/projects/sse-streaming-pass/package.json diff --git a/registry/tests/projects/sse-streaming-pass/src/index.js b/packages/runtime-core/tests/integration/projects/sse-streaming-pass/src/index.js similarity index 100% rename from registry/tests/projects/sse-streaming-pass/src/index.js rename to packages/runtime-core/tests/integration/projects/sse-streaming-pass/src/index.js diff --git a/registry/tests/projects/ssh2-pass/fixture.json b/packages/runtime-core/tests/integration/projects/ssh2-pass/fixture.json similarity index 100% rename from registry/tests/projects/ssh2-pass/fixture.json rename to packages/runtime-core/tests/integration/projects/ssh2-pass/fixture.json diff --git a/registry/tests/projects/ssh2-pass/package.json b/packages/runtime-core/tests/integration/projects/ssh2-pass/package.json similarity index 100% rename from registry/tests/projects/ssh2-pass/package.json rename to packages/runtime-core/tests/integration/projects/ssh2-pass/package.json diff --git a/registry/tests/projects/ssh2-pass/src/index.js b/packages/runtime-core/tests/integration/projects/ssh2-pass/src/index.js similarity index 100% rename from registry/tests/projects/ssh2-pass/src/index.js rename to packages/runtime-core/tests/integration/projects/ssh2-pass/src/index.js diff --git a/registry/tests/projects/ssh2-sftp-client-pass/fixture.json b/packages/runtime-core/tests/integration/projects/ssh2-sftp-client-pass/fixture.json similarity index 100% rename from registry/tests/projects/ssh2-sftp-client-pass/fixture.json rename to packages/runtime-core/tests/integration/projects/ssh2-sftp-client-pass/fixture.json diff --git a/registry/tests/projects/ssh2-sftp-client-pass/package.json b/packages/runtime-core/tests/integration/projects/ssh2-sftp-client-pass/package.json similarity index 100% rename from registry/tests/projects/ssh2-sftp-client-pass/package.json rename to packages/runtime-core/tests/integration/projects/ssh2-sftp-client-pass/package.json diff --git a/registry/tests/projects/ssh2-sftp-client-pass/src/index.js b/packages/runtime-core/tests/integration/projects/ssh2-sftp-client-pass/src/index.js similarity index 100% rename from registry/tests/projects/ssh2-sftp-client-pass/src/index.js rename to packages/runtime-core/tests/integration/projects/ssh2-sftp-client-pass/src/index.js diff --git a/registry/tests/projects/transitive-deps-pass/fixture.json b/packages/runtime-core/tests/integration/projects/transitive-deps-pass/fixture.json similarity index 100% rename from registry/tests/projects/transitive-deps-pass/fixture.json rename to packages/runtime-core/tests/integration/projects/transitive-deps-pass/fixture.json diff --git a/registry/tests/projects/transitive-deps-pass/package-lock.json b/packages/runtime-core/tests/integration/projects/transitive-deps-pass/package-lock.json similarity index 100% rename from registry/tests/projects/transitive-deps-pass/package-lock.json rename to packages/runtime-core/tests/integration/projects/transitive-deps-pass/package-lock.json diff --git a/registry/tests/projects/transitive-deps-pass/package.json b/packages/runtime-core/tests/integration/projects/transitive-deps-pass/package.json similarity index 100% rename from registry/tests/projects/transitive-deps-pass/package.json rename to packages/runtime-core/tests/integration/projects/transitive-deps-pass/package.json diff --git a/registry/tests/projects/transitive-deps-pass/packages/level-a/index.js b/packages/runtime-core/tests/integration/projects/transitive-deps-pass/packages/level-a/index.js similarity index 100% rename from registry/tests/projects/transitive-deps-pass/packages/level-a/index.js rename to packages/runtime-core/tests/integration/projects/transitive-deps-pass/packages/level-a/index.js diff --git a/registry/tests/projects/transitive-deps-pass/packages/level-a/package.json b/packages/runtime-core/tests/integration/projects/transitive-deps-pass/packages/level-a/package.json similarity index 100% rename from registry/tests/projects/transitive-deps-pass/packages/level-a/package.json rename to packages/runtime-core/tests/integration/projects/transitive-deps-pass/packages/level-a/package.json diff --git a/registry/tests/projects/transitive-deps-pass/packages/level-b/index.js b/packages/runtime-core/tests/integration/projects/transitive-deps-pass/packages/level-b/index.js similarity index 100% rename from registry/tests/projects/transitive-deps-pass/packages/level-b/index.js rename to packages/runtime-core/tests/integration/projects/transitive-deps-pass/packages/level-b/index.js diff --git a/registry/tests/projects/transitive-deps-pass/packages/level-b/package.json b/packages/runtime-core/tests/integration/projects/transitive-deps-pass/packages/level-b/package.json similarity index 100% rename from registry/tests/projects/transitive-deps-pass/packages/level-b/package.json rename to packages/runtime-core/tests/integration/projects/transitive-deps-pass/packages/level-b/package.json diff --git a/registry/tests/projects/transitive-deps-pass/packages/level-c/index.js b/packages/runtime-core/tests/integration/projects/transitive-deps-pass/packages/level-c/index.js similarity index 100% rename from registry/tests/projects/transitive-deps-pass/packages/level-c/index.js rename to packages/runtime-core/tests/integration/projects/transitive-deps-pass/packages/level-c/index.js diff --git a/registry/tests/projects/transitive-deps-pass/packages/level-c/package.json b/packages/runtime-core/tests/integration/projects/transitive-deps-pass/packages/level-c/package.json similarity index 100% rename from registry/tests/projects/transitive-deps-pass/packages/level-c/package.json rename to packages/runtime-core/tests/integration/projects/transitive-deps-pass/packages/level-c/package.json diff --git a/registry/tests/projects/transitive-deps-pass/src/index.js b/packages/runtime-core/tests/integration/projects/transitive-deps-pass/src/index.js similarity index 100% rename from registry/tests/projects/transitive-deps-pass/src/index.js rename to packages/runtime-core/tests/integration/projects/transitive-deps-pass/src/index.js diff --git a/registry/tests/projects/uuid-pass/fixture.json b/packages/runtime-core/tests/integration/projects/uuid-pass/fixture.json similarity index 100% rename from registry/tests/projects/uuid-pass/fixture.json rename to packages/runtime-core/tests/integration/projects/uuid-pass/fixture.json diff --git a/registry/tests/projects/uuid-pass/package.json b/packages/runtime-core/tests/integration/projects/uuid-pass/package.json similarity index 100% rename from registry/tests/projects/uuid-pass/package.json rename to packages/runtime-core/tests/integration/projects/uuid-pass/package.json diff --git a/registry/tests/projects/uuid-pass/src/index.js b/packages/runtime-core/tests/integration/projects/uuid-pass/src/index.js similarity index 100% rename from registry/tests/projects/uuid-pass/src/index.js rename to packages/runtime-core/tests/integration/projects/uuid-pass/src/index.js diff --git a/registry/tests/projects/vite-pass/app/main.jsx b/packages/runtime-core/tests/integration/projects/vite-pass/app/main.jsx similarity index 100% rename from registry/tests/projects/vite-pass/app/main.jsx rename to packages/runtime-core/tests/integration/projects/vite-pass/app/main.jsx diff --git a/registry/tests/projects/vite-pass/fixture.json b/packages/runtime-core/tests/integration/projects/vite-pass/fixture.json similarity index 100% rename from registry/tests/projects/vite-pass/fixture.json rename to packages/runtime-core/tests/integration/projects/vite-pass/fixture.json diff --git a/registry/tests/projects/vite-pass/index.html b/packages/runtime-core/tests/integration/projects/vite-pass/index.html similarity index 100% rename from registry/tests/projects/vite-pass/index.html rename to packages/runtime-core/tests/integration/projects/vite-pass/index.html diff --git a/registry/tests/projects/vite-pass/package.json b/packages/runtime-core/tests/integration/projects/vite-pass/package.json similarity index 100% rename from registry/tests/projects/vite-pass/package.json rename to packages/runtime-core/tests/integration/projects/vite-pass/package.json diff --git a/registry/tests/projects/vite-pass/src/index.js b/packages/runtime-core/tests/integration/projects/vite-pass/src/index.js similarity index 100% rename from registry/tests/projects/vite-pass/src/index.js rename to packages/runtime-core/tests/integration/projects/vite-pass/src/index.js diff --git a/registry/tests/projects/vite-pass/vite.config.mjs b/packages/runtime-core/tests/integration/projects/vite-pass/vite.config.mjs similarity index 100% rename from registry/tests/projects/vite-pass/vite.config.mjs rename to packages/runtime-core/tests/integration/projects/vite-pass/vite.config.mjs diff --git a/registry/tests/projects/workspace-layout-pass/fixture.json b/packages/runtime-core/tests/integration/projects/workspace-layout-pass/fixture.json similarity index 100% rename from registry/tests/projects/workspace-layout-pass/fixture.json rename to packages/runtime-core/tests/integration/projects/workspace-layout-pass/fixture.json diff --git a/registry/tests/projects/workspace-layout-pass/package.json b/packages/runtime-core/tests/integration/projects/workspace-layout-pass/package.json similarity index 100% rename from registry/tests/projects/workspace-layout-pass/package.json rename to packages/runtime-core/tests/integration/projects/workspace-layout-pass/package.json diff --git a/registry/tests/projects/workspace-layout-pass/packages/app/package.json b/packages/runtime-core/tests/integration/projects/workspace-layout-pass/packages/app/package.json similarity index 100% rename from registry/tests/projects/workspace-layout-pass/packages/app/package.json rename to packages/runtime-core/tests/integration/projects/workspace-layout-pass/packages/app/package.json diff --git a/registry/tests/projects/workspace-layout-pass/packages/app/src/index.js b/packages/runtime-core/tests/integration/projects/workspace-layout-pass/packages/app/src/index.js similarity index 100% rename from registry/tests/projects/workspace-layout-pass/packages/app/src/index.js rename to packages/runtime-core/tests/integration/projects/workspace-layout-pass/packages/app/src/index.js diff --git a/registry/tests/projects/workspace-layout-pass/packages/lib/package.json b/packages/runtime-core/tests/integration/projects/workspace-layout-pass/packages/lib/package.json similarity index 100% rename from registry/tests/projects/workspace-layout-pass/packages/lib/package.json rename to packages/runtime-core/tests/integration/projects/workspace-layout-pass/packages/lib/package.json diff --git a/registry/tests/projects/workspace-layout-pass/packages/lib/src/index.js b/packages/runtime-core/tests/integration/projects/workspace-layout-pass/packages/lib/src/index.js similarity index 100% rename from registry/tests/projects/workspace-layout-pass/packages/lib/src/index.js rename to packages/runtime-core/tests/integration/projects/workspace-layout-pass/packages/lib/src/index.js diff --git a/registry/tests/projects/ws-pass/fixture.json b/packages/runtime-core/tests/integration/projects/ws-pass/fixture.json similarity index 100% rename from registry/tests/projects/ws-pass/fixture.json rename to packages/runtime-core/tests/integration/projects/ws-pass/fixture.json diff --git a/registry/tests/projects/ws-pass/package-lock.json b/packages/runtime-core/tests/integration/projects/ws-pass/package-lock.json similarity index 100% rename from registry/tests/projects/ws-pass/package-lock.json rename to packages/runtime-core/tests/integration/projects/ws-pass/package-lock.json diff --git a/registry/tests/projects/ws-pass/package.json b/packages/runtime-core/tests/integration/projects/ws-pass/package.json similarity index 100% rename from registry/tests/projects/ws-pass/package.json rename to packages/runtime-core/tests/integration/projects/ws-pass/package.json diff --git a/registry/tests/projects/ws-pass/src/index.js b/packages/runtime-core/tests/integration/projects/ws-pass/src/index.js similarity index 100% rename from registry/tests/projects/ws-pass/src/index.js rename to packages/runtime-core/tests/integration/projects/ws-pass/src/index.js diff --git a/registry/tests/projects/yaml-pass/fixture.json b/packages/runtime-core/tests/integration/projects/yaml-pass/fixture.json similarity index 100% rename from registry/tests/projects/yaml-pass/fixture.json rename to packages/runtime-core/tests/integration/projects/yaml-pass/fixture.json diff --git a/registry/tests/projects/yaml-pass/package.json b/packages/runtime-core/tests/integration/projects/yaml-pass/package.json similarity index 100% rename from registry/tests/projects/yaml-pass/package.json rename to packages/runtime-core/tests/integration/projects/yaml-pass/package.json diff --git a/registry/tests/projects/yaml-pass/src/index.js b/packages/runtime-core/tests/integration/projects/yaml-pass/src/index.js similarity index 100% rename from registry/tests/projects/yaml-pass/src/index.js rename to packages/runtime-core/tests/integration/projects/yaml-pass/src/index.js diff --git a/registry/tests/projects/yarn-berry-layout-pass/.yarnrc.yml b/packages/runtime-core/tests/integration/projects/yarn-berry-layout-pass/.yarnrc.yml similarity index 100% rename from registry/tests/projects/yarn-berry-layout-pass/.yarnrc.yml rename to packages/runtime-core/tests/integration/projects/yarn-berry-layout-pass/.yarnrc.yml diff --git a/registry/tests/projects/yarn-berry-layout-pass/fixture.json b/packages/runtime-core/tests/integration/projects/yarn-berry-layout-pass/fixture.json similarity index 100% rename from registry/tests/projects/yarn-berry-layout-pass/fixture.json rename to packages/runtime-core/tests/integration/projects/yarn-berry-layout-pass/fixture.json diff --git a/registry/tests/projects/yarn-berry-layout-pass/package.json b/packages/runtime-core/tests/integration/projects/yarn-berry-layout-pass/package.json similarity index 100% rename from registry/tests/projects/yarn-berry-layout-pass/package.json rename to packages/runtime-core/tests/integration/projects/yarn-berry-layout-pass/package.json diff --git a/registry/tests/projects/yarn-berry-layout-pass/src/index.js b/packages/runtime-core/tests/integration/projects/yarn-berry-layout-pass/src/index.js similarity index 100% rename from registry/tests/projects/yarn-berry-layout-pass/src/index.js rename to packages/runtime-core/tests/integration/projects/yarn-berry-layout-pass/src/index.js diff --git a/registry/tests/projects/yarn-berry-layout-pass/yarn.lock b/packages/runtime-core/tests/integration/projects/yarn-berry-layout-pass/yarn.lock similarity index 100% rename from registry/tests/projects/yarn-berry-layout-pass/yarn.lock rename to packages/runtime-core/tests/integration/projects/yarn-berry-layout-pass/yarn.lock diff --git a/registry/tests/projects/yarn-classic-layout-pass/fixture.json b/packages/runtime-core/tests/integration/projects/yarn-classic-layout-pass/fixture.json similarity index 100% rename from registry/tests/projects/yarn-classic-layout-pass/fixture.json rename to packages/runtime-core/tests/integration/projects/yarn-classic-layout-pass/fixture.json diff --git a/registry/tests/projects/yarn-classic-layout-pass/package.json b/packages/runtime-core/tests/integration/projects/yarn-classic-layout-pass/package.json similarity index 100% rename from registry/tests/projects/yarn-classic-layout-pass/package.json rename to packages/runtime-core/tests/integration/projects/yarn-classic-layout-pass/package.json diff --git a/registry/tests/projects/yarn-classic-layout-pass/src/index.js b/packages/runtime-core/tests/integration/projects/yarn-classic-layout-pass/src/index.js similarity index 100% rename from registry/tests/projects/yarn-classic-layout-pass/src/index.js rename to packages/runtime-core/tests/integration/projects/yarn-classic-layout-pass/src/index.js diff --git a/registry/tests/projects/yarn-classic-layout-pass/yarn.lock b/packages/runtime-core/tests/integration/projects/yarn-classic-layout-pass/yarn.lock similarity index 100% rename from registry/tests/projects/yarn-classic-layout-pass/yarn.lock rename to packages/runtime-core/tests/integration/projects/yarn-classic-layout-pass/yarn.lock diff --git a/registry/tests/projects/zod-pass/fixture.json b/packages/runtime-core/tests/integration/projects/zod-pass/fixture.json similarity index 100% rename from registry/tests/projects/zod-pass/fixture.json rename to packages/runtime-core/tests/integration/projects/zod-pass/fixture.json diff --git a/registry/tests/projects/zod-pass/package.json b/packages/runtime-core/tests/integration/projects/zod-pass/package.json similarity index 100% rename from registry/tests/projects/zod-pass/package.json rename to packages/runtime-core/tests/integration/projects/zod-pass/package.json diff --git a/registry/tests/projects/zod-pass/pnpm-lock.yaml b/packages/runtime-core/tests/integration/projects/zod-pass/pnpm-lock.yaml similarity index 100% rename from registry/tests/projects/zod-pass/pnpm-lock.yaml rename to packages/runtime-core/tests/integration/projects/zod-pass/pnpm-lock.yaml diff --git a/registry/tests/projects/zod-pass/src/index.js b/packages/runtime-core/tests/integration/projects/zod-pass/src/index.js similarity index 100% rename from registry/tests/projects/zod-pass/src/index.js rename to packages/runtime-core/tests/integration/projects/zod-pass/src/index.js diff --git a/registry/tests/kernel/repro-npm-install.ts b/packages/runtime-core/tests/integration/repro-npm-install.ts similarity index 93% rename from registry/tests/kernel/repro-npm-install.ts rename to packages/runtime-core/tests/integration/repro-npm-install.ts index 03358f9204..7e24de262e 100644 --- a/registry/tests/kernel/repro-npm-install.ts +++ b/packages/runtime-core/tests/integration/repro-npm-install.ts @@ -1,7 +1,7 @@ import { mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import path from 'node:path'; -import { COMMANDS_DIR, createKernel, NodeFileSystem, createWasmVmRuntime, createNodeRuntime } from '../helpers.ts'; +import { COMMANDS_DIR, createKernel, NodeFileSystem, createWasmVmRuntime, createNodeRuntime } from '@rivet-dev/agentos-vm-test-harness'; const tempDir = await mkdtemp(path.join(tmpdir(), 'kernel-npm-install-repro-')); console.log('tempDir', tempDir); diff --git a/registry/tests/kernel/shim-streaming.test.ts b/packages/runtime-core/tests/integration/shim-streaming.test.ts similarity index 86% rename from registry/tests/kernel/shim-streaming.test.ts rename to packages/runtime-core/tests/integration/shim-streaming.test.ts index aad4e4de59..634f9729c7 100644 --- a/registry/tests/kernel/shim-streaming.test.ts +++ b/packages/runtime-core/tests/integration/shim-streaming.test.ts @@ -3,8 +3,8 @@ import { createIntegrationKernel, describeIf, skipUnlessWasmBuilt, -} from './helpers.ts'; -import type { IntegrationKernelResult } from './helpers.ts'; +} from '@rivet-dev/agentos-vm-test-harness'; +import type { IntegrationKernelResult } from '@rivet-dev/agentos-vm-test-harness'; const skipReason = skipUnlessWasmBuilt(); diff --git a/registry/tests/kernel/signal-forwarding.test.ts b/packages/runtime-core/tests/integration/signal-forwarding.test.ts similarity index 96% rename from registry/tests/kernel/signal-forwarding.test.ts rename to packages/runtime-core/tests/integration/signal-forwarding.test.ts index 117b079dab..ac085e0818 100644 --- a/registry/tests/kernel/signal-forwarding.test.ts +++ b/packages/runtime-core/tests/integration/signal-forwarding.test.ts @@ -11,8 +11,8 @@ import { describeIf, createIntegrationKernel, skipUnlessWasmBuilt, -} from './helpers.ts'; -import type { IntegrationKernelResult } from './helpers.ts'; +} from '@rivet-dev/agentos-vm-test-harness'; +import type { IntegrationKernelResult } from '@rivet-dev/agentos-vm-test-harness'; const skipReason = skipUnlessWasmBuilt(); diff --git a/packages/runtime-core/tests/integration/signal-handler.test.ts b/packages/runtime-core/tests/integration/signal-handler.test.ts new file mode 100644 index 0000000000..7a7a8f8d70 --- /dev/null +++ b/packages/runtime-core/tests/integration/signal-handler.test.ts @@ -0,0 +1,215 @@ +/** + * Integration test for WasmVM cooperative signal handling. + * + * Spawns the signal_handler C program as WASM (sigaction(SIGINT, ...) → + * busy-loop with sleep → verify handler called), delivers SIGINT via + * kernel.kill(), and verifies the handler fires at a syscall boundary. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { createWasmVmRuntime } from '@rivet-dev/agentos-vm-test-harness'; +import { + COMMANDS_DIR, + C_BUILD_DIR, + createKernel, + describeIf, + hasWasmBinaries, + SIGTERM, +} from '@rivet-dev/agentos-vm-test-harness'; +import type { Kernel } from '@rivet-dev/agentos-vm-test-harness'; +import { existsSync } from 'node:fs'; +import { join } from 'node:path'; + +const hasCWasmBinaries = existsSync(join(C_BUILD_DIR, 'signal_handler')); +const SIGINT = 2; + +function skipReason(): string | false { + if (!hasWasmBinaries) return 'WASM binaries not built (run make wasm in native/wasmvm/)'; + if (!hasCWasmBinaries) return 'signal_handler WASM binary not built (run make -C native/wasmvm/c sysroot && make -C native/wasmvm/c programs)'; + return false; +} + +// Minimal in-memory VFS +class SimpleVFS { + private files = new Map(); + private dirs = new Set(['/']); + private symlinks = new Map(); + + async readFile(path: string): Promise { + const data = this.files.get(path); + if (!data) throw new Error(`ENOENT: ${path}`); + return data; + } + async readTextFile(path: string): Promise { + return new TextDecoder().decode(await this.readFile(path)); + } + async pread(path: string, offset: number, length: number): Promise { + const data = await this.readFile(path); + return data.slice(offset, offset + length); + } + async readDir(path: string): Promise { + const prefix = path === '/' ? '/' : path + '/'; + const entries: string[] = []; + for (const p of [...this.files.keys(), ...this.dirs]) { + if (p !== path && p.startsWith(prefix)) { + const rest = p.slice(prefix.length); + if (!rest.includes('/')) entries.push(rest); + } + } + return entries; + } + async readDirWithTypes(path: string) { + return (await this.readDir(path)).map((name) => ({ + name, + isDirectory: this.dirs.has(path === '/' ? `/${name}` : `${path}/${name}`), + })); + } + async writeFile(path: string, content: string | Uint8Array): Promise { + const data = typeof content === 'string' ? new TextEncoder().encode(content) : content; + this.files.set(path, new Uint8Array(data)); + const parts = path.split('/').filter(Boolean); + for (let i = 1; i < parts.length; i++) { + this.dirs.add('/' + parts.slice(0, i).join('/')); + } + } + async createDir(path: string) { this.dirs.add(path); } + async mkdir(path: string, _options?: { recursive?: boolean }) { this.dirs.add(path); } + async exists(path: string): Promise { + return this.files.has(path) || this.dirs.has(path) || this.symlinks.has(path); + } + async stat(path: string) { + const isDir = this.dirs.has(path); + const isSymlink = this.symlinks.has(path); + const data = this.files.get(path); + if (!isDir && !isSymlink && !data) throw new Error(`ENOENT: ${path}`); + return { + mode: isSymlink ? 0o120777 : (isDir ? 0o40755 : 0o100644), + size: data?.length ?? 0, + isDirectory: isDir, + isSymbolicLink: isSymlink, + atimeMs: Date.now(), + mtimeMs: Date.now(), + ctimeMs: Date.now(), + birthtimeMs: Date.now(), + ino: 0, + nlink: 1, + uid: 1000, + gid: 1000, + }; + } + lstat(path: string) { return this.stat(path); } + async chmod() {} + async rename(from: string, to: string) { + const data = this.files.get(from); + if (data) { this.files.set(to, data); this.files.delete(from); } + } + async unlink(path: string) { this.files.delete(path); this.symlinks.delete(path); } + async rmdir(path: string) { this.dirs.delete(path); } + async symlink(target: string, linkPath: string) { + this.symlinks.set(linkPath, target); + const parts = linkPath.split('/').filter(Boolean); + for (let i = 1; i < parts.length; i++) { + this.dirs.add('/' + parts.slice(0, i).join('/')); + } + } + async readlink(path: string): Promise { + const target = this.symlinks.get(path); + if (!target) throw new Error(`EINVAL: ${path}`); + return target; + } +} + +async function waitForSignalRegistration( + kernel: Kernel, + pid: number, + signal: number, +): Promise<{ mask: Set; flags: number }> { + const proxy = (kernel as any).proxy; + const entry = proxy?.trackedProcesses?.get(pid); + let lastDirectKeys: number[] = []; + const deadline = Date.now() + 10_000; + while (Date.now() < deadline) { + const cached = kernel.processTable.getSignalState(pid).handlers.get(signal); + if (cached) { + return cached as { mask: Set; flags: number }; + } + + if (proxy && entry) { + const snapshot = await proxy.client.getSignalState(proxy.session, proxy.vm, entry.processId); + lastDirectKeys = [...snapshot.handlers.keys()]; + const direct = snapshot?.handlers?.get(signal); + if (direct) { + return { + mask: new Set(direct.mask), + flags: direct.flags, + }; + } + } + + await new Promise((r) => setTimeout(r, 20)); + } + throw new Error( + `timed out waiting for signal ${signal} registration for pid ${pid}; processId=${entry?.processId ?? 'unknown'}; sidecarSignals=${JSON.stringify(lastDirectKeys)}`, + ); +} + +async function waitForOutput( + getOutput: () => string, + needle: string, + timeoutMs = 2_000, +): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline && !getOutput().includes(needle)) { + await new Promise((r) => setTimeout(r, 20)); + } +} + +describeIf(!skipReason(), 'WasmVM signal handler integration', { timeout: 30_000 }, () => { + let kernel: Kernel; + let vfs: SimpleVFS; + + beforeEach(async () => { + vfs = new SimpleVFS(); + kernel = createKernel({ filesystem: vfs as any }); + await kernel.mount(createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] })); + }); + + afterEach(async () => { + await kernel?.dispose(); + }); + + it('signal_handler: sigaction registration preserves mask and fires at syscall boundary', async () => { + // Spawn the WASM signal_handler program (registers SIGINT handler, then loops) + let stdout = ''; + let stderr = ''; + const proc = kernel.spawn('signal_handler', [], { + onStdout: (data) => { stdout += new TextDecoder().decode(data); }, + onStderr: (data) => { stderr += new TextDecoder().decode(data); }, + }); + + // Wait for the program to register its handler and start waiting + const deadline = Date.now() + 10_000; + while (Date.now() < deadline && !stdout.includes('waiting')) { + await new Promise((r) => setTimeout(r, 20)); + } + expect(stdout).toContain('handler_registered'); + expect(stdout).toContain('waiting'); + + const registration = await waitForSignalRegistration(kernel, proc.pid, SIGINT).catch((error) => { + throw new Error(`${error instanceof Error ? error.message : String(error)}; stderr=${JSON.stringify(stderr)}`); + }); + expect(registration?.mask).toEqual(new Set([SIGTERM])); + + // Deliver SIGINT via ManagedProcess.kill() — routes through kernel process table + proc.kill(SIGINT); + + // Wait for the program to handle the signal and exit + const exitCode = await proc.wait(); + await waitForOutput(() => stdout, 'caught_signal=2'); + + if (!stdout.includes('caught_signal=2')) { + throw new Error(`missing caught signal output; exitCode=${exitCode}; stdout=${JSON.stringify(stdout)}; stderr=${JSON.stringify(stderr)}`); + } + expect(exitCode).toBe(0); + }); +}); diff --git a/registry/tests/kernel/vfs-consistency.test.ts b/packages/runtime-core/tests/integration/vfs-consistency.test.ts similarity index 96% rename from registry/tests/kernel/vfs-consistency.test.ts rename to packages/runtime-core/tests/integration/vfs-consistency.test.ts index c4a1fa04a6..19939a6ff8 100644 --- a/registry/tests/kernel/vfs-consistency.test.ts +++ b/packages/runtime-core/tests/integration/vfs-consistency.test.ts @@ -12,8 +12,8 @@ import { describeIf, createIntegrationKernel, skipUnlessWasmBuilt, -} from './helpers.ts'; -import type { IntegrationKernelResult } from './helpers.ts'; +} from '@rivet-dev/agentos-vm-test-harness'; +import type { IntegrationKernelResult } from '@rivet-dev/agentos-vm-test-harness'; const skipReason = skipUnlessWasmBuilt(); diff --git a/registry/tests/wasmvm/wasi-http.test.ts b/packages/runtime-core/tests/integration/wasi-http.test.ts similarity index 98% rename from registry/tests/wasmvm/wasi-http.test.ts rename to packages/runtime-core/tests/integration/wasi-http.test.ts index 06f6cec3cf..1ae71fc27b 100644 --- a/registry/tests/wasmvm/wasi-http.test.ts +++ b/packages/runtime-core/tests/integration/wasi-http.test.ts @@ -12,9 +12,9 @@ */ import { describe, it, expect, afterEach, beforeAll, afterAll } from 'vitest'; -import { createWasmVmRuntime } from '../helpers.js'; -import { COMMANDS_DIR, createKernel, describeIf, hasWasmBinaries } from '../helpers.js'; -import type { Kernel } from '../helpers.js'; +import { createWasmVmRuntime } from '@rivet-dev/agentos-vm-test-harness'; +import { COMMANDS_DIR, createKernel, describeIf, hasWasmBinaries } from '@rivet-dev/agentos-vm-test-harness'; +import type { Kernel } from '@rivet-dev/agentos-vm-test-harness'; import { createServer as createHttpServer, type Server, type IncomingMessage, type ServerResponse } from 'node:http'; import { createServer as createHttpsServer, type Server as HttpsServer } from 'node:https'; import { execSync } from 'node:child_process'; diff --git a/registry/tests/wasmvm/wasi-spawn.test.ts b/packages/runtime-core/tests/integration/wasi-spawn.test.ts similarity index 96% rename from registry/tests/wasmvm/wasi-spawn.test.ts rename to packages/runtime-core/tests/integration/wasi-spawn.test.ts index 5f74ef1eae..910364314b 100644 --- a/registry/tests/wasmvm/wasi-spawn.test.ts +++ b/packages/runtime-core/tests/integration/wasi-spawn.test.ts @@ -9,9 +9,9 @@ */ import { describe, it, expect, beforeEach, afterEach } from 'vitest'; -import { createWasmVmRuntime } from '../helpers.js'; -import { COMMANDS_DIR, createKernel, describeIf, hasWasmBinaries } from '../helpers.js'; -import type { Kernel } from '../helpers.js'; +import { createWasmVmRuntime } from '@rivet-dev/agentos-vm-test-harness'; +import { COMMANDS_DIR, createKernel, describeIf, hasWasmBinaries } from '@rivet-dev/agentos-vm-test-harness'; +import type { Kernel } from '@rivet-dev/agentos-vm-test-harness'; import { existsSync } from 'node:fs'; import { resolve } from 'node:path'; diff --git a/packages/runtime-core/tests/mount-fs-custom-vfs.test.ts b/packages/runtime-core/tests/mount-fs-custom-vfs.test.ts index e81efc2567..a2a8ed8d9e 100644 --- a/packages/runtime-core/tests/mount-fs-custom-vfs.test.ts +++ b/packages/runtime-core/tests/mount-fs-custom-vfs.test.ts @@ -83,4 +83,22 @@ describe("Kernel.mountFs custom JS VFS", () => { }, 120_000, ); + + test("routes positioned writes through a mounted JS VFS", async () => { + const mounted = createRecordingFilesystem(); + kernel = createKernel({ filesystem: createInMemoryFileSystem() }); + + kernel.mountFs("/mnt/custom", mounted.fs); + await kernel.writeFile("/mnt/custom/db.bin", "abcde"); + await kernel.pwrite( + "/mnt/custom/db.bin", + 2, + new TextEncoder().encode("XYZ"), + ); + + expect( + new TextDecoder().decode(await kernel.readFile("/mnt/custom/db.bin")), + ).toBe("abXYZ"); + expect(mounted.calls).toContain("pwrite:/db.bin"); + }); }); diff --git a/packages/shell/CLAUDE.md b/packages/shell/CLAUDE.md index 21e058d520..aba1777d57 100644 --- a/packages/shell/CLAUDE.md +++ b/packages/shell/CLAUDE.md @@ -5,4 +5,4 @@ - `just shell` always uses the in-repo AgentOS sidecar it just built, even if the host environment exports `AGENTOS_SIDECAR_BIN`. To deliberately test another binary, invoke the shell CLI directly with that override. - `just shell --actor` additionally builds and pins the in-repo actor plugin. Actor development requires the Rivet repo at `../r6` (override with `AGENTOS_R6_ROOT`); the recipe installs the filtered `rivetkit` dependency graph there when its `tsx` loader is missing. - Do not implement or route through a custom/synthetic shell, prompt, line editor, or command parser; interactive shell mode must launch native Bash through the terminal/PTY path so behavior matches `docker run -it bash`. -- Keep `agentos-shell` loading every command-providing package from the in-repo `registry/software/`; when that registry changes, update the imports, package dependencies, and smoke coverage here. +- Keep `agentos-shell` loading every command-providing package from the in-repo `software/`; when that registry changes, update the imports, package dependencies, and smoke coverage here. diff --git a/packages/shell/package.json b/packages/shell/package.json index f76cda5d3c..38631b38c9 100644 --- a/packages/shell/package.json +++ b/packages/shell/package.json @@ -26,7 +26,6 @@ "@agentos-software/git": "workspace:*", "@agentos-software/grep": "workspace:*", "@agentos-software/gzip": "workspace:*", - "@agentos-software/http-get": "workspace:*", "@agentos-software/jq": "workspace:*", "@agentos-software/ripgrep": "workspace:*", "@agentos-software/sed": "workspace:*", @@ -35,7 +34,6 @@ "@agentos-software/tree": "workspace:*", "@agentos-software/unzip": "workspace:*", "@agentos-software/vim": "workspace:*", - "@agentos-software/vix": "workspace:*", "@agentos-software/wget": "workspace:*", "@agentos-software/yq": "workspace:*", "@agentos-software/zip": "workspace:*", diff --git a/packages/shell/src/main.ts b/packages/shell/src/main.ts index 1fb50aea0c..a4f9e17c9a 100644 --- a/packages/shell/src/main.ts +++ b/packages/shell/src/main.ts @@ -36,16 +36,15 @@ import gawk from "@agentos-software/gawk"; import git from "@agentos-software/git"; import grep from "@agentos-software/grep"; import gzip from "@agentos-software/gzip"; -import httpGet from "@agentos-software/http-get"; import jq from "@agentos-software/jq"; import ripgrep from "@agentos-software/ripgrep"; import sed from "@agentos-software/sed"; import sqlite3 from "@agentos-software/sqlite3"; import vim from "@agentos-software/vim"; -import vix from "@agentos-software/vix"; import tar from "@agentos-software/tar"; import tree from "@agentos-software/tree"; import unzip from "@agentos-software/unzip"; +import wget from "@agentos-software/wget"; import yq from "@agentos-software/yq"; import zip from "@agentos-software/zip"; import type { MountConfig, SoftwareInput } from "@rivet-dev/agentos-core"; @@ -59,7 +58,7 @@ const workspaceRoot = resolve(__dirname, "../../.."); const fallbackCommandDirs = [ resolve( workspaceRoot, - "registry/native/target/wasm32-wasip1/release/commands", + "toolchain/target/wasm32-wasip1/release/commands", ), ]; const BRUSH_SHELL_COMMANDS = new Set(["bash", "sh"]); @@ -154,18 +153,18 @@ const software: SoftwareInput[] = [ yq, codex, git, - httpGet, sqlite3, + wget, ] .map(withLocalCommandFallback) .filter((input): input is SoftwareInput => input !== null); -// The vi-like editors ship as registry packages: vix, and vim (which carries -// its runtime tree + VIMRUNTIME via the manifest `provides`). An unbuilt -// package is a valid empty placeholder — skip it rather than projecting a -// command-less package (build them locally: drop the wasm binaries into -// registry/native/target/.../commands and run `just registry-build`). -for (const editor of [vix, vim] as RegistryPackage[]) { +// vim ships as a software package (which carries its runtime tree + VIMRUNTIME +// via the manifest `provides`). An unbuilt package is a valid empty placeholder +// — skip it rather than projecting a command-less package (build it locally: +// drop the wasm binary into toolchain/target/.../commands and run +// `just software-build`). +for (const editor of [vim] as RegistryPackage[]) { if (isUsablePackageFile(editor.packagePath)) { software.push({ packagePath: editor.packagePath }); continue; diff --git a/packages/vm-test-harness/package.json b/packages/vm-test-harness/package.json new file mode 100644 index 0000000000..ae0cd7b34a --- /dev/null +++ b/packages/vm-test-harness/package.json @@ -0,0 +1,27 @@ +{ + "name": "@rivet-dev/agentos-vm-test-harness", + "version": "0.0.1", + "private": true, + "type": "module", + "license": "Apache-2.0", + "exports": { + ".": { + "types": "./src/index.ts", + "import": "./src/index.ts" + } + }, + "scripts": { + "check-types": "tsc --noEmit", + "build": "tsc --noEmit", + "test": "node -e \"process.exit(0)\"" + }, + "dependencies": { + "@rivet-dev/agentos-runtime-core": "workspace:*", + "@xterm/headless": "^6.0.0" + }, + "devDependencies": { + "@types/node": "^22.10.2", + "typescript": "^5.9.2", + "vitest": "^2.1.9" + } +} diff --git a/packages/vm-test-harness/src/index.ts b/packages/vm-test-harness/src/index.ts new file mode 100644 index 0000000000..4e6335b9c5 --- /dev/null +++ b/packages/vm-test-harness/src/index.ts @@ -0,0 +1,168 @@ +import { existsSync } from "node:fs"; +import { resolve } from "node:path"; +import { describe, it } from "vitest"; + +/** Directory containing WASM command binaries built from Rust. */ +export const COMMANDS_DIR = resolve( + process.env.AGENTOS_WASM_COMMANDS_DIR ?? + resolve(import.meta.dirname, "../../runtime-core/commands"), +); + +/** Directory containing C-compiled WASM binaries. */ +export const C_BUILD_DIR = resolve( + process.env.AGENTOS_C_WASM_COMMANDS_DIR ?? + resolve(import.meta.dirname, "../../../toolchain/c/build"), +); + +/** Whether the main WASM command binaries are available (includes 'sh'). */ +export const hasWasmBinaries = + existsSync(COMMANDS_DIR) && existsSync(resolve(COMMANDS_DIR, "sh")); + +/** + * Check whether specific C WASM binaries are present. + * @param names - Binary names to check for inside C_BUILD_DIR. + * @returns true if all requested binaries exist. + */ +export function hasCWasmBinaries(...names: string[]): boolean { + if (!existsSync(C_BUILD_DIR)) return false; + return names.every((name) => existsSync(resolve(C_BUILD_DIR, name))); +} + +/** + * Returns a skip-reason string if WASM binaries are missing, or false if + * they are available and tests should run. + */ +export function skipReason(): string | false { + if (!hasWasmBinaries) { + return `WASM binaries not found at ${COMMANDS_DIR} - build toolchain first`; + } + return false; +} + +export function describeIf( + condition: unknown, + ...args: Parameters +): void { + if (condition) { + describe(...args); + return; + } + const [name] = args; + describe.skip(`${String(name)} [environment prerequisites not met]`, () => {}); +} + +export function itIf(condition: unknown, ...args: Parameters): void { + if (condition) { + it(...args); + return; + } + const [name] = args; + it.skip(`${String(name)} [environment prerequisites not met]`, () => {}); +} + +export { + AF_INET, + AF_UNIX, + allowAll, + createInMemoryFileSystem, + SIGTERM, + SOCK_DGRAM, + SOCK_STREAM, +} from "../../runtime-core/src/test-runtime.js"; +import { + allowAll, + createInMemoryFileSystem, + createKernel as createKernelBase, + createNodeRuntime, + createWasmVmRuntime, +} from "../../runtime-core/src/test-runtime.js"; +export type { + DriverProcess, + Kernel, + KernelInterface, + KernelRuntimeDriver, + Permissions, + ProcessContext, + VirtualFileSystem, +} from "../../runtime-core/src/test-runtime.js"; +export { + createWasmVmRuntime, + DEFAULT_FIRST_PARTY_TIERS, + WASMVM_COMMANDS, + type PermissionTier, + type WasmVmRuntimeOptions, +} from "../../runtime-core/src/test-runtime.js"; +export { + createNodeHostNetworkAdapter, + createNodeRuntime, + NodeFileSystem, +} from "../../runtime-core/src/test-runtime.js"; +export { TerminalHarness } from "./terminal-harness.js"; + +/** + * Registry integration tests assume they can bootstrap runtimes and /bin stubs + * unless they explicitly opt into a stricter permission policy. + */ +export function createKernel( + options: Parameters[0], +): ReturnType { + return createKernelBase({ + ...options, + permissions: options.permissions ?? allowAll, + }); +} + +export interface IntegrationKernelResult { + kernel: ReturnType; + vfs: ReturnType; + dispose: () => Promise; +} + +export interface IntegrationKernelOptions { + runtimes?: ("wasmvm" | "node")[]; + loopbackExemptPorts?: number[]; + commandDirs?: string[]; + permissions?: Parameters[0]["permissions"]; +} + +/** + * Create a kernel with the in-scope runtime drivers for integration testing. + * + * Mount order matters. Last-mounted driver wins for overlapping commands: + * 1. WasmVM first: provides sh/bash/coreutils (90+ commands) + * 2. Node second: overrides WasmVM's 'node' stub with real V8 + */ +export async function createIntegrationKernel( + options?: IntegrationKernelOptions, +): Promise { + const runtimes = options?.runtimes ?? ["wasmvm"]; + const vfs = createInMemoryFileSystem(); + const kernel = createKernel({ + filesystem: vfs, + loopbackExemptPorts: options?.loopbackExemptPorts, + permissions: options?.permissions, + }); + + if (runtimes.includes("wasmvm")) { + await kernel.mount( + createWasmVmRuntime({ commandDirs: options?.commandDirs ?? [COMMANDS_DIR] }), + ); + } + if (runtimes.includes("node")) { + await kernel.mount(createNodeRuntime()); + } + + return { + kernel, + vfs, + dispose: () => kernel.dispose(), + }; +} + +/** + * Skip helper: returns a reason string if the WASM binaries are not built, + * or false if the commands directory exists and tests can run. + */ +export function skipUnlessWasmBuilt(): string | false { + return skipReason(); +} diff --git a/packages/vm-test-harness/src/terminal-harness.ts b/packages/vm-test-harness/src/terminal-harness.ts new file mode 100644 index 0000000000..3ec00c1caa --- /dev/null +++ b/packages/vm-test-harness/src/terminal-harness.ts @@ -0,0 +1,159 @@ +/** + * TerminalHarness wires openShell() to a headless xterm Terminal so tests can + * assert against deterministic terminal screen state. + */ + +import { Terminal } from "@xterm/headless"; +import type { Kernel } from "./index.js"; + +type ShellHandle = ReturnType; + +const SETTLE_MS = 50; +const POLL_MS = 20; +const DEFAULT_WAIT_TIMEOUT_MS = 5_000; + +export class TerminalHarness { + readonly term: Terminal; + readonly shell: ShellHandle; + private typing = false; + private disposed = false; + + constructor( + kernel: Kernel, + options?: { + cols?: number; + rows?: number; + env?: Record; + cwd?: string; + }, + ) { + const cols = options?.cols ?? 80; + const rows = options?.rows ?? 24; + + this.term = new Terminal({ cols, rows, allowProposedApi: true }); + this.shell = kernel.openShell({ + cols, + rows, + env: options?.env, + cwd: options?.cwd, + onStderr: (data: Uint8Array) => { + this.term.write(data); + }, + }); + this.shell.onData = (data: Uint8Array) => { + this.term.write(data); + }; + } + + async type(input: string): Promise { + if (this.typing) { + throw new Error( + "TerminalHarness.type() called while previous type() is still in-flight", + ); + } + this.typing = true; + try { + await this.typeInternal(input); + } finally { + this.typing = false; + } + } + + private typeInternal(input: string): Promise { + return new Promise((resolve) => { + let timer: ReturnType | null = null; + const originalOnData = this.shell.onData; + + const resetTimer = () => { + if (timer !== null) clearTimeout(timer); + timer = setTimeout(() => { + this.shell.onData = originalOnData; + resolve(); + }, SETTLE_MS); + }; + + this.shell.onData = (data: Uint8Array) => { + this.term.write(data); + resetTimer(); + }; + + resetTimer(); + this.shell.write(input); + }); + } + + screenshotTrimmed(): string { + const buf = this.term.buffer.active; + const lines: string[] = []; + + for (let row = 0; row < this.term.rows; row++) { + const line = buf.getLine(buf.viewportY + row); + lines.push(line ? line.translateToString(true) : ""); + } + + while (lines.length > 0 && lines[lines.length - 1] === "") { + lines.pop(); + } + + return lines.join("\n"); + } + + line(row: number): string { + const buf = this.term.buffer.active; + const line = buf.getLine(buf.viewportY + row); + return line ? line.translateToString(true) : ""; + } + + async waitFor( + text: string, + occurrence: number = 1, + timeoutMs: number = DEFAULT_WAIT_TIMEOUT_MS, + ): Promise { + const deadline = Date.now() + timeoutMs; + + while (true) { + const screen = this.screenshotTrimmed(); + + let count = 0; + let idx = -1; + while (true) { + idx = screen.indexOf(text, idx + 1); + if (idx === -1) break; + count++; + if (count >= occurrence) return; + } + + if (Date.now() >= deadline) { + throw new Error( + `waitFor("${text}", ${occurrence}) timed out after ${timeoutMs}ms.\n` + + `Expected: "${text}" (occurrence ${occurrence})\n` + + `Screen:\n${screen}`, + ); + } + + await new Promise((resolve) => setTimeout(resolve, POLL_MS)); + } + } + + async exit(): Promise { + this.shell.write("\x04"); + return this.shell.wait(); + } + + async dispose(): Promise { + if (this.disposed) return; + this.disposed = true; + + try { + this.shell.kill(); + await Promise.race([ + this.shell.wait(), + new Promise((resolve) => setTimeout(resolve, 500)), + ]); + } catch { + // Shell may already be gone. + } + + this.term.dispose(); + } +} diff --git a/packages/vm-test-harness/tsconfig.json b/packages/vm-test-harness/tsconfig.json new file mode 100644 index 0000000000..639c948caf --- /dev/null +++ b/packages/vm-test-harness/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "module": "NodeNext", + "moduleResolution": "NodeNext", + "esModuleInterop": true, + "noEmit": true + }, + "include": ["src/**/*"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ab1a636334..b7c1814c38 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -25,16 +25,19 @@ importers: devDependencies: '@agentos-software/claude-code': specifier: workspace:* - version: link:registry/agent/claude + version: link:software/claude '@agentos-software/codex': specifier: workspace:* - version: link:registry/agent/codex + version: link:software/codex '@agentos-software/common': specifier: workspace:* - version: link:registry/software/common + version: link:software/common '@agentos-software/pi': specifier: workspace:* - version: link:registry/agent/pi + version: link:software/pi + '@agentos/test-harness': + specifier: workspace:* + version: link:test-harness '@biomejs/biome': specifier: ^2.3 version: 2.4.10 @@ -50,12 +53,18 @@ importers: '@rivet-dev/agentos-runtime-core': specifier: workspace:* version: link:packages/runtime-core + '@rivet-dev/agentos-vm-test-harness': + specifier: workspace:* + version: link:packages/vm-test-harness '@types/node': specifier: ^22.19.15 version: 22.19.15 jszip: specifier: ^3.10.1 version: 3.10.1 + npm: + specifier: ^11.18.0 + version: 11.18.0 pdf-lib: specifier: ^1.17.1 version: 1.17.1 @@ -70,16 +79,16 @@ importers: dependencies: '@agentos-software/claude-code': specifier: workspace:* - version: link:../../registry/agent/claude + version: link:../../software/claude '@agentos-software/git': specifier: workspace:* - version: link:../../registry/software/git + version: link:../../software/git '@agentos-software/opencode': specifier: workspace:* - version: link:../../registry/agent/opencode + version: link:../../software/opencode '@agentos-software/pi': specifier: workspace:* - version: link:../../registry/agent/pi + version: link:../../software/pi '@rivet-dev/agentos': specifier: workspace:* version: link:../../packages/agentos @@ -135,16 +144,16 @@ importers: dependencies: '@agentos-software/claude-code': specifier: workspace:* - version: link:../../registry/agent/claude + version: link:../../software/claude '@agentos-software/git': specifier: workspace:* - version: link:../../registry/software/git + version: link:../../software/git '@agentos-software/opencode': specifier: workspace:* - version: link:../../registry/agent/opencode + version: link:../../software/opencode '@agentos-software/pi': specifier: workspace:* - version: link:../../registry/agent/pi + version: link:../../software/pi '@rivet-dev/agentos': specifier: workspace:* version: link:../../packages/agentos @@ -184,16 +193,16 @@ importers: dependencies: '@agentos-software/claude-code': specifier: workspace:* - version: link:../../registry/agent/claude + version: link:../../software/claude '@agentos-software/git': specifier: workspace:* - version: link:../../registry/software/git + version: link:../../software/git '@agentos-software/opencode': specifier: workspace:* - version: link:../../registry/agent/opencode + version: link:../../software/opencode '@agentos-software/pi': specifier: workspace:* - version: link:../../registry/agent/pi + version: link:../../software/pi '@rivet-dev/agentos': specifier: workspace:* version: link:../../packages/agentos @@ -233,16 +242,16 @@ importers: dependencies: '@agentos-software/claude-code': specifier: workspace:* - version: link:../../registry/agent/claude + version: link:../../software/claude '@agentos-software/git': specifier: workspace:* - version: link:../../registry/software/git + version: link:../../software/git '@agentos-software/opencode': specifier: workspace:* - version: link:../../registry/agent/opencode + version: link:../../software/opencode '@agentos-software/pi': specifier: workspace:* - version: link:../../registry/agent/pi + version: link:../../software/pi '@rivet-dev/agentos': specifier: workspace:* version: link:../../packages/agentos @@ -282,22 +291,22 @@ importers: dependencies: '@agentos-software/common': specifier: workspace:* - version: link:../../registry/software/common + version: link:../../software/common '@agentos-software/curl': specifier: workspace:* - version: link:../../registry/software/curl + version: link:../../software/curl '@agentos-software/git': specifier: workspace:* - version: link:../../registry/software/git + version: link:../../software/git '@agentos-software/jq': specifier: workspace:* - version: link:../../registry/software/jq + version: link:../../software/jq '@agentos-software/ripgrep': specifier: workspace:* - version: link:../../registry/software/ripgrep + version: link:../../software/ripgrep '@agentos-software/sqlite3': specifier: workspace:* - version: link:../../registry/software/sqlite3 + version: link:../../software/sqlite3 '@rivet-dev/agentos': specifier: workspace:* version: link:../../packages/agentos @@ -346,13 +355,13 @@ importers: dependencies: '@agentos-software/browserbase': specifier: workspace:* - version: link:../../registry/software/browserbase + version: link:../../software/browserbase '@agentos-software/claude-code': specifier: workspace:* - version: link:../../registry/agent/claude + version: link:../../software/claude '@agentos-software/pi': specifier: workspace:* - version: link:../../registry/agent/pi + version: link:../../software/pi '@rivet-dev/agentos': specifier: workspace:* version: link:../../packages/agentos @@ -374,16 +383,16 @@ importers: dependencies: '@agentos-software/claude-code': specifier: workspace:* - version: link:../../registry/agent/claude + version: link:../../software/claude '@agentos-software/git': specifier: workspace:* - version: link:../../registry/software/git + version: link:../../software/git '@agentos-software/opencode': specifier: workspace:* - version: link:../../registry/agent/opencode + version: link:../../software/opencode '@agentos-software/pi': specifier: workspace:* - version: link:../../registry/agent/pi + version: link:../../software/pi '@rivet-dev/agentos': specifier: workspace:* version: link:../../packages/agentos @@ -423,16 +432,16 @@ importers: dependencies: '@agentos-software/claude-code': specifier: workspace:* - version: link:../../registry/agent/claude + version: link:../../software/claude '@agentos-software/git': specifier: workspace:* - version: link:../../registry/software/git + version: link:../../software/git '@agentos-software/opencode': specifier: workspace:* - version: link:../../registry/agent/opencode + version: link:../../software/opencode '@agentos-software/pi': specifier: workspace:* - version: link:../../registry/agent/pi + version: link:../../software/pi '@rivet-dev/agentos': specifier: workspace:* version: link:../../packages/agentos @@ -472,16 +481,16 @@ importers: dependencies: '@agentos-software/claude-code': specifier: workspace:* - version: link:../../registry/agent/claude + version: link:../../software/claude '@agentos-software/git': specifier: workspace:* - version: link:../../registry/software/git + version: link:../../software/git '@agentos-software/opencode': specifier: workspace:* - version: link:../../registry/agent/opencode + version: link:../../software/opencode '@agentos-software/pi': specifier: workspace:* - version: link:../../registry/agent/pi + version: link:../../software/pi '@rivet-dev/agentos': specifier: workspace:* version: link:../../packages/agentos @@ -521,16 +530,16 @@ importers: dependencies: '@agentos-software/claude-code': specifier: workspace:* - version: link:../../registry/agent/claude + version: link:../../software/claude '@agentos-software/git': specifier: workspace:* - version: link:../../registry/software/git + version: link:../../software/git '@agentos-software/opencode': specifier: workspace:* - version: link:../../registry/agent/opencode + version: link:../../software/opencode '@agentos-software/pi': specifier: workspace:* - version: link:../../registry/agent/pi + version: link:../../software/pi '@rivet-dev/agentos': specifier: workspace:* version: link:../../packages/agentos @@ -570,16 +579,16 @@ importers: dependencies: '@agentos-software/claude-code': specifier: workspace:* - version: link:../../registry/agent/claude + version: link:../../software/claude '@agentos-software/git': specifier: workspace:* - version: link:../../registry/software/git + version: link:../../software/git '@agentos-software/opencode': specifier: workspace:* - version: link:../../registry/agent/opencode + version: link:../../software/opencode '@agentos-software/pi': specifier: workspace:* - version: link:../../registry/agent/pi + version: link:../../software/pi '@rivet-dev/agentos': specifier: workspace:* version: link:../../packages/agentos @@ -619,16 +628,16 @@ importers: dependencies: '@agentos-software/claude-code': specifier: workspace:* - version: link:../../registry/agent/claude + version: link:../../software/claude '@agentos-software/git': specifier: workspace:* - version: link:../../registry/software/git + version: link:../../software/git '@agentos-software/opencode': specifier: workspace:* - version: link:../../registry/agent/opencode + version: link:../../software/opencode '@agentos-software/pi': specifier: workspace:* - version: link:../../registry/agent/pi + version: link:../../software/pi '@rivet-dev/agentos': specifier: workspace:* version: link:../../packages/agentos @@ -668,16 +677,16 @@ importers: dependencies: '@agentos-software/claude-code': specifier: workspace:* - version: link:../../registry/agent/claude + version: link:../../software/claude '@agentos-software/git': specifier: workspace:* - version: link:../../registry/software/git + version: link:../../software/git '@agentos-software/opencode': specifier: workspace:* - version: link:../../registry/agent/opencode + version: link:../../software/opencode '@agentos-software/pi': specifier: workspace:* - version: link:../../registry/agent/pi + version: link:../../software/pi '@rivet-dev/agentos': specifier: workspace:* version: link:../../packages/agentos @@ -717,16 +726,16 @@ importers: dependencies: '@agentos-software/claude-code': specifier: workspace:* - version: link:../../registry/agent/claude + version: link:../../software/claude '@agentos-software/git': specifier: workspace:* - version: link:../../registry/software/git + version: link:../../software/git '@agentos-software/opencode': specifier: workspace:* - version: link:../../registry/agent/opencode + version: link:../../software/opencode '@agentos-software/pi': specifier: workspace:* - version: link:../../registry/agent/pi + version: link:../../software/pi '@rivet-dev/agentos': specifier: workspace:* version: link:../../packages/agentos @@ -766,16 +775,16 @@ importers: dependencies: '@agentos-software/claude-code': specifier: workspace:* - version: link:../../registry/agent/claude + version: link:../../software/claude '@agentos-software/git': specifier: workspace:* - version: link:../../registry/software/git + version: link:../../software/git '@agentos-software/opencode': specifier: workspace:* - version: link:../../registry/agent/opencode + version: link:../../software/opencode '@agentos-software/pi': specifier: workspace:* - version: link:../../registry/agent/pi + version: link:../../software/pi '@rivet-dev/agentos': specifier: workspace:* version: link:../../packages/agentos @@ -815,16 +824,16 @@ importers: dependencies: '@agentos-software/claude-code': specifier: workspace:* - version: link:../../registry/agent/claude + version: link:../../software/claude '@agentos-software/git': specifier: workspace:* - version: link:../../registry/software/git + version: link:../../software/git '@agentos-software/opencode': specifier: workspace:* - version: link:../../registry/agent/opencode + version: link:../../software/opencode '@agentos-software/pi': specifier: workspace:* - version: link:../../registry/agent/pi + version: link:../../software/pi '@rivet-dev/agentos': specifier: workspace:* version: link:../../packages/agentos @@ -864,16 +873,16 @@ importers: dependencies: '@agentos-software/claude-code': specifier: workspace:* - version: link:../../registry/agent/claude + version: link:../../software/claude '@agentos-software/git': specifier: workspace:* - version: link:../../registry/software/git + version: link:../../software/git '@agentos-software/opencode': specifier: workspace:* - version: link:../../registry/agent/opencode + version: link:../../software/opencode '@agentos-software/pi': specifier: workspace:* - version: link:../../registry/agent/pi + version: link:../../software/pi '@rivet-dev/agentos': specifier: workspace:* version: link:../../packages/agentos @@ -913,16 +922,16 @@ importers: dependencies: '@agentos-software/claude-code': specifier: workspace:* - version: link:../../registry/agent/claude + version: link:../../software/claude '@agentos-software/git': specifier: workspace:* - version: link:../../registry/software/git + version: link:../../software/git '@agentos-software/opencode': specifier: workspace:* - version: link:../../registry/agent/opencode + version: link:../../software/opencode '@agentos-software/pi': specifier: workspace:* - version: link:../../registry/agent/pi + version: link:../../software/pi '@rivet-dev/agentos': specifier: workspace:* version: link:../../packages/agentos @@ -962,16 +971,16 @@ importers: dependencies: '@agentos-software/claude-code': specifier: workspace:* - version: link:../../registry/agent/claude + version: link:../../software/claude '@agentos-software/git': specifier: workspace:* - version: link:../../registry/software/git + version: link:../../software/git '@agentos-software/opencode': specifier: workspace:* - version: link:../../registry/agent/opencode + version: link:../../software/opencode '@agentos-software/pi': specifier: workspace:* - version: link:../../registry/agent/pi + version: link:../../software/pi '@rivet-dev/agentos': specifier: workspace:* version: link:../../packages/agentos @@ -1011,16 +1020,16 @@ importers: dependencies: '@agentos-software/claude-code': specifier: workspace:* - version: link:../../registry/agent/claude + version: link:../../software/claude '@agentos-software/git': specifier: workspace:* - version: link:../../registry/software/git + version: link:../../software/git '@agentos-software/opencode': specifier: workspace:* - version: link:../../registry/agent/opencode + version: link:../../software/opencode '@agentos-software/pi': specifier: workspace:* - version: link:../../registry/agent/pi + version: link:../../software/pi '@rivet-dev/agentos': specifier: workspace:* version: link:../../packages/agentos @@ -1060,16 +1069,16 @@ importers: dependencies: '@agentos-software/claude-code': specifier: workspace:* - version: link:../../registry/agent/claude + version: link:../../software/claude '@agentos-software/git': specifier: workspace:* - version: link:../../registry/software/git + version: link:../../software/git '@agentos-software/opencode': specifier: workspace:* - version: link:../../registry/agent/opencode + version: link:../../software/opencode '@agentos-software/pi': specifier: workspace:* - version: link:../../registry/agent/pi + version: link:../../software/pi '@rivet-dev/agentos': specifier: workspace:* version: link:../../packages/agentos @@ -1109,16 +1118,16 @@ importers: dependencies: '@agentos-software/claude-code': specifier: workspace:* - version: link:../../registry/agent/claude + version: link:../../software/claude '@agentos-software/git': specifier: workspace:* - version: link:../../registry/software/git + version: link:../../software/git '@agentos-software/opencode': specifier: workspace:* - version: link:../../registry/agent/opencode + version: link:../../software/opencode '@agentos-software/pi': specifier: workspace:* - version: link:../../registry/agent/pi + version: link:../../software/pi '@rivet-dev/agentos': specifier: workspace:* version: link:../../packages/agentos @@ -1158,16 +1167,16 @@ importers: dependencies: '@agentos-software/claude-code': specifier: workspace:* - version: link:../../registry/agent/claude + version: link:../../software/claude '@agentos-software/git': specifier: workspace:* - version: link:../../registry/software/git + version: link:../../software/git '@agentos-software/opencode': specifier: workspace:* - version: link:../../registry/agent/opencode + version: link:../../software/opencode '@agentos-software/pi': specifier: workspace:* - version: link:../../registry/agent/pi + version: link:../../software/pi '@rivet-dev/agentos': specifier: workspace:* version: link:../../packages/agentos @@ -1207,16 +1216,16 @@ importers: dependencies: '@agentos-software/claude-code': specifier: workspace:* - version: link:../../registry/agent/claude + version: link:../../software/claude '@agentos-software/git': specifier: workspace:* - version: link:../../registry/software/git + version: link:../../software/git '@agentos-software/opencode': specifier: workspace:* - version: link:../../registry/agent/opencode + version: link:../../software/opencode '@agentos-software/pi': specifier: workspace:* - version: link:../../registry/agent/pi + version: link:../../software/pi '@rivet-dev/agentos': specifier: workspace:* version: link:../../packages/agentos @@ -1256,16 +1265,16 @@ importers: dependencies: '@agentos-software/claude-code': specifier: workspace:* - version: link:../../registry/agent/claude + version: link:../../software/claude '@agentos-software/git': specifier: workspace:* - version: link:../../registry/software/git + version: link:../../software/git '@agentos-software/opencode': specifier: workspace:* - version: link:../../registry/agent/opencode + version: link:../../software/opencode '@agentos-software/pi': specifier: workspace:* - version: link:../../registry/agent/pi + version: link:../../software/pi '@rivet-dev/agentos': specifier: workspace:* version: link:../../packages/agentos @@ -1314,16 +1323,16 @@ importers: dependencies: '@agentos-software/claude-code': specifier: workspace:* - version: link:../../../registry/agent/claude + version: link:../../../software/claude '@agentos-software/git': specifier: workspace:* - version: link:../../../registry/software/git + version: link:../../../software/git '@agentos-software/opencode': specifier: workspace:* - version: link:../../../registry/agent/opencode + version: link:../../../software/opencode '@agentos-software/pi': specifier: workspace:* - version: link:../../../registry/agent/pi + version: link:../../../software/pi '@rivet-dev/agentos': specifier: workspace:* version: link:../../../packages/agentos @@ -1363,16 +1372,16 @@ importers: dependencies: '@agentos-software/claude-code': specifier: workspace:* - version: link:../../../registry/agent/claude + version: link:../../../software/claude '@agentos-software/git': specifier: workspace:* - version: link:../../../registry/software/git + version: link:../../../software/git '@agentos-software/opencode': specifier: workspace:* - version: link:../../../registry/agent/opencode + version: link:../../../software/opencode '@agentos-software/pi': specifier: workspace:* - version: link:../../../registry/agent/pi + version: link:../../../software/pi '@rivet-dev/agentos': specifier: workspace:* version: link:../../../packages/agentos @@ -1412,16 +1421,16 @@ importers: dependencies: '@agentos-software/claude-code': specifier: workspace:* - version: link:../../../registry/agent/claude + version: link:../../../software/claude '@agentos-software/git': specifier: workspace:* - version: link:../../../registry/software/git + version: link:../../../software/git '@agentos-software/opencode': specifier: workspace:* - version: link:../../../registry/agent/opencode + version: link:../../../software/opencode '@agentos-software/pi': specifier: workspace:* - version: link:../../../registry/agent/pi + version: link:../../../software/pi '@rivet-dev/agentos': specifier: workspace:* version: link:../../../packages/agentos @@ -1461,16 +1470,16 @@ importers: dependencies: '@agentos-software/claude-code': specifier: workspace:* - version: link:../../../registry/agent/claude + version: link:../../../software/claude '@agentos-software/git': specifier: workspace:* - version: link:../../../registry/software/git + version: link:../../../software/git '@agentos-software/opencode': specifier: workspace:* - version: link:../../../registry/agent/opencode + version: link:../../../software/opencode '@agentos-software/pi': specifier: workspace:* - version: link:../../../registry/agent/pi + version: link:../../../software/pi '@rivet-dev/agentos': specifier: workspace:* version: link:../../../packages/agentos @@ -1510,16 +1519,16 @@ importers: dependencies: '@agentos-software/claude-code': specifier: workspace:* - version: link:../../../registry/agent/claude + version: link:../../../software/claude '@agentos-software/git': specifier: workspace:* - version: link:../../../registry/software/git + version: link:../../../software/git '@agentos-software/opencode': specifier: workspace:* - version: link:../../../registry/agent/opencode + version: link:../../../software/opencode '@agentos-software/pi': specifier: workspace:* - version: link:../../../registry/agent/pi + version: link:../../../software/pi '@rivet-dev/agentos': specifier: workspace:* version: link:../../../packages/agentos @@ -1559,16 +1568,16 @@ importers: dependencies: '@agentos-software/claude-code': specifier: workspace:* - version: link:../../../registry/agent/claude + version: link:../../../software/claude '@agentos-software/git': specifier: workspace:* - version: link:../../../registry/software/git + version: link:../../../software/git '@agentos-software/opencode': specifier: workspace:* - version: link:../../../registry/agent/opencode + version: link:../../../software/opencode '@agentos-software/pi': specifier: workspace:* - version: link:../../../registry/agent/pi + version: link:../../../software/pi '@rivet-dev/agentos': specifier: workspace:* version: link:../../../packages/agentos @@ -1608,16 +1617,16 @@ importers: dependencies: '@agentos-software/claude-code': specifier: workspace:* - version: link:../../../registry/agent/claude + version: link:../../../software/claude '@agentos-software/git': specifier: workspace:* - version: link:../../../registry/software/git + version: link:../../../software/git '@agentos-software/opencode': specifier: workspace:* - version: link:../../../registry/agent/opencode + version: link:../../../software/opencode '@agentos-software/pi': specifier: workspace:* - version: link:../../../registry/agent/pi + version: link:../../../software/pi '@rivet-dev/agentos': specifier: workspace:* version: link:../../../packages/agentos @@ -1657,16 +1666,16 @@ importers: dependencies: '@agentos-software/claude-code': specifier: workspace:* - version: link:../../../registry/agent/claude + version: link:../../../software/claude '@agentos-software/git': specifier: workspace:* - version: link:../../../registry/software/git + version: link:../../../software/git '@agentos-software/opencode': specifier: workspace:* - version: link:../../../registry/agent/opencode + version: link:../../../software/opencode '@agentos-software/pi': specifier: workspace:* - version: link:../../../registry/agent/pi + version: link:../../../software/pi '@rivet-dev/agentos': specifier: workspace:* version: link:../../../packages/agentos @@ -1706,16 +1715,16 @@ importers: dependencies: '@agentos-software/claude-code': specifier: workspace:* - version: link:../../../registry/agent/claude + version: link:../../../software/claude '@agentos-software/git': specifier: workspace:* - version: link:../../../registry/software/git + version: link:../../../software/git '@agentos-software/opencode': specifier: workspace:* - version: link:../../../registry/agent/opencode + version: link:../../../software/opencode '@agentos-software/pi': specifier: workspace:* - version: link:../../../registry/agent/pi + version: link:../../../software/pi '@rivet-dev/agentos': specifier: workspace:* version: link:../../../packages/agentos @@ -1755,16 +1764,16 @@ importers: dependencies: '@agentos-software/claude-code': specifier: workspace:* - version: link:../../../registry/agent/claude + version: link:../../../software/claude '@agentos-software/git': specifier: workspace:* - version: link:../../../registry/software/git + version: link:../../../software/git '@agentos-software/opencode': specifier: workspace:* - version: link:../../../registry/agent/opencode + version: link:../../../software/opencode '@agentos-software/pi': specifier: workspace:* - version: link:../../../registry/agent/pi + version: link:../../../software/pi '@rivet-dev/agentos': specifier: workspace:* version: link:../../../packages/agentos @@ -1804,16 +1813,16 @@ importers: dependencies: '@agentos-software/claude-code': specifier: workspace:* - version: link:../../../registry/agent/claude + version: link:../../../software/claude '@agentos-software/git': specifier: workspace:* - version: link:../../../registry/software/git + version: link:../../../software/git '@agentos-software/opencode': specifier: workspace:* - version: link:../../../registry/agent/opencode + version: link:../../../software/opencode '@agentos-software/pi': specifier: workspace:* - version: link:../../../registry/agent/pi + version: link:../../../software/pi '@rivet-dev/agentos': specifier: workspace:* version: link:../../../packages/agentos @@ -1853,16 +1862,16 @@ importers: dependencies: '@agentos-software/claude-code': specifier: workspace:* - version: link:../../../registry/agent/claude + version: link:../../../software/claude '@agentos-software/git': specifier: workspace:* - version: link:../../../registry/software/git + version: link:../../../software/git '@agentos-software/opencode': specifier: workspace:* - version: link:../../../registry/agent/opencode + version: link:../../../software/opencode '@agentos-software/pi': specifier: workspace:* - version: link:../../../registry/agent/pi + version: link:../../../software/pi '@rivet-dev/agentos': specifier: workspace:* version: link:../../../packages/agentos @@ -1902,16 +1911,16 @@ importers: dependencies: '@agentos-software/claude-code': specifier: workspace:* - version: link:../../../registry/agent/claude + version: link:../../../software/claude '@agentos-software/git': specifier: workspace:* - version: link:../../../registry/software/git + version: link:../../../software/git '@agentos-software/opencode': specifier: workspace:* - version: link:../../../registry/agent/opencode + version: link:../../../software/opencode '@agentos-software/pi': specifier: workspace:* - version: link:../../../registry/agent/pi + version: link:../../../software/pi '@rivet-dev/agentos': specifier: workspace:* version: link:../../../packages/agentos @@ -1951,16 +1960,16 @@ importers: dependencies: '@agentos-software/claude-code': specifier: workspace:* - version: link:../../registry/agent/claude + version: link:../../software/claude '@agentos-software/git': specifier: workspace:* - version: link:../../registry/software/git + version: link:../../software/git '@agentos-software/opencode': specifier: workspace:* - version: link:../../registry/agent/opencode + version: link:../../software/opencode '@agentos-software/pi': specifier: workspace:* - version: link:../../registry/agent/pi + version: link:../../software/pi '@rivet-dev/agentos': specifier: workspace:* version: link:../../packages/agentos @@ -2000,16 +2009,16 @@ importers: dependencies: '@agentos-software/claude-code': specifier: workspace:* - version: link:../../registry/agent/claude + version: link:../../software/claude '@agentos-software/git': specifier: workspace:* - version: link:../../registry/software/git + version: link:../../software/git '@agentos-software/opencode': specifier: workspace:* - version: link:../../registry/agent/opencode + version: link:../../software/opencode '@agentos-software/pi': specifier: workspace:* - version: link:../../registry/agent/pi + version: link:../../software/pi '@rivet-dev/agentos': specifier: workspace:* version: link:../../packages/agentos @@ -2049,16 +2058,16 @@ importers: dependencies: '@agentos-software/claude-code': specifier: workspace:* - version: link:../../registry/agent/claude + version: link:../../software/claude '@agentos-software/git': specifier: workspace:* - version: link:../../registry/software/git + version: link:../../software/git '@agentos-software/opencode': specifier: workspace:* - version: link:../../registry/agent/opencode + version: link:../../software/opencode '@agentos-software/pi': specifier: workspace:* - version: link:../../registry/agent/pi + version: link:../../software/pi '@rivet-dev/agentos': specifier: workspace:* version: link:../../packages/agentos @@ -2098,25 +2107,25 @@ importers: dependencies: '@agentos-software/claude-code': specifier: workspace:* - version: link:../../registry/agent/claude + version: link:../../software/claude '@agentos-software/coreutils': specifier: workspace:* - version: link:../../registry/software/coreutils + version: link:../../software/coreutils '@agentos-software/git': specifier: workspace:* - version: link:../../registry/software/git + version: link:../../software/git '@agentos-software/jq': specifier: workspace:* - version: link:../../registry/software/jq + version: link:../../software/jq '@agentos-software/opencode': specifier: workspace:* - version: link:../../registry/agent/opencode + version: link:../../software/opencode '@agentos-software/pi': specifier: workspace:* - version: link:../../registry/agent/pi + version: link:../../software/pi '@agentos-software/ripgrep': specifier: workspace:* - version: link:../../registry/software/ripgrep + version: link:../../software/ripgrep '@rivet-dev/agentos': specifier: workspace:* version: link:../../packages/agentos @@ -2156,16 +2165,16 @@ importers: dependencies: '@agentos-software/claude-code': specifier: workspace:* - version: link:../../registry/agent/claude + version: link:../../software/claude '@agentos-software/git': specifier: workspace:* - version: link:../../registry/software/git + version: link:../../software/git '@agentos-software/opencode': specifier: workspace:* - version: link:../../registry/agent/opencode + version: link:../../software/opencode '@agentos-software/pi': specifier: workspace:* - version: link:../../registry/agent/pi + version: link:../../software/pi '@rivet-dev/agentos': specifier: workspace:* version: link:../../packages/agentos @@ -2208,16 +2217,16 @@ importers: dependencies: '@agentos-software/claude-code': specifier: workspace:* - version: link:../../registry/agent/claude + version: link:../../software/claude '@agentos-software/git': specifier: workspace:* - version: link:../../registry/software/git + version: link:../../software/git '@agentos-software/opencode': specifier: workspace:* - version: link:../../registry/agent/opencode + version: link:../../software/opencode '@agentos-software/pi': specifier: workspace:* - version: link:../../registry/agent/pi + version: link:../../software/pi '@rivet-dev/agentos': specifier: workspace:* version: link:../../packages/agentos @@ -2257,7 +2266,7 @@ importers: dependencies: '@agentos-software/common': specifier: workspace:* - version: link:../../registry/software/common + version: link:../../software/common '@rivet-dev/agentos-core': specifier: workspace:* version: link:../core @@ -2354,7 +2363,7 @@ importers: devDependencies: '@agentos-software/common': specifier: workspace:* - version: link:../../registry/software/common + version: link:../../software/common '@types/node': specifier: ^22.10.2 version: 22.19.15 @@ -2440,25 +2449,25 @@ importers: dependencies: '@agentos-software/claude-code': specifier: workspace:* - version: link:../../registry/agent/claude + version: link:../../software/claude '@agentos-software/codex-cli': specifier: workspace:* - version: link:../../registry/software/codex-cli + version: link:../../software/codex-cli '@agentos-software/common': specifier: workspace:* - version: link:../../registry/software/common + version: link:../../software/common '@agentos-software/manifest': specifier: workspace:* version: link:../manifest '@agentos-software/opencode': specifier: workspace:* - version: link:../../registry/agent/opencode + version: link:../../software/opencode '@agentos-software/pi': specifier: workspace:* - version: link:../../registry/agent/pi + version: link:../../software/pi '@agentos-software/pi-cli': specifier: workspace:* - version: link:../../registry/agent/pi-cli + version: link:../../software/pi-cli '@aws-sdk/client-s3': specifier: ^3.1019.0 version: 3.1020.0 @@ -2501,64 +2510,64 @@ importers: devDependencies: '@agentos-software/codex': specifier: workspace:* - version: link:../../registry/agent/codex + version: link:../../software/codex '@agentos-software/coreutils': specifier: workspace:* - version: link:../../registry/software/coreutils + version: link:../../software/coreutils '@agentos-software/curl': specifier: workspace:* - version: link:../../registry/software/curl + version: link:../../software/curl '@agentos-software/diffutils': specifier: workspace:* - version: link:../../registry/software/diffutils + version: link:../../software/diffutils '@agentos-software/duckdb': specifier: workspace:* - version: link:../../registry/software/duckdb + version: link:../../software/duckdb '@agentos-software/fd': specifier: workspace:* - version: link:../../registry/software/fd + version: link:../../software/fd '@agentos-software/file': specifier: workspace:* - version: link:../../registry/software/file + version: link:../../software/file '@agentos-software/findutils': specifier: workspace:* - version: link:../../registry/software/findutils + version: link:../../software/findutils '@agentos-software/gawk': specifier: workspace:* - version: link:../../registry/software/gawk + version: link:../../software/gawk '@agentos-software/git': specifier: workspace:* - version: link:../../registry/software/git + version: link:../../software/git '@agentos-software/grep': specifier: workspace:* - version: link:../../registry/software/grep + version: link:../../software/grep '@agentos-software/gzip': specifier: workspace:* - version: link:../../registry/software/gzip - '@agentos-software/http-get': - specifier: workspace:* - version: link:../../registry/software/http-get + version: link:../../software/gzip '@agentos-software/jq': specifier: workspace:* - version: link:../../registry/software/jq + version: link:../../software/jq '@agentos-software/ripgrep': specifier: workspace:* - version: link:../../registry/software/ripgrep + version: link:../../software/ripgrep '@agentos-software/sed': specifier: workspace:* - version: link:../../registry/software/sed + version: link:../../software/sed '@agentos-software/tar': specifier: workspace:* - version: link:../../registry/software/tar + version: link:../../software/tar '@agentos-software/tree': specifier: workspace:* - version: link:../../registry/software/tree + version: link:../../software/tree '@agentos-software/vim': specifier: workspace:* - version: link:../../registry/software/vim + version: link:../../software/vim + '@agentos-software/wget': + specifier: workspace:* + version: link:../../software/wget '@agentos-software/yq': specifier: workspace:* - version: link:../../registry/software/yq + version: link:../../software/yq '@anthropic-ai/claude-agent-sdk': specifier: ^0.2.87 version: 0.2.87(@cfworker/json-schema@4.1.1)(zod@4.3.6) @@ -2807,79 +2816,73 @@ importers: dependencies: '@agentos-software/codex-cli': specifier: workspace:* - version: link:../../registry/software/codex-cli + version: link:../../software/codex-cli '@agentos-software/coreutils': specifier: workspace:* - version: link:../../registry/software/coreutils + version: link:../../software/coreutils '@agentos-software/curl': specifier: workspace:* - version: link:../../registry/software/curl + version: link:../../software/curl '@agentos-software/diffutils': specifier: workspace:* - version: link:../../registry/software/diffutils + version: link:../../software/diffutils '@agentos-software/duckdb': specifier: workspace:* - version: link:../../registry/software/duckdb + version: link:../../software/duckdb '@agentos-software/fd': specifier: workspace:* - version: link:../../registry/software/fd + version: link:../../software/fd '@agentos-software/file': specifier: workspace:* - version: link:../../registry/software/file + version: link:../../software/file '@agentos-software/findutils': specifier: workspace:* - version: link:../../registry/software/findutils + version: link:../../software/findutils '@agentos-software/gawk': specifier: workspace:* - version: link:../../registry/software/gawk + version: link:../../software/gawk '@agentos-software/git': specifier: workspace:* - version: link:../../registry/software/git + version: link:../../software/git '@agentos-software/grep': specifier: workspace:* - version: link:../../registry/software/grep + version: link:../../software/grep '@agentos-software/gzip': specifier: workspace:* - version: link:../../registry/software/gzip - '@agentos-software/http-get': - specifier: workspace:* - version: link:../../registry/software/http-get + version: link:../../software/gzip '@agentos-software/jq': specifier: workspace:* - version: link:../../registry/software/jq + version: link:../../software/jq '@agentos-software/ripgrep': specifier: workspace:* - version: link:../../registry/software/ripgrep + version: link:../../software/ripgrep '@agentos-software/sed': specifier: workspace:* - version: link:../../registry/software/sed + version: link:../../software/sed '@agentos-software/sqlite3': specifier: workspace:* - version: link:../../registry/software/sqlite3 + version: link:../../software/sqlite3 '@agentos-software/tar': specifier: workspace:* - version: link:../../registry/software/tar + version: link:../../software/tar '@agentos-software/tree': specifier: workspace:* - version: link:../../registry/software/tree + version: link:../../software/tree '@agentos-software/unzip': specifier: workspace:* - version: link:../../registry/software/unzip + version: link:../../software/unzip '@agentos-software/vim': specifier: workspace:* - version: link:../../registry/software/vim - '@agentos-software/vix': - specifier: workspace:* - version: link:../../registry/software/vix + version: link:../../software/vim '@agentos-software/wget': specifier: workspace:* - version: link:../../registry/software/wget + version: link:../../software/wget '@agentos-software/yq': specifier: workspace:* - version: link:../../registry/software/yq + version: link:../../software/yq '@agentos-software/zip': specifier: workspace:* - version: link:../../registry/software/zip + version: link:../../software/zip '@rivet-dev/agentos': specifier: workspace:* version: link:../agentos @@ -2924,127 +2927,127 @@ importers: specifier: ^2.1.8 version: 2.1.9(@types/node@22.19.15) - registry/agent/claude: + packages/vm-test-harness: dependencies: - '@agentclientprotocol/sdk': - specifier: ^0.16.1 - version: 0.16.1(zod@4.3.6) - '@anthropic-ai/claude-agent-sdk': - specifier: 0.2.87 - version: 0.2.87(@cfworker/json-schema@4.1.1)(zod@4.3.6) - zod: - specifier: ^4.1.11 - version: 4.3.6 - devDependencies: - '@agentos-software/manifest': - specifier: workspace:* - version: link:../../../packages/manifest - '@rivet-dev/agentos-toolchain': + '@rivet-dev/agentos-runtime-core': specifier: workspace:* - version: link:../../../packages/agentos-toolchain + version: link:../runtime-core + '@xterm/headless': + specifier: ^6.0.0 + version: 6.0.0 + devDependencies: '@types/node': specifier: ^22.10.2 version: 22.19.15 typescript: - specifier: ^5.7.2 + specifier: ^5.9.2 version: 5.9.3 + vitest: + specifier: ^2.1.9 + version: 2.1.9(@types/node@22.19.15) - registry/agent/codex: + scripts/publish: dependencies: - '@agentos-software/codex-cli': - specifier: workspace:* - version: link:../../software/codex-cli + commander: + specifier: ^12.1.0 + version: 12.1.0 + execa: + specifier: ^8.0.1 + version: 8.0.1 + glob: + specifier: ^10.3.10 + version: 10.5.0 + semver: + specifier: ^7.6.0 + version: 7.7.4 devDependencies: - '@agentos-software/manifest': - specifier: workspace:* - version: link:../../../packages/manifest '@types/node': - specifier: ^22.10.2 - version: 22.19.15 + specifier: ^24.3.0 + version: 24.13.2 + '@types/semver': + specifier: ^7.5.8 + version: 7.7.1 + tsx: + specifier: ^4.0.0 + version: 4.21.0 typescript: - specifier: ^5.7.2 + specifier: ^5.5.2 version: 5.9.3 - registry/agent/opencode: + software/browserbase: + dependencies: + browse: + specifier: ^0.9.3 + version: 0.9.3(@cfworker/json-schema@4.1.1)(bufferutil@4.1.0)(patchright-core@1.59.4)(playwright-core@1.59.1) devDependencies: '@agentos-software/manifest': specifier: workspace:* - version: link:../../../packages/manifest + version: link:../../packages/manifest + '@agentos/test-harness': + specifier: workspace:* + version: link:../../test-harness '@rivet-dev/agentos-toolchain': specifier: workspace:* - version: link:../../../packages/agentos-toolchain + version: link:../../packages/agentos-toolchain '@types/node': specifier: ^22.10.2 version: 22.19.15 - bun: - specifier: 1.3.11 - version: 1.3.11 typescript: specifier: ^5.7.2 version: 5.9.3 + vitest: + specifier: ^2.1.9 + version: 2.1.9(@types/node@22.19.15) - registry/agent/pi: + software/build-essential: dependencies: - '@agentclientprotocol/sdk': - specifier: ^0.16.1 - version: 0.16.1(zod@4.3.6) - '@mariozechner/pi-agent-core': - specifier: 0.60.0 - version: 0.60.0(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.3.6))(bufferutil@4.1.0)(ws@8.21.0(bufferutil@4.1.0))(zod@4.3.6) - '@mariozechner/pi-ai': - specifier: 0.60.0 - version: 0.60.0(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.3.6))(bufferutil@4.1.0)(ws@8.21.0(bufferutil@4.1.0))(zod@4.3.6) - '@mariozechner/pi-coding-agent': - specifier: 0.60.0 - version: 0.60.0(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.3.6))(bufferutil@4.1.0)(ws@8.21.0(bufferutil@4.1.0))(zod@4.3.6) - devDependencies: - '@agentos-software/manifest': + '@agentos-software/common': specifier: workspace:* - version: link:../../../packages/manifest - '@rivet-dev/agentos-toolchain': + version: link:../common + '@agentos-software/curl': specifier: workspace:* - version: link:../../../packages/agentos-toolchain - '@types/node': - specifier: ^22.10.2 - version: 22.19.15 - typescript: - specifier: ^5.7.2 - version: 5.9.3 - - registry/agent/pi-cli: - dependencies: - '@mariozechner/pi-coding-agent': - specifier: ^0.60.0 - version: 0.60.0 - pi-acp: - specifier: ^0.0.23 - version: 0.0.23 + version: link:../curl + '@agentos-software/git': + specifier: workspace:* + version: link:../git devDependencies: '@agentos-software/manifest': specifier: workspace:* - version: link:../../../packages/manifest - '@rivet-dev/agentos-toolchain': + version: link:../../packages/manifest + '@agentos/test-harness': specifier: workspace:* - version: link:../../../packages/agentos-toolchain + version: link:../../test-harness '@types/node': specifier: ^22.10.2 version: 22.19.15 typescript: - specifier: ^5.7.2 + specifier: ^5.9.2 version: 5.9.3 + vitest: + specifier: ^2.1.9 + version: 2.1.9(@types/node@22.19.15) - registry/software/browserbase: + software/claude: dependencies: - browse: - specifier: ^0.9.3 - version: 0.9.3(@cfworker/json-schema@4.1.1)(bufferutil@4.1.0)(patchright-core@1.59.4)(playwright-core@1.59.1) + '@agentclientprotocol/sdk': + specifier: ^0.16.1 + version: 0.16.1(zod@4.3.6) + '@anthropic-ai/claude-agent-sdk': + specifier: 0.2.87 + version: 0.2.87(@cfworker/json-schema@4.1.1)(zod@4.3.6) + zod: + specifier: ^4.1.11 + version: 4.3.6 devDependencies: '@agentos-software/manifest': specifier: workspace:* - version: link:../../../packages/manifest + version: link:../../packages/manifest + '@agentos/test-harness': + specifier: workspace:* + version: link:../../test-harness '@rivet-dev/agentos-toolchain': specifier: workspace:* - version: link:../../../packages/agentos-toolchain + version: link:../../packages/agentos-toolchain '@types/node': specifier: ^22.10.2 version: 22.19.15 @@ -3052,44 +3055,47 @@ importers: specifier: ^5.7.2 version: 5.9.3 - registry/software/build-essential: + software/codex: dependencies: - '@agentos-software/common': - specifier: workspace:* - version: link:../common - '@agentos-software/curl': - specifier: workspace:* - version: link:../curl - '@agentos-software/git': + '@agentos-software/codex-cli': specifier: workspace:* - version: link:../git + version: link:../codex-cli devDependencies: '@agentos-software/manifest': specifier: workspace:* - version: link:../../../packages/manifest + version: link:../../packages/manifest + '@agentos/test-harness': + specifier: workspace:* + version: link:../../test-harness '@types/node': specifier: ^22.10.2 version: 22.19.15 typescript: - specifier: ^5.9.2 + specifier: ^5.7.2 version: 5.9.3 - registry/software/codex-cli: + software/codex-cli: devDependencies: '@agentos-software/manifest': specifier: workspace:* - version: link:../../../packages/manifest + version: link:../../packages/manifest + '@agentos/test-harness': + specifier: workspace:* + version: link:../../test-harness '@rivet-dev/agentos-toolchain': specifier: workspace:* - version: link:../../../packages/agentos-toolchain + version: link:../../packages/agentos-toolchain '@types/node': specifier: ^22.10.2 version: 22.19.15 typescript: specifier: ^5.9.2 version: 5.9.3 + vitest: + specifier: ^2.1.9 + version: 2.1.9(@types/node@22.19.15) - registry/software/common: + software/common: dependencies: '@agentos-software/coreutils': specifier: workspace:* @@ -3118,75 +3124,126 @@ importers: devDependencies: '@agentos-software/manifest': specifier: workspace:* - version: link:../../../packages/manifest + version: link:../../packages/manifest + '@agentos/test-harness': + specifier: workspace:* + version: link:../../test-harness '@types/node': specifier: ^22.10.2 version: 22.19.15 typescript: specifier: ^5.9.2 version: 5.9.3 + vitest: + specifier: ^2.1.9 + version: 2.1.9(@types/node@22.19.15) - registry/software/coreutils: + software/coreutils: devDependencies: '@agentos-software/manifest': specifier: workspace:* - version: link:../../../packages/manifest + version: link:../../packages/manifest + '@agentos/test-harness': + specifier: workspace:* + version: link:../../test-harness '@rivet-dev/agentos-toolchain': specifier: workspace:* - version: link:../../../packages/agentos-toolchain + version: link:../../packages/agentos-toolchain '@types/node': specifier: ^22.10.2 version: 22.19.15 typescript: specifier: ^5.9.2 version: 5.9.3 + vitest: + specifier: ^2.1.9 + version: 2.1.9(@types/node@22.19.15) - registry/software/curl: + software/curl: devDependencies: '@agentos-software/manifest': specifier: workspace:* - version: link:../../../packages/manifest + version: link:../../packages/manifest + '@agentos/test-harness': + specifier: workspace:* + version: link:../../test-harness '@rivet-dev/agentos-toolchain': specifier: workspace:* - version: link:../../../packages/agentos-toolchain + version: link:../../packages/agentos-toolchain '@types/node': specifier: ^22.10.2 version: 22.19.15 typescript: specifier: ^5.9.2 version: 5.9.3 + vitest: + specifier: ^2.1.9 + version: 2.1.9(@types/node@22.19.15) - registry/software/diffutils: + software/diffutils: devDependencies: '@agentos-software/manifest': specifier: workspace:* - version: link:../../../packages/manifest + version: link:../../packages/manifest + '@agentos/test-harness': + specifier: workspace:* + version: link:../../test-harness '@rivet-dev/agentos-toolchain': specifier: workspace:* - version: link:../../../packages/agentos-toolchain + version: link:../../packages/agentos-toolchain '@types/node': specifier: ^22.10.2 version: 22.19.15 typescript: specifier: ^5.9.2 version: 5.9.3 + vitest: + specifier: ^2.1.9 + version: 2.1.9(@types/node@22.19.15) - registry/software/duckdb: + software/duckdb: devDependencies: '@agentos-software/manifest': specifier: workspace:* - version: link:../../../packages/manifest + version: link:../../packages/manifest + '@agentos/test-harness': + specifier: workspace:* + version: link:../../test-harness '@rivet-dev/agentos-toolchain': specifier: workspace:* - version: link:../../../packages/agentos-toolchain + version: link:../../packages/agentos-toolchain '@types/node': specifier: ^22.10.2 version: 22.19.15 typescript: specifier: ^5.9.2 version: 5.9.3 + vitest: + specifier: ^2.1.9 + version: 2.1.9(@types/node@22.19.15) - registry/software/everything: + software/envsubst: + devDependencies: + '@agentos-software/manifest': + specifier: workspace:* + version: link:../../packages/manifest + '@agentos/test-harness': + specifier: workspace:* + version: link:../../test-harness + '@rivet-dev/agentos-toolchain': + specifier: workspace:* + version: link:../../packages/agentos-toolchain + '@types/node': + specifier: ^22.10.2 + version: 22.19.15 + typescript: + specifier: ^5.9.2 + version: 5.9.3 + vitest: + specifier: ^2.1.9 + version: 2.1.9(@types/node@22.19.15) + + software/everything: dependencies: '@agentos-software/codex-cli': specifier: workspace:* @@ -3200,6 +3257,12 @@ importers: '@agentos-software/diffutils': specifier: workspace:* version: link:../diffutils + '@agentos-software/duckdb': + specifier: workspace:* + version: link:../duckdb + '@agentos-software/envsubst': + specifier: workspace:* + version: link:../envsubst '@agentos-software/fd': specifier: workspace:* version: link:../fd @@ -3212,6 +3275,9 @@ importers: '@agentos-software/gawk': specifier: workspace:* version: link:../gawk + '@agentos-software/git': + specifier: workspace:* + version: link:../git '@agentos-software/grep': specifier: workspace:* version: link:../grep @@ -3227,6 +3293,9 @@ importers: '@agentos-software/sed': specifier: workspace:* version: link:../sed + '@agentos-software/sqlite3': + specifier: workspace:* + version: link:../sqlite3 '@agentos-software/tar': specifier: workspace:* version: link:../tar @@ -3236,6 +3305,12 @@ importers: '@agentos-software/unzip': specifier: workspace:* version: link:../unzip + '@agentos-software/vim': + specifier: workspace:* + version: link:../vim + '@agentos-software/wget': + specifier: workspace:* + version: link:../wget '@agentos-software/yq': specifier: workspace:* version: link:../yq @@ -3245,340 +3320,489 @@ importers: devDependencies: '@agentos-software/manifest': specifier: workspace:* - version: link:../../../packages/manifest + version: link:../../packages/manifest + '@agentos/test-harness': + specifier: workspace:* + version: link:../../test-harness '@types/node': specifier: ^22.10.2 version: 22.19.15 typescript: specifier: ^5.9.2 version: 5.9.3 + vitest: + specifier: ^2.1.9 + version: 2.1.9(@types/node@22.19.15) - registry/software/fd: + software/fd: devDependencies: '@agentos-software/manifest': specifier: workspace:* - version: link:../../../packages/manifest + version: link:../../packages/manifest + '@agentos/test-harness': + specifier: workspace:* + version: link:../../test-harness '@rivet-dev/agentos-toolchain': specifier: workspace:* - version: link:../../../packages/agentos-toolchain + version: link:../../packages/agentos-toolchain '@types/node': specifier: ^22.10.2 version: 22.19.15 typescript: specifier: ^5.9.2 version: 5.9.3 + vitest: + specifier: ^2.1.9 + version: 2.1.9(@types/node@22.19.15) - registry/software/file: + software/file: devDependencies: '@agentos-software/manifest': specifier: workspace:* - version: link:../../../packages/manifest + version: link:../../packages/manifest + '@agentos/test-harness': + specifier: workspace:* + version: link:../../test-harness '@rivet-dev/agentos-toolchain': specifier: workspace:* - version: link:../../../packages/agentos-toolchain + version: link:../../packages/agentos-toolchain '@types/node': specifier: ^22.10.2 version: 22.19.15 typescript: specifier: ^5.9.2 version: 5.9.3 + vitest: + specifier: ^2.1.9 + version: 2.1.9(@types/node@22.19.15) - registry/software/findutils: + software/findutils: devDependencies: '@agentos-software/manifest': specifier: workspace:* - version: link:../../../packages/manifest + version: link:../../packages/manifest + '@agentos/test-harness': + specifier: workspace:* + version: link:../../test-harness '@rivet-dev/agentos-toolchain': specifier: workspace:* - version: link:../../../packages/agentos-toolchain + version: link:../../packages/agentos-toolchain '@types/node': specifier: ^22.10.2 version: 22.19.15 typescript: specifier: ^5.9.2 version: 5.9.3 + vitest: + specifier: ^2.1.9 + version: 2.1.9(@types/node@22.19.15) - registry/software/gawk: + software/gawk: devDependencies: '@agentos-software/manifest': specifier: workspace:* - version: link:../../../packages/manifest + version: link:../../packages/manifest + '@agentos/test-harness': + specifier: workspace:* + version: link:../../test-harness '@rivet-dev/agentos-toolchain': specifier: workspace:* - version: link:../../../packages/agentos-toolchain + version: link:../../packages/agentos-toolchain '@types/node': specifier: ^22.10.2 version: 22.19.15 typescript: specifier: ^5.9.2 version: 5.9.3 + vitest: + specifier: ^2.1.9 + version: 2.1.9(@types/node@22.19.15) - registry/software/git: + software/git: devDependencies: '@agentos-software/manifest': specifier: workspace:* - version: link:../../../packages/manifest + version: link:../../packages/manifest + '@agentos/test-harness': + specifier: workspace:* + version: link:../../test-harness '@rivet-dev/agentos-toolchain': specifier: workspace:* - version: link:../../../packages/agentos-toolchain + version: link:../../packages/agentos-toolchain '@types/node': specifier: ^22.10.2 version: 22.19.15 typescript: specifier: ^5.9.2 version: 5.9.3 + vitest: + specifier: ^2.1.9 + version: 2.1.9(@types/node@22.19.15) - registry/software/grep: + software/grep: devDependencies: '@agentos-software/manifest': specifier: workspace:* - version: link:../../../packages/manifest + version: link:../../packages/manifest + '@agentos/test-harness': + specifier: workspace:* + version: link:../../test-harness '@rivet-dev/agentos-toolchain': specifier: workspace:* - version: link:../../../packages/agentos-toolchain + version: link:../../packages/agentos-toolchain '@types/node': specifier: ^22.10.2 version: 22.19.15 typescript: specifier: ^5.9.2 version: 5.9.3 + vitest: + specifier: ^2.1.9 + version: 2.1.9(@types/node@22.19.15) - registry/software/gzip: + software/gzip: devDependencies: '@agentos-software/manifest': specifier: workspace:* - version: link:../../../packages/manifest + version: link:../../packages/manifest + '@agentos/test-harness': + specifier: workspace:* + version: link:../../test-harness '@rivet-dev/agentos-toolchain': specifier: workspace:* - version: link:../../../packages/agentos-toolchain + version: link:../../packages/agentos-toolchain '@types/node': specifier: ^22.10.2 version: 22.19.15 typescript: specifier: ^5.9.2 version: 5.9.3 + vitest: + specifier: ^2.1.9 + version: 2.1.9(@types/node@22.19.15) - registry/software/http-get: + software/jq: devDependencies: '@agentos-software/manifest': specifier: workspace:* - version: link:../../../packages/manifest + version: link:../../packages/manifest + '@agentos/test-harness': + specifier: workspace:* + version: link:../../test-harness '@rivet-dev/agentos-toolchain': specifier: workspace:* - version: link:../../../packages/agentos-toolchain + version: link:../../packages/agentos-toolchain '@types/node': specifier: ^22.10.2 version: 22.19.15 typescript: specifier: ^5.9.2 version: 5.9.3 + vitest: + specifier: ^2.1.9 + version: 2.1.9(@types/node@22.19.15) - registry/software/jq: + software/opencode: devDependencies: '@agentos-software/manifest': specifier: workspace:* - version: link:../../../packages/manifest + version: link:../../packages/manifest + '@agentos/test-harness': + specifier: workspace:* + version: link:../../test-harness '@rivet-dev/agentos-toolchain': specifier: workspace:* - version: link:../../../packages/agentos-toolchain + version: link:../../packages/agentos-toolchain '@types/node': specifier: ^22.10.2 version: 22.19.15 + bun: + specifier: 1.3.11 + version: 1.3.11 typescript: - specifier: ^5.9.2 + specifier: ^5.7.2 version: 5.9.3 + vitest: + specifier: ^2.1.9 + version: 2.1.9(@types/node@22.19.15) - registry/software/ripgrep: + software/pi: + dependencies: + '@agentclientprotocol/sdk': + specifier: ^0.16.1 + version: 0.16.1(zod@4.3.6) + '@mariozechner/pi-ai': + specifier: 0.60.0 + version: 0.60.0(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.3.6))(bufferutil@4.1.0)(ws@8.21.0(bufferutil@4.1.0))(zod@4.3.6) + '@mariozechner/pi-coding-agent': + specifier: 0.60.0 + version: 0.60.0(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.3.6))(bufferutil@4.1.0)(ws@8.21.0(bufferutil@4.1.0))(zod@4.3.6) devDependencies: '@agentos-software/manifest': specifier: workspace:* - version: link:../../../packages/manifest + version: link:../../packages/manifest + '@agentos/test-harness': + specifier: workspace:* + version: link:../../test-harness '@rivet-dev/agentos-toolchain': specifier: workspace:* - version: link:../../../packages/agentos-toolchain + version: link:../../packages/agentos-toolchain '@types/node': specifier: ^22.10.2 version: 22.19.15 + esbuild: + specifier: ^0.27.4 + version: 0.27.4 typescript: - specifier: ^5.9.2 + specifier: ^5.7.2 version: 5.9.3 - registry/software/sed: + software/pi-cli: + dependencies: + '@mariozechner/pi-coding-agent': + specifier: ^0.60.0 + version: 0.60.0 + pi-acp: + specifier: ^0.0.23 + version: 0.0.23 devDependencies: '@agentos-software/manifest': specifier: workspace:* - version: link:../../../packages/manifest + version: link:../../packages/manifest + '@agentos/test-harness': + specifier: workspace:* + version: link:../../test-harness + '@rivet-dev/agentos-toolchain': + specifier: workspace:* + version: link:../../packages/agentos-toolchain + '@types/node': + specifier: ^22.10.2 + version: 22.19.15 + typescript: + specifier: ^5.7.2 + version: 5.9.3 + vitest: + specifier: ^2.1.9 + version: 2.1.9(@types/node@22.19.15) + + software/ripgrep: + devDependencies: + '@agentos-software/manifest': + specifier: workspace:* + version: link:../../packages/manifest + '@agentos/test-harness': + specifier: workspace:* + version: link:../../test-harness '@rivet-dev/agentos-toolchain': specifier: workspace:* - version: link:../../../packages/agentos-toolchain + version: link:../../packages/agentos-toolchain '@types/node': specifier: ^22.10.2 version: 22.19.15 typescript: specifier: ^5.9.2 version: 5.9.3 + vitest: + specifier: ^2.1.9 + version: 2.1.9(@types/node@22.19.15) - registry/software/sqlite3: + software/sed: devDependencies: '@agentos-software/manifest': specifier: workspace:* - version: link:../../../packages/manifest + version: link:../../packages/manifest + '@agentos/test-harness': + specifier: workspace:* + version: link:../../test-harness '@rivet-dev/agentos-toolchain': specifier: workspace:* - version: link:../../../packages/agentos-toolchain + version: link:../../packages/agentos-toolchain '@types/node': specifier: ^22.10.2 version: 22.19.15 typescript: specifier: ^5.9.2 version: 5.9.3 + vitest: + specifier: ^2.1.9 + version: 2.1.9(@types/node@22.19.15) - registry/software/tar: + software/sqlite3: devDependencies: '@agentos-software/manifest': specifier: workspace:* - version: link:../../../packages/manifest + version: link:../../packages/manifest + '@agentos/test-harness': + specifier: workspace:* + version: link:../../test-harness '@rivet-dev/agentos-toolchain': specifier: workspace:* - version: link:../../../packages/agentos-toolchain + version: link:../../packages/agentos-toolchain '@types/node': specifier: ^22.10.2 version: 22.19.15 typescript: specifier: ^5.9.2 version: 5.9.3 + vitest: + specifier: ^2.1.9 + version: 2.1.9(@types/node@22.19.15) - registry/software/tree: + software/tar: devDependencies: '@agentos-software/manifest': specifier: workspace:* - version: link:../../../packages/manifest + version: link:../../packages/manifest + '@agentos/test-harness': + specifier: workspace:* + version: link:../../test-harness '@rivet-dev/agentos-toolchain': specifier: workspace:* - version: link:../../../packages/agentos-toolchain + version: link:../../packages/agentos-toolchain '@types/node': specifier: ^22.10.2 version: 22.19.15 typescript: specifier: ^5.9.2 version: 5.9.3 + vitest: + specifier: ^2.1.9 + version: 2.1.9(@types/node@22.19.15) - registry/software/unzip: + software/tree: devDependencies: '@agentos-software/manifest': specifier: workspace:* - version: link:../../../packages/manifest + version: link:../../packages/manifest + '@agentos/test-harness': + specifier: workspace:* + version: link:../../test-harness '@rivet-dev/agentos-toolchain': specifier: workspace:* - version: link:../../../packages/agentos-toolchain + version: link:../../packages/agentos-toolchain '@types/node': specifier: ^22.10.2 version: 22.19.15 typescript: specifier: ^5.9.2 version: 5.9.3 + vitest: + specifier: ^2.1.9 + version: 2.1.9(@types/node@22.19.15) - registry/software/vim: + software/unzip: devDependencies: '@agentos-software/manifest': specifier: workspace:* - version: link:../../../packages/manifest + version: link:../../packages/manifest + '@agentos/test-harness': + specifier: workspace:* + version: link:../../test-harness '@rivet-dev/agentos-toolchain': specifier: workspace:* - version: link:../../../packages/agentos-toolchain + version: link:../../packages/agentos-toolchain '@types/node': specifier: ^22.10.2 version: 22.19.15 typescript: specifier: ^5.9.2 version: 5.9.3 + vitest: + specifier: ^2.1.9 + version: 2.1.9(@types/node@22.19.15) - registry/software/vix: + software/vim: devDependencies: '@agentos-software/manifest': specifier: workspace:* - version: link:../../../packages/manifest + version: link:../../packages/manifest + '@agentos/test-harness': + specifier: workspace:* + version: link:../../test-harness '@rivet-dev/agentos-toolchain': specifier: workspace:* - version: link:../../../packages/agentos-toolchain + version: link:../../packages/agentos-toolchain '@types/node': specifier: ^22.10.2 version: 22.19.15 typescript: specifier: ^5.9.2 version: 5.9.3 + vitest: + specifier: ^2.1.9 + version: 2.1.9(@types/node@22.19.15) - registry/software/wget: + software/wget: devDependencies: '@agentos-software/manifest': specifier: workspace:* - version: link:../../../packages/manifest + version: link:../../packages/manifest + '@agentos/test-harness': + specifier: workspace:* + version: link:../../test-harness '@rivet-dev/agentos-toolchain': specifier: workspace:* - version: link:../../../packages/agentos-toolchain + version: link:../../packages/agentos-toolchain '@types/node': specifier: ^22.10.2 version: 22.19.15 typescript: specifier: ^5.9.2 version: 5.9.3 + vitest: + specifier: ^2.1.9 + version: 2.1.9(@types/node@22.19.15) - registry/software/yq: + software/yq: devDependencies: '@agentos-software/manifest': specifier: workspace:* - version: link:../../../packages/manifest + version: link:../../packages/manifest + '@agentos/test-harness': + specifier: workspace:* + version: link:../../test-harness '@rivet-dev/agentos-toolchain': specifier: workspace:* - version: link:../../../packages/agentos-toolchain + version: link:../../packages/agentos-toolchain '@types/node': specifier: ^22.10.2 version: 22.19.15 typescript: specifier: ^5.9.2 version: 5.9.3 + vitest: + specifier: ^2.1.9 + version: 2.1.9(@types/node@22.19.15) - registry/software/zip: + software/zip: devDependencies: '@agentos-software/manifest': specifier: workspace:* - version: link:../../../packages/manifest + version: link:../../packages/manifest + '@agentos/test-harness': + specifier: workspace:* + version: link:../../test-harness '@rivet-dev/agentos-toolchain': specifier: workspace:* - version: link:../../../packages/agentos-toolchain + version: link:../../packages/agentos-toolchain '@types/node': specifier: ^22.10.2 version: 22.19.15 typescript: specifier: ^5.9.2 version: 5.9.3 + vitest: + specifier: ^2.1.9 + version: 2.1.9(@types/node@22.19.15) - scripts/publish: + test-harness: dependencies: - commander: - specifier: ^12.1.0 - version: 12.1.0 - execa: - specifier: ^8.0.1 - version: 8.0.1 - glob: - specifier: ^10.3.10 - version: 10.5.0 - semver: - specifier: ^7.6.0 - version: 7.7.4 + '@rivet-dev/agentos-vm-test-harness': + specifier: workspace:* + version: link:../packages/vm-test-harness devDependencies: - '@types/node': - specifier: ^24.3.0 - version: 24.13.2 - '@types/semver': - specifier: ^7.5.8 - version: 7.7.1 - tsx: - specifier: ^4.0.0 - version: 4.21.0 typescript: - specifier: ^5.5.2 + specifier: ^5.9.2 version: 5.9.3 packages: @@ -7573,6 +7797,77 @@ packages: resolution: {integrity: sha512-ppwTtiJZq0O/ai0z7yfudtBpWIoxM8yE6nHi1X47eFR2EWORqfbu6CnPlNsjeN683eT0qG6H/Pyf9fCcvjnnnQ==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + npm@11.18.0: + resolution: {integrity: sha512-T67M4L5wNm0cZ7EBLErcEkY1SmzEW/WJ+SADBzsFUY1UdAPfFHXFQtZ6SEXiK0+vzXysCvAsepbMaBTwnrAD+w==} + engines: {node: ^20.17.0 || >=22.9.0} + hasBin: true + bundledDependencies: + - '@isaacs/string-locale-compare' + - '@npmcli/arborist' + - '@npmcli/config' + - '@npmcli/fs' + - '@npmcli/map-workspaces' + - '@npmcli/metavuln-calculator' + - '@npmcli/package-json' + - '@npmcli/promise-spawn' + - '@npmcli/redact' + - '@npmcli/run-script' + - '@sigstore/tuf' + - abbrev + - archy + - cacache + - chalk + - ci-info + - fastest-levenshtein + - fs-minipass + - glob + - graceful-fs + - hosted-git-info + - ini + - init-package-json + - is-cidr + - json-parse-even-better-errors + - libnpmaccess + - libnpmdiff + - libnpmexec + - libnpmfund + - libnpmorg + - libnpmpack + - libnpmpublish + - libnpmsearch + - libnpmteam + - libnpmversion + - make-fetch-happen + - minimatch + - minipass + - minipass-pipeline + - ms + - node-gyp + - nopt + - npm-audit-report + - npm-install-checks + - npm-package-arg + - npm-pick-manifest + - npm-profile + - npm-registry-fetch + - npm-user-validate + - p-map + - pacote + - parse-conflict-json + - proc-log + - qrcode-terminal + - read + - semver + - spdx-expression-parse + - ssri + - supports-color + - tar + - text-table + - tiny-relative-date + - treeverse + - validate-npm-package-name + - which + nth-check@2.1.1: resolution: {integrity: sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==} @@ -13541,6 +13836,8 @@ snapshots: dependencies: path-key: 4.0.0 + npm@11.18.0: {} + nth-check@2.1.1: dependencies: boolbase: 1.0.0 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index abbcff2e92..624818f06b 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -15,14 +15,14 @@ packages: - packages/runtime-core - packages/runtime-sidecar - packages/runtime-sidecar/npm/* + - packages/vm-test-harness - packages/secure-exec - packages/secure-exec-example-ai-agent-type-check - packages/typescript - packages/shell - packages/sidecar-binary - - registry/agent/* - - registry/software/* - - registry/tool/* + - software/* + - test-harness - examples/* # Nested example packages (e.g. examples/quickstart/hello-world) so every # embeddable example is a workspace member that `turbo check-types` covers. diff --git a/registry/CONTRIBUTING.md b/registry/CONTRIBUTING.md deleted file mode 100644 index 1caef14b93..0000000000 --- a/registry/CONTRIBUTING.md +++ /dev/null @@ -1,99 +0,0 @@ -# Contributing a Registry Package - -Software and agent packages for agentOS VMs, published under the -`@agentos-software/*` npm scope. This is the quick path to adding one; the -full documentation lives on the website: - -- [Software Definition](https://agentos-sdk.dev/docs/custom-software/definition) — package anatomy and manifest fields -- [Building Binaries](https://agentos-sdk.dev/docs/custom-software/building-wasm) — compiling commands to WASM -- [Publishing Packages](https://agentos-sdk.dev/docs/custom-software/publishing) — shipping to npm with the toolchain - -## File structure - -``` -registry/ -├── agent// JavaScript agent adapters (claude, codex, opencode, pi) -├── software// WASM command packages (jq, ripgrep, git, …) -└── native/ toolchains that compile the command binaries -``` - -Each package contains: - -``` -registry/software// -├── package.json name, per-package semver, build script -├── agentos-package.json runtime manifest (commands/aliases/provides) + -│ `registry` block (title/description/priority/image) -│ that lists the package on agentos-sdk.dev/registry -├── src/index.ts descriptor export consumed by `software: []` -├── bin/ staged binaries (gitignored, built) -└── dist/package/ assembled runtime dir shipped in the npm tarball -``` - -## Building - -From the repo root: - -```bash -just registry-native # compile the fast native wasm command gate -just registry-native-cmd # build one command (required for git, duckdb, vim, wget, codex) -pnpm --filter @agentos-software/ build # stage bin/ + assemble one package -just registry-build # stage + assemble every software package -just registry-status # per-package state -just registry-test # registry integration tests -``` - -See [Building Binaries](https://agentos-sdk.dev/docs/custom-software/building-wasm) -for toolchain details (Rust vs C builds, the patched WASI sysroot). - -## Adding a package - -1. Copy an existing package of the same kind (`software/jq` is a minimal - example) to `registry/software//`. -2. Add the command source under `registry/native/` (Rust: a - `crates/commands/` crate; C: wire it into `registry/native/c`). -3. Fill in `agentos-package.json`: `commands` (and `aliases`/`provides` if - needed) plus a `registry` block with `title` and `description` — without - that block the package is not listed on the website registry page. -4. Register the directory in `pnpm-workspace.yaml` (it is covered by the - `registry/software/*` glob) and run `pnpm install`. -5. Run `pnpm --filter @agentos-software/ build` and check - `just registry-status`. - -See [Software Definition](https://agentos-sdk.dev/docs/custom-software/definition) -for every manifest field. - -## Testing in an external project - -Inside this repo, tests and examples resolve packages via `workspace:*` — no -publishing needed. To try a package in an external project, pack the built -tarball and install it by path: - -```bash -cd registry/software/ -npm pack # produces agentos-software--.tgz -cd /path/to/your-project -npm install /path/to/agentos-software--.tgz -``` - -Then register it in your VM and run a command: - -```typescript -import myPkg from "@agentos-software/"; -const vm = agentOS({ software: [myPkg] }); -``` - -Real publishes go through `agentos-toolchain publish` (dist-tag `dev` by -default) — see [Publishing Packages](https://agentos-sdk.dev/docs/custom-software/publishing). - -## Opening a PR - -- Branch, commit with a plain conventional-commit title - (`feat(registry): add package`), no agent attribution. -- Include: the package directory, the native build wiring, and the - `registry` block so the website picks it up. -- Keep the package version at its own semver (packages version - independently); never touch other packages' versions or the `latest` - dist-tag. -- Cheap gates before pushing: `cargo check --workspace`, `pnpm build`, - `pnpm check-types`, and `pnpm --filter @agentos-software/ build`. diff --git a/registry/README.md b/registry/README.md deleted file mode 100644 index a4b904a248..0000000000 --- a/registry/README.md +++ /dev/null @@ -1,168 +0,0 @@ -# secure-exec Registry - -Software packages for secure-exec VMs: WASM command binaries -(`registry/software/*`), JavaScript agent adapters (`registry/agent/*`), and -tool packages (`registry/tool/*`). Everything under the `@agentos-software/*` -npm scope. - -## Consuming packages - -```bash -npm install @agentos-software/coreutils @agentos-software/grep -# or a meta-package for a complete set: -npm install @agentos-software/common -``` - -Each package default-exports a descriptor whose `packageDir` points at the -self-contained runtime dir the sidecar projects under -`/opt/agentos//` (meta-packages export an array of descriptors): - -```typescript -import coreutils from "@agentos-software/coreutils"; -import grep from "@agentos-software/grep"; - -export const software = [coreutils, grep]; -``` - -## Package anatomy - -``` -registry/software// -├── package.json name, per-package semver version, build script -├── agentos-package.json manifest: runtime fields (name/agent/provides) + -│ staging fields (commands/aliases/stubs) -├── src/index.ts descriptor: packageDir -> ./package/ (dist/package) -├── bin/ staged command binaries (generated build output) -└── dist/package/ the assembled runtime dir (shipped in the npm tarball): - ├── package.json { name, version, bin: { : "bin/" } } - ├── agentos-package.json - └── bin/ the binaries, copied verbatim -``` - -The whole lifecycle is owned by **`@rivet-dev/agentos-toolchain`** -(`packages/agentos-toolchain`) — the same CLI 3rd-party repos use to build and -publish their own agentOS packages (`npx @rivet-dev/agentos-toolchain`): - -- `stage --commands-dir ` — populate `bin/` from a compiled commands - directory, per the `commands` / `aliases` / `stubs` lists in - `agentos-package.json`. -- `build` — assemble the clean `dist/package/` runtime dir from `bin/`. -- `pack` — build a self-contained node-closure package (JS agents). -- `publish` — publish to npm; dist-tag `dev` by default, `latest` only with an - explicit `--latest`. - -## Building - -All recipes run from the repo root (see `justfile`): - -```bash -just registry-native # compile the fast native wasm command gate -just registry-native-cmd # build ONE command binary, whatever its toolchain -just registry-build # stage + assemble every registry package -pnpm --filter @agentos-software/coreutils build:runtime # assemble runnable coreutils -just registry-status # per-package state; --remote adds npm dist-tags -just registry-test # registry integration tests (registry/tests) -``` - -### Building coreutils from a clean checkout - -`registry/software/coreutils/bin/` is deliberately gitignored and must never be -committed. A fresh checkout has no runnable coreutils package: VMs, browser -demos, and tests that use `sh`, `cat`, `chmod`, or the other bundled commands -will not work until the native WASM commands are compiled and staged. - -Run the complete build from the repository root: - -```bash -pnpm install --frozen-lockfile -just registry-native -pnpm --filter @agentos-software/coreutils build:runtime -``` - -The native build compiles the patched Rust and C WASI sources into -`registry/native/target/wasm32-wasip1/release/commands/`. `build:runtime` -strictly stages every command, alias, and stub into -`registry/software/coreutils/bin/`, then assembles `dist/package/`. It fails if -the native command set is absent or incomplete; an empty placeholder is not a -usable coreutils package. - -The ordinary `build` script retains `--if-missing skip` so repository-wide -TypeScript builds can run without first compiling the WASM toolchain. On a -source-only checkout that script assembles an empty placeholder. It does not -make `sh` or any coreutils command available and is not a runtime build. -Publishing coreutils runs `build:runtime` through its `prepublishOnly` lifecycle -and fails rather than publishing an empty or incomplete command package. - -`just registry-native-cmd sh` is useful when iterating on only the shell, but it -does not build the other commands and therefore is not enough to assemble -coreutils. Use the complete sequence above for demos, packaging, and E2E tests. - -`registry-native-cmd` (= `make -C registry/native cmd/`) is the uniform -per-binary entry point; it dispatches to whichever toolchain owns the command: - -| kind | commands | what it runs | -|---|---|---| -| Rust | any `crates/commands/` (sh, ls, rg, …) | `cargo build -p cmd-` (build-std) + `wasm-opt` | -| C | `zip unzip envsubst sqlite3 curl wget duckdb` | `make -C c sysroot build/` + per-command install | -| codex | `codex`, `codex-exec` | the codex fork build (needs the fork checkout) | -| C | `vim` (pinned upstream clone + bridge in `c/vim/`) | `make -C c sysroot build/vim` + install | -| external | `vix` | validates the hand-built binary is in the drop zone; errors with instructions otherwise | - -The default native build (`registry/native`) compiles the fast command gate to -`wasm32-wasip1` with a patched std (`-Z build-std`, `patches/`), runs -`wasm-opt -O3`, and drops the binaries in -`registry/native/target/wasm32-wasip1/release/commands/`. The bulk gate -intentionally excludes slow/heavy or non-default commands: `git`, `duckdb`, -`vim`, `wget`, and the external `codex`/`codex-exec` fork build. Build those explicitly with -`just registry-native-cmd ` when working on them. Package builds then run -`agentos-toolchain stage`, followed by `tsc` and `agentos-toolchain build`. -Use coreutils `build:runtime` for strict staging as described above; software -packages may use skip mode for source-only checks or commands outside the -default native gate. - -Within this repo, everything consumes the LOCAL builds by default: the registry -packages are pnpm workspace members, so tests and examples resolve them via -`workspace:*` — no publish needed for local development. - -Exceptions: -- `software/codex/wasm/` is the install target of the codex fork's build - (`make -C registry/native codex`); `software/codex-cli` stages from it. -- C-built commands (sqlite3, zip, unzip, curl, wget, duckdb) need the patched - sysroot; `just registry-native-cmd ` builds it on demand. Without it - those packages stay empty placeholders. -- `vim` builds from source: `just registry-native-cmd vim` clones the pinned - vim tag and compiles it against the patched sysroot + the termios/termcap - bridge in `registry/native/c/vim/` (its runtime tree is staged by the - package `scripts/stage-runtime.mjs` and applied via manifest `provides`). -- `vix` is the one remaining external drop-zone binary (no source pipeline): - place the hand-built wasm at `registry/native/target/.../commands/vix`. - -## Publishing - -Packages **version independently** (per-package semver in each -`package.json`). Publishing NEVER moves the `latest` dist-tag unless asked: - -```bash -just registry-publish coreutils # publish @agentos-software/coreutils under dist-tag `dev` -just registry-publish coreutils my-branch # ... under a custom tag -just registry-publish coreutils latest # DELIBERATE release: moves `latest` -just registry-publish-all # every built software package, dist-tag `dev` -``` - -Bump the package's `version` in its `package.json` (commit it) before -publishing. CI does not publish these packages (the publish workflow's package -discovery skips `@agentos-software/*` except the manifest); the agent packages -under `registry/agent/*` preview-publish via `.github/workflows/publish.yaml` -under a branch dist-tag. - -agent-os consumes the published packages pinned per-package in its catalog -(`just agentos-pkgs-status` there), and flips to these local checkouts with -`just agentos-pkgs-local`. - -## Contributing - -See [CONTRIBUTING.md](CONTRIBUTING.md) for how to add new packages. - -## License - -Apache-2.0 diff --git a/registry/agent/claude/agentos-package.json b/registry/agent/claude/agentos-package.json deleted file mode 100644 index 5d09a62725..0000000000 --- a/registry/agent/claude/agentos-package.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "name": "claude", - "agent": { - "acpEntrypoint": "claude-sdk-acp", - "env": { - "CLAUDE_AGENT_SDK_CLIENT_APP": "@rivet-dev/agentos", - "CLAUDE_CODE_SIMPLE": "1", - "CLAUDE_CODE_FORCE_AGENT_OS_RIPGREP": "1", - "CLAUDE_CODE_DEFER_GROWTHBOOK_INIT": "1", - "CLAUDE_CODE_DISABLE_CWD_PERSIST": "1", - "CLAUDE_CODE_DISABLE_DEV_NULL_REDIRECT": "1", - "CLAUDE_CODE_NODE_SHELL_WRAPPER": "1", - "CLAUDE_CODE_DISABLE_STREAM_JSON_HOOK_EVENTS": "1", - "CLAUDE_CODE_SHELL": "/bin/sh", - "CLAUDE_CODE_SKIP_INITIAL_MESSAGES": "1", - "CLAUDE_CODE_SKIP_SANDBOX_INIT": "1", - "CLAUDE_CODE_SIMPLE_SHELL_EXEC": "1", - "CLAUDE_CODE_SWAP_STDIO": "0", - "CLAUDE_CODE_USE_PIPE_OUTPUT": "1", - "DISABLE_TELEMETRY": "1", - "SHELL": "/bin/sh", - "USE_BUILTIN_RIPGREP": "0" - } - }, - "registry": { - "slug": "claude-code", - "title": "Claude Code", - "description": "Run Claude Code as an agentOS agent with full tool access, file editing, and shell execution.", - "beta": true, - "docsHref": "/docs/agents/claude", - "image": "/images/registry/claude-code.svg", - "priority": 90 - } -} diff --git a/registry/agent/claude/package.json b/registry/agent/claude/package.json deleted file mode 100644 index 3b056fe9be..0000000000 --- a/registry/agent/claude/package.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "name": "@agentos-software/claude-code", - "version": "0.2.1", - "type": "module", - "license": "Apache-2.0", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", - "bin": { - "claude": "./dist/claude-cli.mjs", - "claude-sdk-acp": "./dist/adapter.js" - }, - "exports": { - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js", - "default": "./dist/index.js" - } - }, - "files": [ - "dist", - "!dist/package", - "!dist/package.tar", - "agentos-package.json" - ], - "scripts": { - "build": "tsc && node ./scripts/build-patched-cli.mjs && rm -rf dist/package && agentos-toolchain pack . --out dist/package --agent claude-sdk-acp --prune-native", - "check-types": "tsc --noEmit", - "test": "pnpm build && node --test --test-force-exit tests/*.test.mjs" - }, - "dependencies": { - "@agentclientprotocol/sdk": "^0.16.1", - "@anthropic-ai/claude-agent-sdk": "0.2.87", - "zod": "^4.1.11" - }, - "devDependencies": { - "@agentos-software/manifest": "workspace:*", - "@rivet-dev/agentos-toolchain": "workspace:*", - "@types/node": "^22.10.2", - "typescript": "^5.7.2" - } -} diff --git a/registry/agent/claude/tsconfig.json b/registry/agent/claude/tsconfig.json deleted file mode 100644 index bff7313256..0000000000 --- a/registry/agent/claude/tsconfig.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "extends": "../../../tsconfig.base.json", - "compilerOptions": { - "declaration": true, - "outDir": "./dist", - "rootDir": "./src" - }, - "include": ["src/**/*"], - "exclude": ["node_modules", "dist"] -} diff --git a/registry/agent/codex/agentos-package.json b/registry/agent/codex/agentos-package.json deleted file mode 100644 index a99b543f00..0000000000 --- a/registry/agent/codex/agentos-package.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "registry": { - "title": "Codex", - "description": "Run OpenAI's Codex coding agent inside agentOS with programmatic API access.", - "beta": true, - "docsHref": "/docs/agents/codex", - "image": "/images/registry/codex.svg", - "priority": 80 - } -} diff --git a/registry/agent/codex/package.json b/registry/agent/codex/package.json deleted file mode 100644 index bdc2e759a0..0000000000 --- a/registry/agent/codex/package.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "name": "@agentos-software/codex", - "version": "0.2.0-rc.3", - "type": "module", - "license": "Apache-2.0", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", - "exports": { - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js", - "default": "./dist/index.js" - } - }, - "scripts": { - "build": "tsc", - "check-types": "tsc --noEmit", - "test": "pnpm build && node --test tests/*.test.mjs" - }, - "dependencies": { - "@agentos-software/codex-cli": "workspace:*" - }, - "devDependencies": { - "@agentos-software/manifest": "workspace:*", - "@types/node": "^22.10.2", - "typescript": "^5.7.2" - } -} diff --git a/registry/agent/codex/tsconfig.json b/registry/agent/codex/tsconfig.json deleted file mode 100644 index bff7313256..0000000000 --- a/registry/agent/codex/tsconfig.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "extends": "../../../tsconfig.base.json", - "compilerOptions": { - "declaration": true, - "outDir": "./dist", - "rootDir": "./src" - }, - "include": ["src/**/*"], - "exclude": ["node_modules", "dist"] -} diff --git a/registry/agent/opencode/agentos-package.json b/registry/agent/opencode/agentos-package.json deleted file mode 100644 index cb9f08643a..0000000000 --- a/registry/agent/opencode/agentos-package.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "name": "opencode", - "agent": { - "acpEntrypoint": "agentos-opencode-acp", - "env": { - "OPENCODE_DISABLE_CONFIG_DEP_INSTALL": "1", - "OPENCODE_DISABLE_EMBEDDED_WEB_UI": "1" - } - }, - "registry": { - "title": "OpenCode", - "description": "Run OpenCode, an open-source coding agent, inside agentOS.", - "docsHref": "/docs/agents/opencode", - "image": "/images/registry/opencode.svg", - "priority": 70 - } -} diff --git a/registry/agent/opencode/package.json b/registry/agent/opencode/package.json deleted file mode 100644 index c3ab2d9c5c..0000000000 --- a/registry/agent/opencode/package.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "name": "@agentos-software/opencode", - "version": "0.2.1", - "type": "module", - "license": "Apache-2.0", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", - "bin": { - "agentos-opencode-acp": "./dist/adapter.js" - }, - "exports": { - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js", - "default": "./dist/index.js" - } - }, - "files": [ - "dist", - "!dist/package", - "!dist/package.tar", - "agentos-package.json" - ], - "scripts": { - "build": "node ./scripts/build-opencode-acp.mjs && tsc && rm -rf dist/package dist/package.tar && agentos-toolchain pack . --out dist/package --agent agentos-opencode-acp", - "check-types": "tsc --noEmit" - }, - "devDependencies": { - "@agentos-software/manifest": "workspace:*", - "@rivet-dev/agentos-toolchain": "workspace:*", - "@types/node": "^22.10.2", - "bun": "1.3.11", - "typescript": "^5.7.2" - } -} diff --git a/registry/agent/opencode/tsconfig.json b/registry/agent/opencode/tsconfig.json deleted file mode 100644 index bff7313256..0000000000 --- a/registry/agent/opencode/tsconfig.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "extends": "../../../tsconfig.base.json", - "compilerOptions": { - "declaration": true, - "outDir": "./dist", - "rootDir": "./src" - }, - "include": ["src/**/*"], - "exclude": ["node_modules", "dist"] -} diff --git a/registry/agent/pi-cli/agentos-package.json b/registry/agent/pi-cli/agentos-package.json deleted file mode 100644 index fdd6c08d5f..0000000000 --- a/registry/agent/pi-cli/agentos-package.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "name": "pi-cli", - "agent": { - "acpEntrypoint": "pi-acp", - "env": { - "PI_ACP_PI_COMMAND": "pi" - } - } -} diff --git a/registry/agent/pi-cli/package.json b/registry/agent/pi-cli/package.json deleted file mode 100644 index 080878fbdc..0000000000 --- a/registry/agent/pi-cli/package.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "name": "@agentos-software/pi-cli", - "version": "0.2.1", - "type": "module", - "license": "Apache-2.0", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", - "bin": { - "pi-acp": "./node_modules/pi-acp/dist/index.js", - "pi": "./node_modules/@mariozechner/pi-coding-agent/dist/cli.js" - }, - "exports": { - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js", - "default": "./dist/index.js" - } - }, - "files": [ - "dist", - "!dist/package", - "!dist/package.tar", - "agentos-package.json" - ], - "scripts": { - "build": "tsc && rm -rf dist/package dist/package.tar && agentos-toolchain pack . --out dist/package --agent pi-acp --prune-native", - "check-types": "tsc --noEmit" - }, - "dependencies": { - "@mariozechner/pi-coding-agent": "^0.60.0", - "pi-acp": "^0.0.23" - }, - "devDependencies": { - "@agentos-software/manifest": "workspace:*", - "@rivet-dev/agentos-toolchain": "workspace:*", - "@types/node": "^22.10.2", - "typescript": "^5.7.2" - } -} diff --git a/registry/agent/pi-cli/tsconfig.json b/registry/agent/pi-cli/tsconfig.json deleted file mode 100644 index bff7313256..0000000000 --- a/registry/agent/pi-cli/tsconfig.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "extends": "../../../tsconfig.base.json", - "compilerOptions": { - "declaration": true, - "outDir": "./dist", - "rootDir": "./src" - }, - "include": ["src/**/*"], - "exclude": ["node_modules", "dist"] -} diff --git a/registry/agent/pi/adapter-browser-entry.mjs b/registry/agent/pi/adapter-browser-entry.mjs deleted file mode 100644 index 2e621a03b7..0000000000 --- a/registry/agent/pi/adapter-browser-entry.mjs +++ /dev/null @@ -1,41 +0,0 @@ -// Browser-bundle entry for the pi ACP adapter. The adapter normally loads its SDK via -// computed dynamic import() from a VM-mounted node_modules; in the browser converged -// executor there is no such mount, so we statically import the SDK submodules (which -// esbuild bundles into a single self-contained file) and publish the same -// `__PI_SDK_RUNTIME__` object used by the native V8 startup snapshot. The adapter is -// otherwise unchanged — same ACP behavior, just bundled instead of VFS-resolved. -// -// Imports use relative node_modules file paths (not bare package subpaths) so esbuild -// resolves the dist files directly, bypassing the packages' restrictive `exports`. -import * as agentCore from "./node_modules/@mariozechner/pi-agent-core/dist/index.js"; -import * as authStorage from "./node_modules/@mariozechner/pi-coding-agent/dist/core/auth-storage.js"; -import * as config from "./node_modules/@mariozechner/pi-coding-agent/dist/config.js"; -import * as defaults from "./node_modules/@mariozechner/pi-coding-agent/dist/core/defaults.js"; -import * as messages from "./node_modules/@mariozechner/pi-coding-agent/dist/core/messages.js"; -import * as modelRegistry from "./node_modules/@mariozechner/pi-coding-agent/dist/core/model-registry.js"; -import * as resourceLoader from "./node_modules/@mariozechner/pi-coding-agent/dist/core/resource-loader.js"; -import * as sdk from "./node_modules/@mariozechner/pi-coding-agent/dist/core/sdk.js"; -import * as sessionManager from "./node_modules/@mariozechner/pi-coding-agent/dist/core/session-manager.js"; -import * as settingsManager from "./node_modules/@mariozechner/pi-coding-agent/dist/core/settings-manager.js"; -import * as tools from "./node_modules/@mariozechner/pi-coding-agent/dist/core/tools/index.js"; - -// Keep this shape identical to `runtime` in src/snapshot-entry.ts. The adapter reads -// it lazily at session/new, so setting it after the adapter import (ESM-hoisted) is -// safe. -globalThis.__PI_SDK_RUNTIME__ = { - Agent: agentCore.Agent, - AuthStorage: authStorage.AuthStorage, - DefaultResourceLoader: resourceLoader.DefaultResourceLoader, - DEFAULT_THINKING_LEVEL: defaults.DEFAULT_THINKING_LEVEL, - ModelRegistry: modelRegistry.ModelRegistry, - SettingsManager: settingsManager.SettingsManager, - SessionManager: sessionManager.SessionManager, - convertToLlm: messages.convertToLlm, - getAgentDir: config.getAgentDir, - getDocsPath: config.getDocsPath, - createAgentSession: sdk.createAgentSession, - createCodingTools: sdk.createCodingTools, - createAllTools: tools.createAllTools, -}; - -import "./dist/adapter.js"; diff --git a/registry/agent/pi/agentos-package.json b/registry/agent/pi/agentos-package.json deleted file mode 100644 index 898e089c6e..0000000000 --- a/registry/agent/pi/agentos-package.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "name": "pi", - "agent": { - "acpEntrypoint": "pi-sdk-acp", - "snapshot": true - }, - "registry": { - "title": "PI", - "description": "Run the PI coding agent with lightweight, fast execution.", - "image": "/images/registry/pi.svg", - "priority": 100 - } -} diff --git a/registry/agent/pi/package.json b/registry/agent/pi/package.json deleted file mode 100644 index 96ae3ce7fa..0000000000 --- a/registry/agent/pi/package.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "name": "@agentos-software/pi", - "version": "0.2.1", - "type": "module", - "license": "Apache-2.0", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", - "bin": { - "pi": "./node_modules/@mariozechner/pi-coding-agent/dist/cli.js", - "pi-sdk-acp": "./dist/adapter.js" - }, - "exports": { - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js", - "default": "./dist/index.js" - } - }, - "files": [ - "dist", - "!dist/package", - "!dist/package.tar", - "agentos-package.json" - ], - "scripts": { - "build": "tsc && node scripts/build-snapshot-bundle.mjs && rm -rf dist/package dist/package.tar && agentos-toolchain pack . --out dist/package --agent pi-sdk-acp --prune-native && node scripts/copy-snapshot-into-package.mjs", - "build:snapshot": "node scripts/build-snapshot-bundle.mjs", - "check-types": "tsc --noEmit", - "test": "pnpm build && node --test --test-force-exit tests/*.test.mjs" - }, - "dependencies": { - "@agentclientprotocol/sdk": "^0.16.1", - "@mariozechner/pi-agent-core": "0.60.0", - "@mariozechner/pi-coding-agent": "0.60.0", - "@mariozechner/pi-ai": "0.60.0" - }, - "devDependencies": { - "@agentos-software/manifest": "workspace:*", - "@rivet-dev/agentos-toolchain": "workspace:*", - "@types/node": "^22.10.2", - "typescript": "^5.7.2" - } -} diff --git a/registry/agent/pi/tsconfig.json b/registry/agent/pi/tsconfig.json deleted file mode 100644 index e15131c84e..0000000000 --- a/registry/agent/pi/tsconfig.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "extends": "../../../tsconfig.base.json", - "compilerOptions": { - "declaration": true, - "outDir": "./dist", - "rootDir": "./src" - }, - "include": ["src/**/*"], - "exclude": ["node_modules", "dist", "src/snapshot-entry.ts"] -} diff --git a/registry/native/Cargo.toml b/registry/native/Cargo.toml deleted file mode 100644 index 4d567ae044..0000000000 --- a/registry/native/Cargo.toml +++ /dev/null @@ -1,168 +0,0 @@ -[workspace] -resolver = "2" -members = [ - "crates/wasi-ext", - "crates/libs/shims", - "crates/libs/builtins", - "crates/libs/stubs", - "crates/libs/grep", - "crates/libs/awk", - "crates/libs/jq", - "crates/libs/yq", - "crates/commands/grep", - "crates/commands/sh", - "crates/commands/sed", - "crates/commands/awk", - "crates/commands/jq", - "crates/commands/yq", - "crates/commands/rg", - "crates/libs/find", - "crates/libs/file-cmd", - "crates/libs/tree", - "crates/libs/du", - "crates/libs/column", - "crates/libs/rev", - "crates/libs/strings-cmd", - "crates/libs/expr", - "crates/libs/diff", - "crates/commands/find", - "crates/commands/file", - "crates/commands/tree", - "crates/commands/du", - "crates/commands/column", - "crates/commands/rev", - "crates/commands/strings", - "crates/commands/expr", - "crates/commands/diff", - "crates/libs/gzip", - "crates/libs/tar", - "crates/commands/gzip", - "crates/commands/tar", - "crates/commands/ls", - "crates/commands/cat", - "crates/commands/cp", - "crates/commands/mv", - "crates/commands/rm", - "crates/commands/mkdir", - "crates/commands/chmod", - "crates/commands/sort", - "crates/commands/head", - "crates/commands/tail", - "crates/commands/wc", - "crates/commands/cut", - "crates/commands/tr", - "crates/commands/echo", - "crates/commands/printf", - # US-009: remaining uutils coreutils - "crates/commands/dd", - "crates/commands/ln", - "crates/commands/logname", - "crates/commands/mktemp", - "crates/commands/pathchk", - "crates/commands/split", - "crates/commands/stat", - "crates/commands/tac", - "crates/commands/touch", - "crates/commands/tsort", - "crates/commands/base32", - "crates/commands/base64", - "crates/commands/basenc", - "crates/commands/basename", - "crates/commands/comm", - "crates/commands/dircolors", - "crates/commands/dirname", - "crates/commands/expand", - "crates/commands/factor", - "crates/commands/false", - "crates/commands/fmt", - "crates/commands/fold", - "crates/commands/join", - "crates/commands/nl", - "crates/commands/numfmt", - "crates/commands/od", - "crates/commands/paste", - "crates/commands/printenv", - "crates/commands/ptx", - "crates/commands/seq", - "crates/commands/shuf", - "crates/commands/true", - "crates/commands/unexpand", - "crates/commands/uniq", - "crates/commands/yes", - "crates/commands/b2sum", - "crates/commands/cksum", - "crates/commands/md5sum", - "crates/commands/sha1sum", - "crates/commands/sha224sum", - "crates/commands/sha256sum", - "crates/commands/sha384sum", - "crates/commands/sha512sum", - "crates/commands/sum", - "crates/commands/link", - "crates/commands/pwd", - "crates/commands/readlink", - "crates/commands/realpath", - "crates/commands/rmdir", - "crates/commands/shred", - "crates/commands/tee", - "crates/commands/truncate", - "crates/commands/unlink", - "crates/commands/arch", - "crates/commands/date", - "crates/commands/nproc", - "crates/commands/uname", - # US-010: builtins, shims, and stubs - "crates/commands/sleep", - "crates/commands/test", - "crates/commands/whoami", - "crates/commands/env", - "crates/commands/timeout", - "crates/commands/which", - "crates/commands/xu", - "crates/commands/xargs", - "crates/commands/nice", - "crates/commands/nohup", - "crates/commands/stdbuf", - "crates/commands/_stubs", - # US-079: fd-find compatible file finder - "crates/libs/fd", - "crates/commands/fd", - # US-099: codex (OpenAI Codex CLI) WASI build skeleton - "crates/commands/codex", - # US-100: WASI process spawning via host_process FFI - "crates/libs/wasi-spawn", - "crates/commands/spawn-test-host", - # US-101: WASI PTY-based process management via host_process FFI - "crates/libs/wasi-pty", - # US-102: WASI HTTP client via host_net TCP/TLS - "crates/libs/wasi-http", - "crates/commands/curl", - "crates/commands/http-test", - # US-104: codex-exec headless agent binary - "crates/commands/codex-exec", - # git: minimal git implementation - "crates/libs/git", - "crates/commands/git", - "stubs/codex-otel", - "stubs/ctrlc", - "stubs/hostname", -] - -[workspace.package] -version = "0.1.0" -edition = "2021" -license = "Apache-2.0" - -[profile.release] -opt-level = "z" -lto = true -codegen-units = 1 -strip = true - -# WASM-compatible stubs for crates that don't support wasm32-wasip1 -[patch.crates-io] -ctrlc = { path = "stubs/ctrlc" } -hostname = { path = "stubs/hostname" } -uucore = { path = "stubs/uucore" } -codex-network-proxy = { path = "stubs/codex-network-proxy" } -codex-otel = { path = "stubs/codex-otel" } diff --git a/registry/native/Makefile b/registry/native/Makefile deleted file mode 100644 index c8e814ffc4..0000000000 --- a/registry/native/Makefile +++ /dev/null @@ -1,326 +0,0 @@ -WASM_TARGET := wasm32-wasip1 -RELEASE_DIR := target/$(WASM_TARGET)/release - -# Standalone binary output directory (configurable) -COMMANDS_DIR ?= $(RELEASE_DIR)/commands - -# Detect patched std: if patches/ has .patch files and patch-std.sh exists, use patched sysroot -PATCHES := $(wildcard patches/*.patch) -PATCH_SCRIPT := scripts/patch-std.sh - -# Resolve WASI sysroot for CRT/libc needed by -Z build-std linking -# Uses rustc without +toolchain override so rust-toolchain.toml is respected -WASI_SYSROOT_SELF_CONTAINED := $(shell rustc --print sysroot)/lib/rustlib/$(WASM_TARGET)/lib/self-contained - -# Discover command binary names from crates/commands/. Keep known slow/heavy -# commands out of the default registry gate; they can still be built explicitly -# with `make cmd/`. -SKIPPED_BULK_COMMANDS := git codex codex-exec -COMMAND_NAMES := $(filter-out $(SKIPPED_BULK_COMMANDS),$(notdir $(wildcard crates/commands/*))) -# `_stubs` is the binary name but its Cargo package is `cmd-stubs`; every -# other command follows the `cmd-` convention. -COMMAND_PACKAGE_NAMES := $(patsubst _stubs,stubs,$(COMMAND_NAMES)) -COMMAND_PACKAGES := $(addprefix -p cmd-,$(COMMAND_PACKAGE_NAMES)) - -# Alias symlinks: link_name:target_binary -ALIAS_SYMLINKS := \ - egrep:grep fgrep:grep \ - gunzip:gzip zcat:gzip \ - bash:sh \ - dir:ls vdir:ls \ - more:cat \ - [:test - -# Stub commands: each gets a symlink to _stubs -STUB_COMMANDS := \ - chcon runcon \ - chgrp chown chroot \ - df \ - groups id \ - hostname hostid \ - install \ - kill \ - mkfifo mknod \ - pinky who users uptime \ - stty sync tty - -.PHONY: all commands wasm wasm-opt-check host test clean patch-std patch-check vendor patch-vendor vendor-patch-check size-report install c-wasm codex - -all: wasm - -# Build the fast command set that lands in COMMANDS_DIR: Rust commands except -# known heavy/external ones, then the opted-in fast C commands. Heavy commands -# (git, duckdb, vim) and the external codex build -# are intentionally excluded from the default registry-build gate. -commands: wasm - $(MAKE) -C c programs install - -# Build the real wasm32-wasip1 `codex-exec` agent engine from the codex fork -# (branch wasi-port-codex-core). The fork ships a committed, idempotent, self-prepping build script -# (it prepares the rustup sysroot for -Z build-std and restores it on exit — no manual stash), so -# this folds codex into the toolchain: `make -C registry/native codex` produces and installs -# registry/software/codex/wasm/{codex-exec,codex}. Override CODEX_REPO if the fork is not at the -# default sibling path. See the fork's docs/wasi-build.md for the toolchain constraints, and the -# remaining work to source the CRT from wasi-sdk so the prebuilt rust target need not be installed. -AGENTOS_ROOT := $(abspath $(CURDIR)/../..) -CODEX_REPO ?= $(abspath $(AGENTOS_ROOT)/../codex-rs/codex-rs) -codex: - @if [ -x "$(CODEX_REPO)/scripts/build-wasi-codex-exec.sh" ]; then \ - AGENTOS_DIR="$(AGENTOS_ROOT)" "$(CODEX_REPO)/scripts/build-wasi-codex-exec.sh"; \ - else \ - echo "codex: fork build script not found at $(CODEX_REPO)/scripts/build-wasi-codex-exec.sh — skipping."; \ - echo " Set CODEX_REPO=/path/to/codex-rs/codex-rs (the wasi-port-codex-core checkout) to build it."; \ - fi - -# Strict variant (fails if the fork is absent) for environments that require the codex artifact. -.PHONY: codex-required -codex-required: - @test -x "$(CODEX_REPO)/scripts/build-wasi-codex-exec.sh" || { \ - echo "ERROR: codex fork build script not found at $(CODEX_REPO); set CODEX_REPO"; exit 1; } - @$(MAKE) codex - -# Apply std patches if they exist -patch-std: -ifneq ($(PATCHES),) - @if [ -x "$(PATCH_SCRIPT)" ]; then \ - echo "Applying std patches..."; \ - ./$(PATCH_SCRIPT); \ - else \ - echo "Warning: patches found but $(PATCH_SCRIPT) not executable or missing"; \ - fi -else - @echo "No patches found, using stock std" -endif - -# Resolve Rust std source for vendoring build-std deps -RUST_STD_SRC := $(shell rustc --print sysroot)/lib/rustlib/src/rust - -# Vendor dependencies from crates.io into vendor/ (includes std deps for -Z build-std) -vendor: - @if [ ! -d vendor ]; then \ - echo "Vendoring dependencies (including std library deps)..."; \ - mkdir -p .cargo; \ - cargo vendor \ - --sync "$(RUST_STD_SRC)/library/std/Cargo.toml" \ - --sync "$(RUST_STD_SRC)/library/test/Cargo.toml" \ - vendor > .cargo/config.toml; \ - else \ - echo "vendor/ exists — skipping (run 'make clean-vendor' to re-vendor)"; \ - fi - @# Source-replacement config so cargo builds from the patched vendor/ tree - @# instead of the crates.io registry cache. This file is gitignored, so a - @# fresh checkout (CI) has none; without it every patches/crates/*.patch - @# wasi fix is silently ignored and crates like uu_tac fail to compile for - @# wasm32-wasip1. `cargo vendor` prints the exact config (including the - @# synced std-dep sources) on stdout, so capture it above. - @test -s .cargo/config.toml || { \ - echo "ERROR: .cargo/config.toml missing or empty after vendor"; \ - echo "Run 'make clean-vendor && make vendor' to regenerate."; \ - exit 1; \ - } - -# Apply crate-level patches to vendored sources -patch-vendor: vendor - @if [ -x scripts/patch-vendor.sh ]; then \ - ./scripts/patch-vendor.sh; \ - else \ - echo "Warning: scripts/patch-vendor.sh not found or not executable"; \ - fi - -# Dry-run verification of crate patches -vendor-patch-check: vendor - @if [ -x scripts/patch-vendor.sh ]; then \ - ./scripts/patch-vendor.sh --check; \ - else \ - echo "Warning: scripts/patch-vendor.sh not found or not executable"; \ - fi - -# Ensure wasm-opt is installed (needed for post-build optimization) -wasm-opt-check: - @if ! command -v wasm-opt >/dev/null 2>&1; then \ - echo "wasm-opt not found — installing via cargo..."; \ - cargo install wasm-opt; \ - else \ - echo "wasm-opt found: $$(wasm-opt --version)"; \ - fi - -# Build all standalone command binaries, optimize, strip .wasm extension, create symlinks -wasm: vendor patch-vendor patch-std wasm-opt-check - # --cfg tokio_unstable lifts tokio's wasm `compile_error!` so the patched - # tokio process/net modules (patches/crates/tokio) compile on wasm32-wasip1. - # Needed by codex (tokio::process/net). Harmless for commands not using tokio. - RUSTFLAGS="-C link-arg=-L$(WASI_SYSROOT_SELF_CONTAINED) --cfg tokio_unstable" \ - cargo build --target $(WASM_TARGET) \ - -Z build-std=std,panic_abort \ - --release \ - $(COMMAND_PACKAGES) - @mkdir -p $(COMMANDS_DIR) - @TOTAL=0; SKIPPED=0; \ - HAS_OPT=$$(command -v wasm-opt >/dev/null 2>&1 && echo 1 || echo 0); \ - for cmd in $(COMMAND_NAMES); do \ - wasm="$(RELEASE_DIR)/$$cmd.wasm"; \ - if [ -f "$$wasm" ]; then \ - TOTAL=$$((TOTAL + 1)); \ - if [ "$$HAS_OPT" = "1" ]; then \ - wasm-opt -O3 --strip-debug --all-features "$$wasm" -o "$(COMMANDS_DIR)/$$cmd"; \ - else \ - cp "$$wasm" "$(COMMANDS_DIR)/$$cmd"; \ - fi; \ - else \ - SKIPPED=$$((SKIPPED + 1)); \ - echo "Warning: $$wasm not found — skipping $$cmd"; \ - fi; \ - done; \ - echo ""; \ - echo "=== Standalone Binary Build Report ==="; \ - echo "Binaries: $$TOTAL built"; \ - if [ $$SKIPPED -gt 0 ]; then \ - echo "Skipped: $$SKIPPED (binary not found)"; \ - fi; \ - if [ "$$HAS_OPT" = "1" ]; then \ - echo "Optimized: wasm-opt -O3 --strip-debug"; \ - else \ - echo "Optimized: none (wasm-opt not found)"; \ - fi - @# Create alias symlinks (fall back to copy on platforms without symlink support) - @cd $(COMMANDS_DIR) && \ - for pair in $(ALIAS_SYMLINKS); do \ - link=$${pair%%:*}; target=$${pair##*:}; \ - if [ -f "$$target" ]; then \ - ln -sf "$$target" "$$link" 2>/dev/null || cp "$$target" "$$link"; \ - fi; \ - done - @# Create stub command symlinks - @cd $(COMMANDS_DIR) && \ - for stub in $(STUB_COMMANDS); do \ - if [ -f _stubs ]; then \ - ln -sf _stubs "$$stub" 2>/dev/null || cp _stubs "$$stub"; \ - fi; \ - done - @TOTAL_FILES=$$(ls -1 $(COMMANDS_DIR) 2>/dev/null | wc -l); \ - echo "Total: $$TOTAL_FILES entries in $(COMMANDS_DIR)"; \ - echo "=== Build complete ===" - -# --------------------------------------------------------------------------- -# Uniform per-binary entry point: `make cmd/` builds ONE command binary -# into $(COMMANDS_DIR), dispatching to the toolchain that owns it: -# Rust crates/commands/ (cargo package cmd-, via wasm-cmd) -# C the C opt-in set below (registry/native/c, patched sysroot) -# codex the codex fork build (codex, codex-exec) -# external hand-built drop-zone binaries with no source pipeline yet -# `just registry-native-cmd ` calls this. `make wasm` builds the fast -# Rust set; `make -C c programs install` builds/installs the fast C set. -C_COMMANDS := zip unzip envsubst sqlite3 curl wget duckdb vim http_get -EXTERNAL_COMMANDS := vix - -.PHONY: cmd/% -cmd/%: - @name="$*"; \ - if [ -d "crates/commands/$$name" ]; then \ - $(MAKE) wasm-cmd CMD=$$name; \ - elif echo " $(C_COMMANDS) " | grep -q " $$name "; then \ - src=$$name; [ "$$name" = "sqlite3" ] && src=sqlite3_cli; \ - $(MAKE) -C c sysroot "build/$$src" && $(MAKE) -C c install COMMANDS="$$name"; \ - elif [ "$$name" = "codex" ] || [ "$$name" = "codex-exec" ]; then \ - $(MAKE) codex-required; \ - elif echo " $(EXTERNAL_COMMANDS) " | grep -q " $$name "; then \ - if [ -f "$(COMMANDS_DIR)/$$name" ]; then \ - echo "cmd/$$name: external hand-built binary present at $(COMMANDS_DIR)/$$name"; \ - else \ - echo "ERROR: '$$name' has no source pipeline yet — place the hand-built wasm binary at $(COMMANDS_DIR)/$$name (see registry/README.md)"; \ - exit 1; \ - fi; \ - else \ - echo "ERROR: unknown command '$$name' (no crates/commands/$$name; not in C_COMMANDS, codex, or EXTERNAL_COMMANDS)"; \ - exit 1; \ - fi - -# Build ONE command binary (cargo package cmd-$(CMD) -> $(COMMANDS_DIR)/$(CMD)). -# Usage: make wasm-cmd CMD=sh -.PHONY: wasm-cmd -wasm-cmd: vendor patch-vendor patch-std wasm-opt-check - @test -n "$(CMD)" || { echo "ERROR: pass CMD= (a dir under crates/commands/)"; exit 1; } - @test -d "crates/commands/$(CMD)" || { echo "ERROR: no crates/commands/$(CMD)"; exit 1; } - RUSTFLAGS="-C link-arg=-L$(WASI_SYSROOT_SELF_CONTAINED) --cfg tokio_unstable" \ - cargo build --target $(WASM_TARGET) \ - -Z build-std=std,panic_abort \ - --release \ - -p cmd-$(CMD) - @mkdir -p $(COMMANDS_DIR) - @if command -v wasm-opt >/dev/null 2>&1; then \ - wasm-opt -O3 --strip-debug --all-features "$(RELEASE_DIR)/$(CMD).wasm" -o "$(COMMANDS_DIR)/$(CMD)"; \ - else \ - cp "$(RELEASE_DIR)/$(CMD).wasm" "$(COMMANDS_DIR)/$(CMD)"; \ - fi - @echo "Built $(COMMANDS_DIR)/$(CMD)" - -# Show per-command binary sizes in COMMANDS_DIR -size-report: - @if [ ! -d "$(COMMANDS_DIR)" ]; then \ - echo "Error: $(COMMANDS_DIR) not found. Run 'make wasm' first."; \ - exit 1; \ - fi - @echo "=== Standalone Binary Size Report ===" - @echo "" - @printf "%-10s %s\n" "SIZE" "COMMAND" - @printf "%-10s %s\n" "----" "-------" - @ls -lhS $(COMMANDS_DIR)/* 2>/dev/null | \ - grep -v "^total" | \ - while read line; do \ - size=$$(echo "$$line" | awk '{print $$5}'); \ - name=$$(echo "$$line" | awk '{print $$NF}'); \ - base=$$(basename "$$name"); \ - link=""; \ - if [ -L "$$name" ]; then \ - target=$$(readlink "$$name"); \ - link=" -> $$target"; \ - fi; \ - printf "%-10s %s%s\n" "$$size" "$$base" "$$link"; \ - done - @echo "" - @TOTAL_SIZE=$$(du -sh $(COMMANDS_DIR) 2>/dev/null | awk '{print $$1}'); \ - TOTAL_FILES=$$(ls -1 $(COMMANDS_DIR) 2>/dev/null | wc -l); \ - REAL_FILES=$$(find $(COMMANDS_DIR) -maxdepth 1 -type f 2>/dev/null | wc -l); \ - SYMLINKS=$$(find $(COMMANDS_DIR) -maxdepth 1 -type l 2>/dev/null | wc -l); \ - echo "Total: $$TOTAL_SIZE ($$TOTAL_FILES entries: $$REAL_FILES binaries, $$SYMLINKS symlinks)" - @echo "" - @echo "=== Per-Crate Size Contribution (rlib, top 30) ===" - @echo "Note: rlib sizes are pre-LTO; actual contribution after LTO/DCE may differ." - @echo "" - @printf "%-10s %s\n" "SIZE" "CRATE" - @printf "%-10s %s\n" "----" "-----" - @ls -lhS $(RELEASE_DIR)/deps/*.rlib 2>/dev/null | \ - awk '{split($$NF, a, "/"); name=a[length(a)]; gsub(/^lib/, "", name); gsub(/-[0-9a-f]+\.rlib$$/, "", name); printf "%-10s %s\n", $$5, name}' | \ - awk '!seen[$$2]++' | \ - head -30 - @echo "" - @echo "=== Dependency Count ===" - @TOTAL=$$(ls $(RELEASE_DIR)/deps/*.rlib 2>/dev/null | \ - sed 's/.*\/lib//' | sed 's/-[0-9a-f]*\.rlib//' | sort -u | wc -l); \ - echo "Unique crates: $$TOTAL" - -# Install standalone binaries to a custom directory - -# Build C programs to WASM (downloads wasi-sdk if needed) - - -test: - cargo test --workspace - -clean: - cargo clean - rm -rf host/dist - -clean-vendor: - rm -rf vendor .cargo/config.toml - -patch-check: -ifneq ($(PATCHES),) - @if [ -x "$(PATCH_SCRIPT)" ]; then \ - ./$(PATCH_SCRIPT) --check; \ - else \ - echo "Warning: $(PATCH_SCRIPT) not found or not executable"; \ - fi -else - @echo "No patches found" -endif diff --git a/registry/native/c/Makefile b/registry/native/c/Makefile deleted file mode 100644 index 29209a762f..0000000000 --- a/registry/native/c/Makefile +++ /dev/null @@ -1,717 +0,0 @@ -# wasmvm/c/Makefile — Build C programs to standalone WASM binaries -# -# Targets: -# wasi-sdk Download and cache wasi-sdk toolchain -# programs Compile all .c files in programs/ to WASM binaries -# sysroot Build patched wasi-libc sysroot (invokes patch-wasi-libc.sh) -# install Copy opted-in real commands to COMMANDS_DIR -# native Compile all .c files to native binaries (for parity testing) -# clean Remove build output -# -# The vanilla (unpatched) wasi-sdk sysroot is used by default. -# After US-025 adds patch-wasi-libc.sh, `make sysroot` builds a patched one. - -# wasi-sdk version and platform detection -WASI_SDK_VERSION := 25 -WASI_SDK_MAJOR := 25.0 - -UNAME_S := $(shell uname -s) -UNAME_M := $(shell uname -m) - -ifeq ($(UNAME_S),Linux) - WASI_SDK_PLATFORM := linux -else ifeq ($(UNAME_S),Darwin) - WASI_SDK_PLATFORM := macos -else - WASI_SDK_PLATFORM := linux -endif - -ifeq ($(UNAME_M),arm64) - WASI_SDK_ARCH := arm64 -else ifeq ($(UNAME_M),aarch64) - WASI_SDK_ARCH := arm64 -else - WASI_SDK_ARCH := x86_64 -endif - -WASI_SDK_TARBALL := wasi-sdk-$(WASI_SDK_MAJOR)-$(WASI_SDK_ARCH)-$(WASI_SDK_PLATFORM).tar.gz -WASI_SDK_URL := https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-$(WASI_SDK_VERSION)/$(WASI_SDK_TARBALL) - -# Directories -VENDOR_DIR := vendor -WASI_SDK_DIR := $(VENDOR_DIR)/wasi-sdk -BUILD_DIR := build -NATIVE_DIR := build/native - -# Toolchain -CC := $(WASI_SDK_DIR)/bin/clang -NATIVE_CC := $(shell command -v cc 2>/dev/null || command -v gcc 2>/dev/null || command -v clang 2>/dev/null) - -# Sysroot: use patched sysroot if built, otherwise use wasi-sdk's vanilla sysroot -PATCHED_SYSROOT := sysroot -ifeq ($(wildcard $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a),) - SYSROOT := $(WASI_SDK_DIR)/share/wasi-sysroot -else - SYSROOT := $(PATCHED_SYSROOT) -endif - -# Compile flags -WASM_CFLAGS := --target=wasm32-wasip1 --sysroot=$(SYSROOT) -O2 -flto -I include/ -NATIVE_CFLAGS := -O0 -g -D_LARGEFILE64_SOURCE -I include/ - -# COMMANDS_DIR for install target (configurable, matches Rust binary output) -COMMANDS_DIR ?= ../target/wasm32-wasip1/release/commands - -# Fast real commands installed by the default registry gate. Slow/heavy commands -# (duckdb, vim) remain available through explicit parent `make cmd/`. -COMMANDS := zip unzip envsubst sqlite3 curl http_get - -# Programs requiring patched sysroot (Tier 2+ custom host imports) -PATCHED_PROGRAMS := http_get_test isatty_test getpid_test getppid_test getppid_verify userinfo pipe_test dup_test spawn_child spawn_exit_code pipeline kill_child waitpid_return waitpid_edge syscall_coverage getpwuid_test signal_tests sigaction_self sigaction_behavior delayed_tcp_echo delayed_kill pipe_edge tcp_accept_spawn tcp_echo tcp_server http_server udp_echo unix_socket signal_handler http_get dns_lookup sqlite3_cli curl wget fs_probe - -# Discover all .c source files in programs/ -ALL_SOURCES := $(wildcard programs/*.c) -SKIPPED_BULK_PROGRAMS := wget - -# Exclude patched-sysroot programs when only vanilla sysroot is available -ifeq ($(wildcard $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a),) - EXCLUDED := $(addsuffix .c,$(addprefix programs/,$(PATCHED_PROGRAMS))) - SOURCES := $(filter-out $(EXCLUDED) $(addsuffix .c,$(addprefix programs/,$(SKIPPED_BULK_PROGRAMS))),$(ALL_SOURCES)) -else - SOURCES := $(filter-out $(addsuffix .c,$(addprefix programs/,$(SKIPPED_BULK_PROGRAMS))),$(ALL_SOURCES)) -endif - -# Program names (without .c extension) -ifeq ($(wildcard $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a),) - CUSTOM_WASM_PROG_NAMES := -else - CUSTOM_WASM_PROG_NAMES := curl -endif - -WASM_PROG_NAMES := $(sort $(basename $(notdir $(SOURCES))) $(CUSTOM_WASM_PROG_NAMES)) -NATIVE_PROG_NAMES := $(basename $(notdir $(SOURCES))) -PROGRAM_REPORT_NAMES := $(WASM_PROG_NAMES) - -# Default WASM output targets -EXTRA_WASM_OUTPUTS := -WASM_OUTPUTS := $(addprefix $(BUILD_DIR)/,$(WASM_PROG_NAMES)) $(EXTRA_WASM_OUTPUTS) -# Native output targets -NATIVE_OUTPUTS := $(addprefix $(NATIVE_DIR)/,$(NATIVE_PROG_NAMES)) - -.PHONY: all wasi-sdk programs sysroot install native clean wasm-opt-check os-test os-test-native - -HAS_WASM_OPT := $(shell command -v wasm-opt >/dev/null 2>&1 && echo 1 || echo 0) - -all: programs - -# --- wasi-sdk download and cache --- - -wasi-sdk: $(WASI_SDK_DIR)/bin/clang - -$(WASI_SDK_DIR)/bin/clang: - @echo "=== Downloading wasi-sdk $(WASI_SDK_MAJOR) ===" - @mkdir -p $(VENDOR_DIR) - @if command -v curl >/dev/null 2>&1; then \ - curl -fSL "$(WASI_SDK_URL)" -o "$(VENDOR_DIR)/$(WASI_SDK_TARBALL)"; \ - elif command -v wget >/dev/null 2>&1; then \ - wget -q "$(WASI_SDK_URL)" -O "$(VENDOR_DIR)/$(WASI_SDK_TARBALL)"; \ - else \ - echo "Error: neither curl nor wget found"; exit 1; \ - fi - @echo "Extracting..." - @tar -xzf "$(VENDOR_DIR)/$(WASI_SDK_TARBALL)" -C "$(VENDOR_DIR)" - @mv "$(VENDOR_DIR)/wasi-sdk-$(WASI_SDK_MAJOR)-$(WASI_SDK_ARCH)-$(WASI_SDK_PLATFORM)" "$(WASI_SDK_DIR)" - @rm -f "$(VENDOR_DIR)/$(WASI_SDK_TARBALL)" - @echo "wasi-sdk $(WASI_SDK_MAJOR) installed to $(WASI_SDK_DIR)" - -# --- C library dependencies (downloaded at build time) --- -# Unmodified upstream libs are fetched from their official sources. -# Modified libs (curl with WASI patches) are fetched from rivet-dev forks. -# All downloads cached in libs/ — add libs/ to .gitignore. - -SQLITE3_URL := https://www.sqlite.org/2024/sqlite-amalgamation-3470200.zip -ZLIB_URL := https://github.com/madler/zlib/archive/refs/tags/v1.3.1.zip -CJSON_URL := https://github.com/DaveGamble/cJSON/archive/refs/tags/v1.7.18.zip -CURL_COMMIT := main -CURL_FORK_REPO_PREFIX := secure -CURL_FORK_REPO_SUFFIX := exec-curl -CURL_FORK_REPO := $(CURL_FORK_REPO_PREFIX)-$(CURL_FORK_REPO_SUFFIX) -CURL_URL := https://github.com/rivet-dev/$(CURL_FORK_REPO)/archive/refs/heads/$(CURL_COMMIT).zip -CURL_TOOL_VERSION := 8_11_1 -CURL_TOOL_URL := https://github.com/curl/curl/archive/refs/tags/curl-$(CURL_TOOL_VERSION).tar.gz -CURL_RELEASE_VERSION := 8.11.1 -CURL_RELEASE_TAG := 8_11_1 -CURL_RELEASE_URL := https://github.com/curl/curl/releases/download/curl-$(CURL_RELEASE_TAG)/curl-$(CURL_RELEASE_VERSION).tar.xz -CURL_UPSTREAM_BUILD_DIR := $(BUILD_DIR)/curl-upstream -CURL_UPSTREAM_OVERLAY_DIR := curl-upstream-overlay -CURL_UPSTREAM_OVERLAY_FILES := $(wildcard $(CURL_UPSTREAM_OVERLAY_DIR)/lib/*.c $(CURL_UPSTREAM_OVERLAY_DIR)/lib/*.h $(CURL_UPSTREAM_OVERLAY_DIR)/lib/vtls/*.c $(CURL_UPSTREAM_OVERLAY_DIR)/lib/vtls/*.h) -MINIZIP_URL := https://github.com/madler/zlib/archive/refs/tags/v1.3.1.zip -# duckdb-wasm currently documents DuckDB v1.5.0 in its README. We build the -# matching upstream DuckDB tag ourselves with our patched WASI/POSIX sysroot. -DUCKDB_VERSION := v1.5.0 -DUCKDB_GIT_DESCRIBE := $(DUCKDB_VERSION)-0-g0123456789 -DUCKDB_URL := https://github.com/duckdb/duckdb/archive/refs/tags/$(DUCKDB_VERSION).tar.gz - -LIBS_DIR := libs -LIBS_CACHE := .cache/libs - -libs/sqlite3/sqlite3.c: - @echo "Fetching sqlite3..." - @mkdir -p $(LIBS_CACHE) $(LIBS_DIR)/sqlite3 - @curl -fSL "$(SQLITE3_URL)" -o "$(LIBS_CACHE)/sqlite3.zip" - @cd $(LIBS_CACHE) && unzip -qo sqlite3.zip - @cp $(LIBS_CACHE)/sqlite-amalgamation-*/sqlite3.c $(LIBS_CACHE)/sqlite-amalgamation-*/sqlite3.h $(LIBS_DIR)/sqlite3/ - -libs/zlib/zutil.c: - @echo "Fetching zlib..." - @mkdir -p $(LIBS_CACHE) $(LIBS_DIR)/zlib - @curl -fSL "$(ZLIB_URL)" -o "$(LIBS_CACHE)/zlib.zip" - @cd $(LIBS_CACHE) && unzip -qo zlib.zip - @cp $(LIBS_CACHE)/zlib-*/*.c $(LIBS_CACHE)/zlib-*/*.h $(LIBS_DIR)/zlib/ - -# The fetch recipe above materializes the complete archive. Declare the other -# files as outputs that depend on its representative target so a clean `make -# programs` can construct its dependency graph before the archive exists. -ZLIB_FETCHED_SRCS := \ - libs/zlib/adler32.c libs/zlib/compress.c libs/zlib/crc32.c \ - libs/zlib/deflate.c libs/zlib/infback.c libs/zlib/inffast.c \ - libs/zlib/inflate.c libs/zlib/inftrees.c libs/zlib/trees.c \ - libs/zlib/uncompr.c -$(ZLIB_FETCHED_SRCS): libs/zlib/zutil.c - @test -f "$@" || { echo "Error: zlib fetch did not produce $@"; exit 1; } - -libs/minizip/ioapi.c: - @echo "Fetching minizip (from zlib contrib)..." - @mkdir -p $(LIBS_CACHE) $(LIBS_DIR)/minizip - @if [ ! -f "$(LIBS_CACHE)/zlib.zip" ]; then \ - curl -fSL "$(MINIZIP_URL)" -o "$(LIBS_CACHE)/zlib.zip"; \ - cd $(LIBS_CACHE) && unzip -qo zlib.zip; \ - fi - @cp $(LIBS_CACHE)/zlib-*/contrib/minizip/ioapi.c $(LIBS_CACHE)/zlib-*/contrib/minizip/ioapi.h \ - $(LIBS_CACHE)/zlib-*/contrib/minizip/zip.c $(LIBS_CACHE)/zlib-*/contrib/minizip/zip.h \ - $(LIBS_CACHE)/zlib-*/contrib/minizip/unzip.c $(LIBS_CACHE)/zlib-*/contrib/minizip/unzip.h \ - $(LIBS_CACHE)/zlib-*/contrib/minizip/crypt.h \ - $(LIBS_DIR)/minizip/ - -MINIZIP_FETCHED_SRCS := libs/minizip/zip.c libs/minizip/unzip.c -$(MINIZIP_FETCHED_SRCS): libs/minizip/ioapi.c - @test -f "$@" || { echo "Error: minizip fetch did not produce $@"; exit 1; } - -libs/cjson/cJSON.c: - @echo "Fetching cJSON..." - @mkdir -p $(LIBS_CACHE) $(LIBS_DIR)/cjson - @curl -fSL "$(CJSON_URL)" -o "$(LIBS_CACHE)/cjson.zip" - @cd $(LIBS_CACHE) && unzip -qo cjson.zip - @cp $(LIBS_CACHE)/cJSON-*/cJSON.c $(LIBS_CACHE)/cJSON-*/cJSON.h $(LIBS_DIR)/cjson/ - -libs/curl/lib/easy.c: - @echo "Fetching curl (rivet-dev/$(CURL_FORK_REPO))..." - @mkdir -p $(LIBS_CACHE) - @curl -fSL "$(CURL_URL)" -o "$(LIBS_CACHE)/curl.zip" - @cd $(LIBS_CACHE) && unzip -qo curl.zip - @rm -rf $(LIBS_DIR)/curl - @mv $(LIBS_CACHE)/$(CURL_FORK_REPO)-* $(LIBS_DIR)/curl - -libs/duckdb/CMakeLists.txt: - @echo "Fetching DuckDB ($(DUCKDB_VERSION))..." - @mkdir -p $(LIBS_CACHE) - @curl -fSL "$(DUCKDB_URL)" -o "$(LIBS_CACHE)/duckdb.tar.gz" - @rm -rf $(LIBS_DIR)/duckdb - @mkdir -p $(LIBS_DIR)/duckdb - @tar -xzf "$(LIBS_CACHE)/duckdb.tar.gz" --strip-components=1 -C $(LIBS_DIR)/duckdb - -.PHONY: fetch-libs fetch-fast-libs clean-libs -fetch-libs: libs/sqlite3/sqlite3.c libs/zlib/zutil.c libs/minizip/ioapi.c libs/cjson/cJSON.c libs/curl/lib/easy.c libs/duckdb/CMakeLists.txt -fetch-fast-libs: libs/sqlite3/sqlite3.c libs/zlib/zutil.c libs/minizip/ioapi.c libs/cjson/cJSON.c libs/curl/lib/easy.c - -clean-libs: - rm -rf $(LIBS_DIR) $(LIBS_CACHE) - -# --- os-test POSIX conformance suite (downloaded at build time) --- -# ISC-licensed test suite from the Sortix project (https://sortix.org/os-test/) - -OS_TEST_VERSION := main -OS_TEST_URL := https://gitlab.com/sortix/os-test/-/archive/$(OS_TEST_VERSION)/os-test-$(OS_TEST_VERSION).tar.gz -OS_TEST_DIR := os-test - -.PHONY: fetch-os-test clean-os-test - -fetch-os-test: $(OS_TEST_DIR)/include -$(OS_TEST_DIR)/include: - @echo "Fetching os-test $(OS_TEST_VERSION)..." - @mkdir -p $(LIBS_CACHE) - @curl -fSL "$(OS_TEST_URL)" -o "$(LIBS_CACHE)/os-test.tar.gz" - @mkdir -p $(OS_TEST_DIR) - @tar -xzf "$(LIBS_CACHE)/os-test.tar.gz" --strip-components=1 -C $(OS_TEST_DIR) - @echo "os-test $(OS_TEST_VERSION) extracted to $(OS_TEST_DIR)/" - @echo " include/ : $$(find $(OS_TEST_DIR)/include -name '*.h' 2>/dev/null | wc -l) headers" - @echo " src/ : $$(find $(OS_TEST_DIR)/src -name '*.c' 2>/dev/null | wc -l) test programs" - -clean-os-test: - rm -rf $(OS_TEST_DIR) $(LIBS_CACHE)/os-test.tar.gz - -# --- os-test POSIX conformance build targets --- -# Compile every os-test C file to WASM and native binaries. -# Individual compile failures are expected (missing headers, unsupported features) -# and do not abort the build. - -OS_TEST_BUILD := $(BUILD_DIR)/os-test -OS_TEST_NATIVE_BUILD := $(NATIVE_DIR)/os-test -OS_TEST_WASM_CFLAGS := --target=wasm32-wasip1 --sysroot=$(SYSROOT) -O0 \ - -D_GNU_SOURCE -D_BSD_SOURCE -D_ALL_SOURCE -D_DEFAULT_SOURCE -OS_TEST_WASM_LDFLAGS := -lc-printscan-long-double -# Stub main() for namespace/ tests (compile-only header conformance checks) -OS_TEST_NS_MAIN := os-test-overrides/namespace_main.c -OS_TEST_NATIVE_CFLAGS := -O0 -g \ - -D_GNU_SOURCE -D_BSD_SOURCE -D_ALL_SOURCE -D_DEFAULT_SOURCE -OS_TEST_NATIVE_LDFLAGS := -lm -ifeq ($(UNAME_S),Linux) - OS_TEST_NATIVE_LDFLAGS += -lpthread -lrt -endif -ifeq ($(UNAME_S),Darwin) - OS_TEST_NATIVE_LDFLAGS += -lpthread -endif - -.PHONY: os-test os-test-native - -os-test: wasi-sdk fetch-os-test - @total=0; pass=0; fail=0; \ - for src in $$(find $(OS_TEST_DIR) -name '*.c' -not -path '*/.expect/*' -not -path '*/misc/*' | sort); do \ - rel=$${src#$(OS_TEST_DIR)/}; \ - name=$${rel%.c}; \ - out=$(OS_TEST_BUILD)/$$name; \ - mkdir -p "$$(dirname "$$out")"; \ - total=$$((total + 1)); \ - extras=""; \ - case "$$rel" in namespace/*) extras="$(OS_TEST_NS_MAIN)";; esac; \ - if $(CC) $(OS_TEST_WASM_CFLAGS) -o "$$out" "$$src" $$extras $(OS_TEST_WASM_LDFLAGS) 2>/dev/null; then \ - pass=$$((pass + 1)); \ - else \ - echo "COMPILE FAILED: $$name"; \ - rm -f "$$out"; \ - fail=$$((fail + 1)); \ - fi; \ - done; \ - echo ""; \ - echo "=== os-test WASM Build Report ==="; \ - echo "Total: $$total"; \ - echo "Compiled: $$pass"; \ - echo "Failed: $$fail"; \ - echo "Output: $(OS_TEST_BUILD)/"; \ - echo "=== Build complete ==="; \ - if [ $$fail -gt 0 ]; then echo "ERROR: $$fail tests failed to compile" >&2; exit 1; fi - -os-test-native: fetch-os-test - @total=0; pass=0; fail=0; \ - for src in $$(find $(OS_TEST_DIR) -name '*.c' -not -path '*/.expect/*' -not -path '*/misc/*' | sort); do \ - rel=$${src#$(OS_TEST_DIR)/}; \ - out=$(OS_TEST_NATIVE_BUILD)/$${rel%.c}; \ - mkdir -p "$$(dirname "$$out")"; \ - total=$$((total + 1)); \ - extras=""; \ - case "$$rel" in namespace/*) extras="$(OS_TEST_NS_MAIN)";; esac; \ - if $(NATIVE_CC) $(OS_TEST_NATIVE_CFLAGS) -o "$$out" "$$src" $$extras $(OS_TEST_NATIVE_LDFLAGS) 2>/dev/null; then \ - pass=$$((pass + 1)); \ - else \ - rm -f "$$out"; \ - fail=$$((fail + 1)); \ - fi; \ - done; \ - echo ""; \ - echo "=== os-test Native Build Report ==="; \ - echo "Total: $$total"; \ - echo "Compiled: $$pass"; \ - echo "Failed: $$fail"; \ - echo "Output: $(OS_TEST_NATIVE_BUILD)/"; \ - echo "=== Build complete ===" - -# --- musl libc-test conformance suite (downloaded at build time) --- -# MIT-licensed test suite from musl (Bytecode Alliance mirror) - -LIBC_TEST_VERSION := master -LIBC_TEST_URL := https://github.com/bytecodealliance/libc-test/archive/refs/heads/$(LIBC_TEST_VERSION).tar.gz -LIBC_TEST_DIR := libc-test - -.PHONY: fetch-libc-test clean-libc-test - -fetch-libc-test: $(LIBC_TEST_DIR)/src -$(LIBC_TEST_DIR)/src: - @echo "Fetching musl libc-test $(LIBC_TEST_VERSION)..." - @mkdir -p $(LIBS_CACHE) - @curl -fSL "$(LIBC_TEST_URL)" -o "$(LIBS_CACHE)/libc-test.tar.gz" - @mkdir -p $(LIBC_TEST_DIR) - @tar -xzf "$(LIBS_CACHE)/libc-test.tar.gz" --strip-components=1 -C $(LIBC_TEST_DIR) - @echo "libc-test $(LIBC_TEST_VERSION) extracted to $(LIBC_TEST_DIR)/" - @echo " src/ : $$(find $(LIBC_TEST_DIR)/src -name '*.c' 2>/dev/null | wc -l) test programs" - -clean-libc-test: - rm -rf $(LIBC_TEST_DIR) $(LIBS_CACHE)/libc-test.tar.gz - -# --- musl libc-test build targets --- -# Compile functional/ and regression/ tests to WASM and native binaries. -# Tests requiring fork/exec/signals/pthreads will fail to compile for WASM — that's expected. -# The test.h framework (src/common/) is compiled into a static library first. - -LIBC_TEST_BUILD := $(BUILD_DIR)/libc-test -LIBC_TEST_NATIVE_BUILD := $(NATIVE_DIR)/libc-test -LIBC_TEST_WASM_CFLAGS := --target=wasm32-wasip1 --sysroot=$(SYSROOT) -O0 \ - -D_GNU_SOURCE -D_BSD_SOURCE -D_ALL_SOURCE -D_DEFAULT_SOURCE \ - -I$(LIBC_TEST_DIR)/src/common -LIBC_TEST_WASM_LDFLAGS := -lc-printscan-long-double -lm -LIBC_TEST_NATIVE_CFLAGS := -O0 -g \ - -D_GNU_SOURCE -D_BSD_SOURCE -D_ALL_SOURCE -D_DEFAULT_SOURCE \ - -I$(LIBC_TEST_DIR)/src/common -LIBC_TEST_NATIVE_LDFLAGS := -lm -ifeq ($(UNAME_S),Linux) - LIBC_TEST_NATIVE_LDFLAGS += -lpthread -lrt -lcrypt -lresolv -lutil -ldl -endif -ifeq ($(UNAME_S),Darwin) - LIBC_TEST_NATIVE_LDFLAGS += -lpthread -endif - -# Test framework sources compiled alongside each test -LIBC_TEST_COMMON_SRCS := $(LIBC_TEST_DIR)/src/common/print.c - -.PHONY: libc-test libc-test-native - -libc-test: wasi-sdk fetch-libc-test - @total=0; pass=0; fail=0; \ - for src in $$(find $(LIBC_TEST_DIR)/src/functional $(LIBC_TEST_DIR)/src/regression -name '*.c' 2>/dev/null | sort); do \ - rel=$${src#$(LIBC_TEST_DIR)/src/}; \ - name=$${rel%.c}; \ - out=$(LIBC_TEST_BUILD)/$$name; \ - mkdir -p "$$(dirname "$$out")"; \ - total=$$((total + 1)); \ - if $(CC) $(LIBC_TEST_WASM_CFLAGS) -o "$$out" "$$src" $(LIBC_TEST_COMMON_SRCS) $(LIBC_TEST_WASM_LDFLAGS) 2>/dev/null; then \ - pass=$$((pass + 1)); \ - else \ - rm -f "$$out"; \ - fail=$$((fail + 1)); \ - fi; \ - done; \ - echo ""; \ - echo "=== libc-test WASM Build Report ==="; \ - echo "Total: $$total"; \ - echo "Compiled: $$pass"; \ - echo "Failed: $$fail"; \ - echo "Output: $(LIBC_TEST_BUILD)/"; \ - echo "=== Build complete ===" - -libc-test-native: fetch-libc-test - @total=0; pass=0; fail=0; \ - for src in $$(find $(LIBC_TEST_DIR)/src/functional $(LIBC_TEST_DIR)/src/regression -name '*.c' 2>/dev/null | sort); do \ - rel=$${src#$(LIBC_TEST_DIR)/src/}; \ - out=$(LIBC_TEST_NATIVE_BUILD)/$${rel%.c}; \ - mkdir -p "$$(dirname "$$out")"; \ - total=$$((total + 1)); \ - if $(NATIVE_CC) $(LIBC_TEST_NATIVE_CFLAGS) -o "$$out" "$$src" $(LIBC_TEST_COMMON_SRCS) $(LIBC_TEST_NATIVE_LDFLAGS) 2>/dev/null; then \ - pass=$$((pass + 1)); \ - else \ - rm -f "$$out"; \ - fail=$$((fail + 1)); \ - fi; \ - done; \ - echo ""; \ - echo "=== libc-test Native Build Report ==="; \ - echo "Total: $$total"; \ - echo "Compiled: $$pass"; \ - echo "Failed: $$fail"; \ - echo "Output: $(LIBC_TEST_NATIVE_BUILD)/"; \ - echo "=== Build complete ===" - -# --- Patched sysroot (delegates to patch-wasi-libc.sh) --- - -WASI_LIBC_PATCHES := $(wildcard ../patches/wasi-libc/*.patch) -WASI_LIBC_OVERRIDES := $(wildcard ../patches/wasi-libc-overrides/*.c) -LLVM_RUNTIME_PATCHES := $(wildcard patches/llvm-project/*.patch) - -$(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a: wasi-sdk ../scripts/patch-wasi-libc.sh scripts/build-llvm-runtimes.sh $(WASI_LIBC_PATCHES) $(WASI_LIBC_OVERRIDES) $(LLVM_RUNTIME_PATCHES) - ../scripts/patch-wasi-libc.sh - -sysroot: $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a - -# --- wasm-opt check --- - -wasm-opt-check: - @if [ "$(HAS_WASM_OPT)" = "1" ]; then \ - echo "wasm-opt found: $$(wasm-opt --version | head -1)"; \ - else \ - echo "Warning: wasm-opt (binaryen) is not installed; skipping WASM optimization."; \ - fi - -# --- Compile all C programs to WASM --- - -programs: wasi-sdk fetch-fast-libs wasm-opt-check $(WASM_OUTPUTS) - @echo "" - @echo "=== C WASM Build Report ===" - @echo "Programs: $(words $(PROGRAM_REPORT_NAMES)) compiled" - @echo "Sysroot: $(SYSROOT)" - @echo "Output: $(BUILD_DIR)/" - @echo "=== Build complete ===" - -# Default rule: single-file programs -$(BUILD_DIR)/%: programs/%.c $(WASI_SDK_DIR)/bin/clang - @mkdir -p $(BUILD_DIR) - $(CC) $(WASM_CFLAGS) -o $@.wasm $< - @if [ "$(HAS_WASM_OPT)" = "1" ]; then \ - wasm-opt -O3 --strip-debug $@.wasm -o $@; \ - else \ - cp $@.wasm $@; \ - fi - @rm -f $@.wasm - -# --- Per-program override rules for multi-file compilation --- -# Add overrides here for programs that link vendored libraries. - -# json_parse: links cJSON library -$(BUILD_DIR)/json_parse: programs/json_parse.c libs/cjson/cJSON.c $(WASI_SDK_DIR)/bin/clang - @mkdir -p $(BUILD_DIR) - $(CC) $(WASM_CFLAGS) -Ilibs/cjson -o $@.wasm programs/json_parse.c libs/cjson/cJSON.c - @if [ "$(HAS_WASM_OPT)" = "1" ]; then \ - wasm-opt -O3 --strip-debug $@.wasm -o $@; \ - else \ - cp $@.wasm $@; \ - fi - @rm -f $@.wasm - -$(NATIVE_DIR)/json_parse: programs/json_parse.c libs/cjson/cJSON.c - @mkdir -p $(NATIVE_DIR) - $(NATIVE_CC) $(NATIVE_CFLAGS) -Ilibs/cjson -o $@ programs/json_parse.c libs/cjson/cJSON.c - -# sqlite3_mem: links SQLite amalgamation -SQLITE_COMMON := -DSQLITE_OMIT_WAL -DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_THREADSAFE=0 \ - -DSQLITE_OMIT_LOCALTIME -SQLITE_WASM := -DSQLITE_OS_OTHER $(SQLITE_COMMON) -lwasi-emulated-signal -lwasi-emulated-mman -SQLITE_NATIVE := $(SQLITE_COMMON) - -$(BUILD_DIR)/sqlite3_mem: programs/sqlite3_mem.c libs/sqlite3/sqlite3.c $(WASI_SDK_DIR)/bin/clang - @mkdir -p $(BUILD_DIR) - $(CC) --target=wasm32-wasip1 --sysroot=$(SYSROOT) -Os -I include/ \ - $(SQLITE_WASM) -Ilibs/sqlite3 -Wl,--initial-memory=16777216 -o $@ \ - programs/sqlite3_mem.c libs/sqlite3/sqlite3.c - -$(NATIVE_DIR)/sqlite3_mem: programs/sqlite3_mem.c libs/sqlite3/sqlite3.c - @mkdir -p $(NATIVE_DIR) - $(NATIVE_CC) $(NATIVE_CFLAGS) $(SQLITE_NATIVE) -Ilibs/sqlite3 -o $@ \ - programs/sqlite3_mem.c libs/sqlite3/sqlite3.c -lm -lpthread - -# sqlite3_cli: full CLI, links SQLite amalgamation (installed as sqlite3) -# Uses -Os (not -O2) + no wasm-opt — higher optimization breaks SQLite's -# internal function pointer tables in WASM indirect call table. -# Uses --initial-memory=16777216 (16MB) for SQLite's malloc requirements. -$(BUILD_DIR)/sqlite3_cli: programs/sqlite3_cli.c libs/sqlite3/sqlite3.c $(WASI_SDK_DIR)/bin/clang - @mkdir -p $(BUILD_DIR) - $(CC) --target=wasm32-wasip1 --sysroot=$(SYSROOT) -Os -I include/ \ - $(SQLITE_WASM) -Ilibs/sqlite3 -Wl,--initial-memory=16777216 \ - -o $@ programs/sqlite3_cli.c libs/sqlite3/sqlite3.c - -$(NATIVE_DIR)/sqlite3_cli: programs/sqlite3_cli.c libs/sqlite3/sqlite3.c - @mkdir -p $(NATIVE_DIR) - $(NATIVE_CC) $(NATIVE_CFLAGS) $(SQLITE_NATIVE) -Ilibs/sqlite3 -o $@ \ - programs/sqlite3_cli.c libs/sqlite3/sqlite3.c -lm -lpthread - -# zip: links zlib + minizip -ZLIB_SRCS := libs/zlib/adler32.c libs/zlib/compress.c libs/zlib/crc32.c libs/zlib/deflate.c \ - libs/zlib/infback.c libs/zlib/inffast.c libs/zlib/inflate.c libs/zlib/inftrees.c \ - libs/zlib/trees.c libs/zlib/uncompr.c libs/zlib/zutil.c -MINIZIP_SRCS := libs/minizip/ioapi.c libs/minizip/zip.c -ZIP_INCLUDES := -Ilibs/zlib -Ilibs/minizip - -$(BUILD_DIR)/zip: programs/zip.c $(ZLIB_SRCS) $(MINIZIP_SRCS) $(WASI_SDK_DIR)/bin/clang - @mkdir -p $(BUILD_DIR) - $(CC) $(WASM_CFLAGS) $(ZIP_INCLUDES) -o $@.wasm programs/zip.c $(ZLIB_SRCS) $(MINIZIP_SRCS) - @if [ "$(HAS_WASM_OPT)" = "1" ]; then \ - wasm-opt -O3 --strip-debug $@.wasm -o $@; \ - else \ - cp $@.wasm $@; \ - fi - @rm -f $@.wasm - -$(NATIVE_DIR)/zip: programs/zip.c $(ZLIB_SRCS) $(MINIZIP_SRCS) - @mkdir -p $(NATIVE_DIR) - $(NATIVE_CC) $(NATIVE_CFLAGS) $(ZIP_INCLUDES) -o $@ programs/zip.c $(ZLIB_SRCS) $(MINIZIP_SRCS) - -# unzip: links zlib + minizip (unzip side) -MINIZIP_UNZIP_SRCS := libs/minizip/ioapi.c libs/minizip/unzip.c - -$(BUILD_DIR)/unzip: programs/unzip.c $(ZLIB_SRCS) $(MINIZIP_UNZIP_SRCS) $(WASI_SDK_DIR)/bin/clang - @mkdir -p $(BUILD_DIR) - $(CC) $(WASM_CFLAGS) $(ZIP_INCLUDES) -o $@.wasm programs/unzip.c $(ZLIB_SRCS) $(MINIZIP_UNZIP_SRCS) - @if [ "$(HAS_WASM_OPT)" = "1" ]; then \ - wasm-opt -O3 --strip-debug $@.wasm -o $@; \ - else \ - cp $@.wasm $@; \ - fi - @rm -f $@.wasm - -$(NATIVE_DIR)/unzip: programs/unzip.c $(ZLIB_SRCS) $(MINIZIP_UNZIP_SRCS) - @mkdir -p $(NATIVE_DIR) - $(NATIVE_CC) $(NATIVE_CFLAGS) $(ZIP_INCLUDES) -o $@ programs/unzip.c $(ZLIB_SRCS) $(MINIZIP_UNZIP_SRCS) - -# curl_test: links libcurl (HTTP/HTTPS build for WASM via host_net + host_tls) -CURL_SRCS := $(wildcard libs/curl/lib/*.c) $(wildcard libs/curl/lib/vauth/*.c) \ - $(wildcard libs/curl/lib/vtls/*.c) $(wildcard libs/curl/lib/vquic/*.c) \ - $(wildcard libs/curl/lib/vssh/*.c) -CURL_INCLUDES := -Ilibs/curl/include -Ilibs/curl/lib -include libs/curl/lib/curl_setup.h -include libs/curl/lib/curl_printf.h -CURL_LIB_DEFS := -DHAVE_CONFIG_H -DBUILDING_LIBCURL -D_WASI_EMULATED_SIGNAL -DHAVE_BASENAME -DHAVE_LIBGEN_H - -$(BUILD_DIR)/curl: scripts/build-curl-upstream.sh $(CURL_UPSTREAM_OVERLAY_FILES) $(WASI_SDK_DIR)/bin/clang - @mkdir -p $(BUILD_DIR) - bash scripts/build-curl-upstream.sh \ - --version "$(CURL_RELEASE_VERSION)" \ - --tag "$(CURL_RELEASE_TAG)" \ - --url "$(CURL_RELEASE_URL)" \ - --cache-dir "$(abspath $(LIBS_CACHE))" \ - --build-dir "$(abspath $(CURL_UPSTREAM_BUILD_DIR))" \ - --overlay-dir "$(abspath $(CURL_UPSTREAM_OVERLAY_DIR))" \ - --cc "$(abspath $(CC)) --target=wasm32-wasip1 --sysroot=$(abspath $(SYSROOT))" \ - --ar "$(abspath $(WASI_SDK_DIR)/bin/llvm-ar)" \ - --ranlib "$(abspath $(WASI_SDK_DIR)/bin/llvm-ranlib)" \ - --output "$(abspath $@)" - -# wget: minimal wget built on libcurl -$(BUILD_DIR)/wget: programs/wget.c $(CURL_SRCS) $(WASI_SDK_DIR)/bin/clang - @mkdir -p $(BUILD_DIR) - $(CC) $(WASM_CFLAGS) $(CURL_LIB_DEFS) $(CURL_INCLUDES) \ - -lwasi-emulated-signal \ - -o $@.wasm programs/wget.c $(CURL_SRCS) - @if [ "$(HAS_WASM_OPT)" = "1" ]; then \ - wasm-opt -O3 --strip-debug --all-features $@.wasm -o $@; \ - else \ - cp $@.wasm $@; \ - fi - @rm -f $@.wasm - -$(NATIVE_DIR)/wget: programs/wget.c - @mkdir -p $(NATIVE_DIR) - $(NATIVE_CC) $(NATIVE_CFLAGS) -o $@ programs/wget.c -lcurl - -# duckdb: upstream DuckDB CLI built from source with our patched WASI/POSIX sysroot -$(BUILD_DIR)/duckdb: libs/duckdb/CMakeLists.txt $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a $(WASI_SDK_DIR)/bin/clang $(WASI_SDK_DIR)/bin/clang++ cmake/FindThreads.cmake scripts/build-duckdb.sh include/fcntl.h include/ifaddrs.h include/net/if.h include/sys/ioctl.h - @mkdir -p $(BUILD_DIR) - DUCKDB_SRC_DIR="$(abspath $(LIBS_DIR)/duckdb)" \ - DUCKDB_BUILD_DIR="$(abspath $(BUILD_DIR)/duckdb-cmake)" \ - DUCKDB_OUTPUT="$(abspath $(BUILD_DIR)/duckdb)" \ - WASI_SDK_DIR="$(abspath $(WASI_SDK_DIR))" \ - SYSROOT_DIR="$(abspath $(PATCHED_SYSROOT))" \ - MODULE_PATH="$(abspath cmake)" \ - OVERLAY_INCLUDE_DIR="$(abspath include)" \ - DUCKDB_GIT_DESCRIBE="$(DUCKDB_GIT_DESCRIBE)" \ - bash scripts/build-duckdb.sh - -# --- Compile all C programs to native binaries (for parity testing) --- - -native: $(NATIVE_OUTPUTS) - @echo "" - @echo "=== Native Build Report ===" - @echo "Programs: $(words $(NATIVE_PROG_NAMES)) compiled" - @echo "Output: $(NATIVE_DIR)/" - @echo "=== Build complete ===" - -$(NATIVE_DIR)/%: programs/%.c - @mkdir -p $(NATIVE_DIR) - $(NATIVE_CC) $(NATIVE_CFLAGS) -o $@ $< - -# --- Install opted-in commands to COMMANDS_DIR --- - -install: - @if [ -z "$(COMMANDS)" ]; then \ - echo "No C commands opted in for install (COMMANDS is empty)"; \ - exit 0; \ - fi - @if [ ! -d "$(BUILD_DIR)" ]; then \ - echo "Error: $(BUILD_DIR) not found. Run 'make programs' first."; \ - exit 1; \ - fi - @mkdir -p $(COMMANDS_DIR) - @INSTALLED=0; \ - for cmd in $(COMMANDS); do \ - src="$$cmd"; \ - case "$$cmd" in \ - sqlite3) src="sqlite3_cli" ;; \ - esac; \ - if [ -f "$(BUILD_DIR)/$$src" ]; then \ - cp "$(BUILD_DIR)/$$src" "$(COMMANDS_DIR)/$$cmd"; \ - INSTALLED=$$((INSTALLED + 1)); \ - else \ - echo "Warning: $(BUILD_DIR)/$$src not found — skipping"; \ - fi; \ - done; \ - echo "Installed $$INSTALLED C command(s) to $(COMMANDS_DIR)" - @# Create symlinks for git remote helpers - @if [ -f "$(COMMANDS_DIR)/git" ]; then \ - ln -sf git "$(COMMANDS_DIR)/git-remote-http"; \ - ln -sf git "$(COMMANDS_DIR)/git-remote-https"; \ - fi - -# --- Clean --- - -clean: - rm -rf $(BUILD_DIR) - -# --- vim: real vim 9.2 -> wasm32-wasip1 against the full-OS libc --- -# -# vim compiles as a normal terminal program against the PATCHED sysroot plus a -# small bridge library (vim/): termios via host_tty (raw-mode + winsize), -# a termcap stub (builtin termcaps), and POSIX stubs the full-OS libc lacks. -# Recipe recovered from the original hand build (see vim/ headers). The result -# lands in $(BUILD_DIR)/vim; `make -C .. cmd/vim` installs it into the shared -# commands dir. NOTE: run as `vim -n ` docs-wise is no longer needed — -# swap files work; kept here for archaeology. -VIM_VERSION ?= v9.2.0780 -VIM_SRC := libs/vim -VIM_BRIDGE_SRC := vim -VIM_BRIDGE_BUILD := $(BUILD_DIR)/vim-bridge -VIM_BRIDGE_OBJS := $(VIM_BRIDGE_BUILD)/termios_bridge.o $(VIM_BRIDGE_BUILD)/termcap_stub.o $(VIM_BRIDGE_BUILD)/posix_stubs.o -VIM_WCC := $(VIM_BRIDGE_BUILD)/wcc - -.PHONY: fetch-vim -fetch-vim: $(VIM_SRC)/src/vim.h -$(VIM_SRC)/src/vim.h: - git clone --depth 1 --branch $(VIM_VERSION) https://github.com/vim/vim $(VIM_SRC) - -$(VIM_BRIDGE_BUILD)/%.o: $(VIM_BRIDGE_SRC)/%.c $(WASI_SDK_DIR)/bin/clang sysroot - @mkdir -p $(VIM_BRIDGE_BUILD) - $(CC) --target=wasm32-wasip1 --sysroot=$(PATCHED_SYSROOT) -O2 -I $(VIM_BRIDGE_SRC) -c -o $@ $< - -$(VIM_BRIDGE_BUILD)/libtermcap.a: $(VIM_BRIDGE_OBJS) - $(WASI_SDK_DIR)/bin/llvm-ar rcs $@ $(VIM_BRIDGE_OBJS) - -# Wrapper compiler: every vim TU gets the sysroot, the bridge headers, and the -# forced compat include, so vim's own Makefile needs no flag surgery. -$(VIM_WCC): $(WASI_SDK_DIR)/bin/clang - @mkdir -p $(VIM_BRIDGE_BUILD) - @printf '#!/bin/sh\nexec %s --target=wasm32-wasip1 --sysroot=%s -O2 -I %s -include %s "$$@"\n' \ - "$(abspath $(WASI_SDK_DIR)/bin/clang)" \ - "$(abspath $(PATCHED_SYSROOT))" \ - "$(abspath $(VIM_BRIDGE_SRC))" \ - "$(abspath $(VIM_BRIDGE_SRC)/compat.h)" > $@ - @chmod +x $@ - -$(BUILD_DIR)/vim: fetch-vim $(VIM_BRIDGE_BUILD)/libtermcap.a $(VIM_WCC) wasm-opt-check - cd $(VIM_SRC)/src && \ - CC="$(abspath $(VIM_WCC))" \ - LDFLAGS="$(abspath $(VIM_BRIDGE_BUILD)/termios_bridge.o) -L$(abspath $(VIM_BRIDGE_BUILD))" \ - vim_cv_toupper_broken=no \ - vim_cv_terminfo=no \ - vim_cv_tgetent=zero \ - vim_cv_getcwd_broken=no \ - vim_cv_stat_ignores_slash=no \ - vim_cv_memmove_handles_overlap=yes \ - vim_cv_bcopy_handles_overlap=yes \ - vim_cv_memcpy_handles_overlap=yes \ - vim_cv_tty_group=world \ - vim_cv_tty_mode=0620 \ - ./configure --host=wasm32-wasip1 --build=$$(cc -dumpmachine 2>/dev/null || echo x86_64-pc-linux-gnu) \ - --with-features=normal --with-tlib=termcap \ - --disable-gui --without-x --without-wayland --disable-nls --disable-channel \ - --disable-netbeans --disable-selinux --disable-gpm --disable-sysmouse \ - --disable-icon-cache-update --disable-desktop-database-update - $(MAKE) -C $(VIM_SRC)/src vim - @mkdir -p $(BUILD_DIR) - @if [ "$(HAS_WASM_OPT)" = "1" ]; then \ - wasm-opt -O3 --strip-debug --all-features $(VIM_SRC)/src/vim -o $(BUILD_DIR)/vim; \ - else \ - cp $(VIM_SRC)/src/vim $(BUILD_DIR)/vim; \ - fi - @echo "Built $(BUILD_DIR)/vim" diff --git a/registry/native/c/curl-upstream-overlay/lib/hostip.c b/registry/native/c/curl-upstream-overlay/lib/hostip.c deleted file mode 100644 index f858d50233..0000000000 --- a/registry/native/c/curl-upstream-overlay/lib/hostip.c +++ /dev/null @@ -1,1490 +0,0 @@ -/*************************************************************************** - * _ _ ____ _ - * Project ___| | | | _ \| | - * / __| | | | |_) | | - * | (__| |_| | _ <| |___ - * \___|\___/|_| \_\_____| - * - * Copyright (C) Daniel Stenberg, , et al. - * - * This software is licensed as described in the file COPYING, which - * you should have received as part of this distribution. The terms - * are also available at https://curl.se/docs/copyright.html. - * - * You may opt to use, copy, modify, merge, publish, distribute and/or sell - * copies of the Software, and permit persons to whom the Software is - * furnished to do so, under the terms of the COPYING file. - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY - * KIND, either express or implied. - * - * SPDX-License-Identifier: curl - * - ***************************************************************************/ - -#include "curl_setup.h" - -#ifdef HAVE_NETINET_IN_H -#include -#endif -#ifdef HAVE_NETINET_IN6_H -#include -#endif -#ifdef HAVE_NETDB_H -#include -#endif -#ifdef HAVE_ARPA_INET_H -#include -#endif -#ifdef __VMS -#include -#include -#endif - -#ifndef __wasi__ -#include -#include -#endif - -#include "urldata.h" -#include "sendf.h" -#include "hostip.h" -#include "hash.h" -#include "rand.h" -#include "share.h" -#include "url.h" -#include "inet_ntop.h" -#include "inet_pton.h" -#include "multiif.h" -#include "doh.h" -#include "warnless.h" -#include "strcase.h" -#include "easy_lock.h" -/* The last 3 #include files should be in this order */ -#include "curl_printf.h" -#include "curl_memory.h" -#include "memdebug.h" - -#if defined(CURLRES_SYNCH) && \ - defined(HAVE_ALARM) && \ - defined(SIGALRM) && \ - defined(HAVE_SIGSETJMP) && \ - defined(GLOBAL_INIT_IS_THREADSAFE) -/* alarm-based timeouts can only be used with all the dependencies satisfied */ -#define USE_ALARM_TIMEOUT -#endif - -#define MAX_HOSTCACHE_LEN (255 + 7) /* max FQDN + colon + port number + zero */ - -#define MAX_DNS_CACHE_SIZE 29999 - -/* - * hostip.c explained - * ================== - * - * The main COMPILE-TIME DEFINES to keep in mind when reading the host*.c - * source file are these: - * - * CURLRES_IPV6 - this host has getaddrinfo() and family, and thus we use - * that. The host may not be able to resolve IPv6, but we do not really have to - * take that into account. Hosts that are not IPv6-enabled have CURLRES_IPV4 - * defined. - * - * CURLRES_ARES - is defined if libcurl is built to use c-ares for - * asynchronous name resolves. This can be Windows or *nix. - * - * CURLRES_THREADED - is defined if libcurl is built to run under (native) - * Windows, and then the name resolve will be done in a new thread, and the - * supported API will be the same as for ares-builds. - * - * If any of the two previous are defined, CURLRES_ASYNCH is defined too. If - * libcurl is not built to use an asynchronous resolver, CURLRES_SYNCH is - * defined. - * - * The host*.c sources files are split up like this: - * - * hostip.c - method-independent resolver functions and utility functions - * hostasyn.c - functions for asynchronous name resolves - * hostsyn.c - functions for synchronous name resolves - * hostip4.c - IPv4 specific functions - * hostip6.c - IPv6 specific functions - * - * The two asynchronous name resolver backends are implemented in: - * asyn-ares.c - functions for ares-using name resolves - * asyn-thread.c - functions for threaded name resolves - - * The hostip.h is the united header file for all this. It defines the - * CURLRES_* defines based on the config*.h and curl_setup.h defines. - */ - -static void hostcache_unlink_entry(void *entry); - -#ifndef CURL_DISABLE_VERBOSE_STRINGS -static void show_resolve_info(struct Curl_easy *data, - struct Curl_dns_entry *dns); -#else -#define show_resolve_info(x,y) Curl_nop_stmt -#endif - -/* - * Curl_printable_address() stores a printable version of the 1st address - * given in the 'ai' argument. The result will be stored in the buf that is - * bufsize bytes big. - * - * If the conversion fails, the target buffer is empty. - */ -void Curl_printable_address(const struct Curl_addrinfo *ai, char *buf, - size_t bufsize) -{ - DEBUGASSERT(bufsize); - buf[0] = 0; - - switch(ai->ai_family) { - case AF_INET: { - const struct sockaddr_in *sa4 = (const void *)ai->ai_addr; - const struct in_addr *ipaddr4 = &sa4->sin_addr; - (void)Curl_inet_ntop(ai->ai_family, (const void *)ipaddr4, buf, bufsize); - break; - } -#ifdef USE_IPV6 - case AF_INET6: { - const struct sockaddr_in6 *sa6 = (const void *)ai->ai_addr; - const struct in6_addr *ipaddr6 = &sa6->sin6_addr; - (void)Curl_inet_ntop(ai->ai_family, (const void *)ipaddr6, buf, bufsize); - break; - } -#endif - default: - break; - } -} - -/* - * Create a hostcache id string for the provided host + port, to be used by - * the DNS caching. Without alloc. Return length of the id string. - */ -static size_t -create_hostcache_id(const char *name, - size_t nlen, /* 0 or actual name length */ - int port, char *ptr, size_t buflen) -{ - size_t len = nlen ? nlen : strlen(name); - DEBUGASSERT(buflen >= MAX_HOSTCACHE_LEN); - if(len > (buflen - 7)) - len = buflen - 7; - /* store and lower case the name */ - Curl_strntolower(ptr, name, len); - return msnprintf(&ptr[len], 7, ":%u", port) + len; -} - -struct hostcache_prune_data { - time_t now; - time_t oldest; /* oldest time in cache not pruned. */ - int max_age_sec; -}; - -/* - * This function is set as a callback to be called for every entry in the DNS - * cache when we want to prune old unused entries. - * - * Returning non-zero means remove the entry, return 0 to keep it in the - * cache. - */ -static int -hostcache_entry_is_stale(void *datap, void *hc) -{ - struct hostcache_prune_data *prune = - (struct hostcache_prune_data *) datap; - struct Curl_dns_entry *dns = (struct Curl_dns_entry *) hc; - - if(dns->timestamp) { - /* age in seconds */ - time_t age = prune->now - dns->timestamp; - if(age >= prune->max_age_sec) - return TRUE; - if(age > prune->oldest) - prune->oldest = age; - } - return FALSE; -} - -/* - * Prune the DNS cache. This assumes that a lock has already been taken. - * Returns the 'age' of the oldest still kept entry. - */ -static time_t -hostcache_prune(struct Curl_hash *hostcache, int cache_timeout, - time_t now) -{ - struct hostcache_prune_data user; - - user.max_age_sec = cache_timeout; - user.now = now; - user.oldest = 0; - - Curl_hash_clean_with_criterium(hostcache, - (void *) &user, - hostcache_entry_is_stale); - - return user.oldest; -} - -/* - * Library-wide function for pruning the DNS cache. This function takes and - * returns the appropriate locks. - */ -void Curl_hostcache_prune(struct Curl_easy *data) -{ - time_t now; - /* the timeout may be set -1 (forever) */ - int timeout = data->set.dns_cache_timeout; - - if(!data->dns.hostcache) - /* NULL hostcache means we cannot do it */ - return; - - if(data->share) - Curl_share_lock(data, CURL_LOCK_DATA_DNS, CURL_LOCK_ACCESS_SINGLE); - - now = time(NULL); - - do { - /* Remove outdated and unused entries from the hostcache */ - time_t oldest = hostcache_prune(data->dns.hostcache, timeout, now); - - if(oldest < INT_MAX) - timeout = (int)oldest; /* we know it fits */ - else - timeout = INT_MAX - 1; - - /* if the cache size is still too big, use the oldest age as new - prune limit */ - } while(timeout && - (Curl_hash_count(data->dns.hostcache) > MAX_DNS_CACHE_SIZE)); - - if(data->share) - Curl_share_unlock(data, CURL_LOCK_DATA_DNS); -} - -#ifdef USE_ALARM_TIMEOUT -/* Beware this is a global and unique instance. This is used to store the - return address that we can jump back to from inside a signal handler. This - is not thread-safe stuff. */ -static sigjmp_buf curl_jmpenv; -static curl_simple_lock curl_jmpenv_lock; -#endif - -/* lookup address, returns entry if found and not stale */ -static struct Curl_dns_entry *fetch_addr(struct Curl_easy *data, - const char *hostname, - int port) -{ - struct Curl_dns_entry *dns = NULL; - char entry_id[MAX_HOSTCACHE_LEN]; - - /* Create an entry id, based upon the hostname and port */ - size_t entry_len = create_hostcache_id(hostname, 0, port, - entry_id, sizeof(entry_id)); - - /* See if it is already in our dns cache */ - dns = Curl_hash_pick(data->dns.hostcache, entry_id, entry_len + 1); - - /* No entry found in cache, check if we might have a wildcard entry */ - if(!dns && data->state.wildcard_resolve) { - entry_len = create_hostcache_id("*", 1, port, entry_id, sizeof(entry_id)); - - /* See if it is already in our dns cache */ - dns = Curl_hash_pick(data->dns.hostcache, entry_id, entry_len + 1); - } - - if(dns && (data->set.dns_cache_timeout != -1)) { - /* See whether the returned entry is stale. Done before we release lock */ - struct hostcache_prune_data user; - - user.now = time(NULL); - user.max_age_sec = data->set.dns_cache_timeout; - user.oldest = 0; - - if(hostcache_entry_is_stale(&user, dns)) { - infof(data, "Hostname in DNS cache was stale, zapped"); - dns = NULL; /* the memory deallocation is being handled by the hash */ - Curl_hash_delete(data->dns.hostcache, entry_id, entry_len + 1); - } - } - - /* See if the returned entry matches the required resolve mode */ - if(dns && data->conn->ip_version != CURL_IPRESOLVE_WHATEVER) { - int pf = PF_INET; - bool found = FALSE; - struct Curl_addrinfo *addr = dns->addr; - -#ifdef PF_INET6 - if(data->conn->ip_version == CURL_IPRESOLVE_V6) - pf = PF_INET6; -#endif - - while(addr) { - if(addr->ai_family == pf) { - found = TRUE; - break; - } - addr = addr->ai_next; - } - - if(!found) { - infof(data, "Hostname in DNS cache does not have needed family, zapped"); - dns = NULL; /* the memory deallocation is being handled by the hash */ - Curl_hash_delete(data->dns.hostcache, entry_id, entry_len + 1); - } - } - return dns; -} - -/* - * Curl_fetch_addr() fetches a 'Curl_dns_entry' already in the DNS cache. - * - * Curl_resolv() checks initially and multi_runsingle() checks each time - * it discovers the handle in the state WAITRESOLVE whether the hostname - * has already been resolved and the address has already been stored in - * the DNS cache. This short circuits waiting for a lot of pending - * lookups for the same hostname requested by different handles. - * - * Returns the Curl_dns_entry entry pointer or NULL if not in the cache. - * - * The returned data *MUST* be "released" with Curl_resolv_unlink() after - * use, or we will leak memory! - */ -struct Curl_dns_entry * -Curl_fetch_addr(struct Curl_easy *data, - const char *hostname, - int port) -{ - struct Curl_dns_entry *dns = NULL; - - if(data->share) - Curl_share_lock(data, CURL_LOCK_DATA_DNS, CURL_LOCK_ACCESS_SINGLE); - - dns = fetch_addr(data, hostname, port); - - if(dns) - dns->refcount++; /* we use it! */ - - if(data->share) - Curl_share_unlock(data, CURL_LOCK_DATA_DNS); - - return dns; -} - -#ifndef CURL_DISABLE_SHUFFLE_DNS -/* - * Return # of addresses in a Curl_addrinfo struct - */ -static int num_addresses(const struct Curl_addrinfo *addr) -{ - int i = 0; - while(addr) { - addr = addr->ai_next; - i++; - } - return i; -} - -UNITTEST CURLcode Curl_shuffle_addr(struct Curl_easy *data, - struct Curl_addrinfo **addr); -/* - * Curl_shuffle_addr() shuffles the order of addresses in a 'Curl_addrinfo' - * struct by re-linking its linked list. - * - * The addr argument should be the address of a pointer to the head node of a - * `Curl_addrinfo` list and it will be modified to point to the new head after - * shuffling. - * - * Not declared static only to make it easy to use in a unit test! - * - * @unittest: 1608 - */ -UNITTEST CURLcode Curl_shuffle_addr(struct Curl_easy *data, - struct Curl_addrinfo **addr) -{ - CURLcode result = CURLE_OK; - const int num_addrs = num_addresses(*addr); - - if(num_addrs > 1) { - struct Curl_addrinfo **nodes; - infof(data, "Shuffling %i addresses", num_addrs); - - nodes = malloc(num_addrs*sizeof(*nodes)); - if(nodes) { - int i; - unsigned int *rnd; - const size_t rnd_size = num_addrs * sizeof(*rnd); - - /* build a plain array of Curl_addrinfo pointers */ - nodes[0] = *addr; - for(i = 1; i < num_addrs; i++) { - nodes[i] = nodes[i-1]->ai_next; - } - - rnd = malloc(rnd_size); - if(rnd) { - /* Fisher-Yates shuffle */ - if(Curl_rand(data, (unsigned char *)rnd, rnd_size) == CURLE_OK) { - struct Curl_addrinfo *swap_tmp; - for(i = num_addrs - 1; i > 0; i--) { - swap_tmp = nodes[rnd[i] % (unsigned int)(i + 1)]; - nodes[rnd[i] % (unsigned int)(i + 1)] = nodes[i]; - nodes[i] = swap_tmp; - } - - /* relink list in the new order */ - for(i = 1; i < num_addrs; i++) { - nodes[i-1]->ai_next = nodes[i]; - } - - nodes[num_addrs-1]->ai_next = NULL; - *addr = nodes[0]; - } - free(rnd); - } - else - result = CURLE_OUT_OF_MEMORY; - free(nodes); - } - else - result = CURLE_OUT_OF_MEMORY; - } - return result; -} -#endif - -/* - * Curl_cache_addr() stores a 'Curl_addrinfo' struct in the DNS cache. - * - * When calling Curl_resolv() has resulted in a response with a returned - * address, we call this function to store the information in the dns - * cache etc - * - * Returns the Curl_dns_entry entry pointer or NULL if the storage failed. - */ -struct Curl_dns_entry * -Curl_cache_addr(struct Curl_easy *data, - struct Curl_addrinfo *addr, - const char *hostname, - size_t hostlen, /* length or zero */ - int port, - bool permanent) -{ - char entry_id[MAX_HOSTCACHE_LEN]; - size_t entry_len; - struct Curl_dns_entry *dns; - struct Curl_dns_entry *dns2; - -#ifndef CURL_DISABLE_SHUFFLE_DNS - /* shuffle addresses if requested */ - if(data->set.dns_shuffle_addresses) { - CURLcode result = Curl_shuffle_addr(data, &addr); - if(result) - return NULL; - } -#endif - if(!hostlen) - hostlen = strlen(hostname); - - /* Create a new cache entry */ - dns = calloc(1, sizeof(struct Curl_dns_entry) + hostlen); - if(!dns) { - return NULL; - } - - /* Create an entry id, based upon the hostname and port */ - entry_len = create_hostcache_id(hostname, hostlen, port, - entry_id, sizeof(entry_id)); - - dns->refcount = 1; /* the cache has the first reference */ - dns->addr = addr; /* this is the address(es) */ - if(permanent) - dns->timestamp = 0; /* an entry that never goes stale */ - else { - dns->timestamp = time(NULL); - if(dns->timestamp == 0) - dns->timestamp = 1; - } - dns->hostport = port; - if(hostlen) - memcpy(dns->hostname, hostname, hostlen); - - /* Store the resolved data in our DNS cache. */ - dns2 = Curl_hash_add(data->dns.hostcache, entry_id, entry_len + 1, - (void *)dns); - if(!dns2) { - free(dns); - return NULL; - } - - dns = dns2; - dns->refcount++; /* mark entry as in-use */ - return dns; -} - -#ifdef USE_IPV6 -/* return a static IPv6 ::1 for the name */ -static struct Curl_addrinfo *get_localhost6(int port, const char *name) -{ - struct Curl_addrinfo *ca; - const size_t ss_size = sizeof(struct sockaddr_in6); - const size_t hostlen = strlen(name); - struct sockaddr_in6 sa6; - unsigned char ipv6[16]; - unsigned short port16 = (unsigned short)(port & 0xffff); - ca = calloc(1, sizeof(struct Curl_addrinfo) + ss_size + hostlen + 1); - if(!ca) - return NULL; - - sa6.sin6_family = AF_INET6; - sa6.sin6_port = htons(port16); - sa6.sin6_flowinfo = 0; - sa6.sin6_scope_id = 0; - - (void)Curl_inet_pton(AF_INET6, "::1", ipv6); - memcpy(&sa6.sin6_addr, ipv6, sizeof(ipv6)); - - ca->ai_flags = 0; - ca->ai_family = AF_INET6; - ca->ai_socktype = SOCK_STREAM; - ca->ai_protocol = IPPROTO_TCP; - ca->ai_addrlen = (curl_socklen_t)ss_size; - ca->ai_next = NULL; - ca->ai_addr = (void *)((char *)ca + sizeof(struct Curl_addrinfo)); - memcpy(ca->ai_addr, &sa6, ss_size); - ca->ai_canonname = (char *)ca->ai_addr + ss_size; - strcpy(ca->ai_canonname, name); - return ca; -} -#else -#define get_localhost6(x,y) NULL -#endif - -/* return a static IPv4 127.0.0.1 for the given name */ -static struct Curl_addrinfo *get_localhost(int port, const char *name) -{ - struct Curl_addrinfo *ca; - struct Curl_addrinfo *ca6; - const size_t ss_size = sizeof(struct sockaddr_in); - const size_t hostlen = strlen(name); - struct sockaddr_in sa; - unsigned int ipv4; - unsigned short port16 = (unsigned short)(port & 0xffff); - - /* memset to clear the sa.sin_zero field */ - memset(&sa, 0, sizeof(sa)); - sa.sin_family = AF_INET; - sa.sin_port = htons(port16); - if(Curl_inet_pton(AF_INET, "127.0.0.1", (char *)&ipv4) < 1) - return NULL; - memcpy(&sa.sin_addr, &ipv4, sizeof(ipv4)); - - ca = calloc(1, sizeof(struct Curl_addrinfo) + ss_size + hostlen + 1); - if(!ca) - return NULL; - ca->ai_flags = 0; - ca->ai_family = AF_INET; - ca->ai_socktype = SOCK_STREAM; - ca->ai_protocol = IPPROTO_TCP; - ca->ai_addrlen = (curl_socklen_t)ss_size; - ca->ai_addr = (void *)((char *)ca + sizeof(struct Curl_addrinfo)); - memcpy(ca->ai_addr, &sa, ss_size); - ca->ai_canonname = (char *)ca->ai_addr + ss_size; - strcpy(ca->ai_canonname, name); - - ca6 = get_localhost6(port, name); - if(!ca6) - return ca; - ca6->ai_next = ca; - return ca6; -} - -#ifdef USE_IPV6 -/* - * Curl_ipv6works() returns TRUE if IPv6 seems to work. - */ -bool Curl_ipv6works(struct Curl_easy *data) -{ - if(data) { - /* the nature of most system is that IPv6 status does not come and go - during a program's lifetime so we only probe the first time and then we - have the info kept for fast reuse */ - DEBUGASSERT(data); - DEBUGASSERT(data->multi); - if(data->multi->ipv6_up == IPV6_UNKNOWN) { - bool works = Curl_ipv6works(NULL); - data->multi->ipv6_up = works ? IPV6_WORKS : IPV6_DEAD; - } - return data->multi->ipv6_up == IPV6_WORKS; - } - else { - int ipv6_works = -1; - /* probe to see if we have a working IPv6 stack */ - curl_socket_t s = socket(PF_INET6, SOCK_DGRAM, 0); - if(s == CURL_SOCKET_BAD) - /* an IPv6 address was requested but we cannot get/use one */ - ipv6_works = 0; - else { - ipv6_works = 1; - sclose(s); - } - return (ipv6_works > 0); - } -} -#endif /* USE_IPV6 */ - -/* - * Curl_host_is_ipnum() returns TRUE if the given string is a numerical IPv4 - * (or IPv6 if supported) address. - */ -bool Curl_host_is_ipnum(const char *hostname) -{ - struct in_addr in; -#ifdef USE_IPV6 - struct in6_addr in6; -#endif - if(Curl_inet_pton(AF_INET, hostname, &in) > 0 -#ifdef USE_IPV6 - || Curl_inet_pton(AF_INET6, hostname, &in6) > 0 -#endif - ) - return TRUE; - return FALSE; -} - - -/* return TRUE if 'part' is a case insensitive tail of 'full' */ -static bool tailmatch(const char *full, const char *part) -{ - size_t plen = strlen(part); - size_t flen = strlen(full); - if(plen > flen) - return FALSE; - return strncasecompare(part, &full[flen - plen], plen); -} - -/* - * Curl_resolv() is the main name resolve function within libcurl. It resolves - * a name and returns a pointer to the entry in the 'entry' argument (if one - * is provided). This function might return immediately if we are using asynch - * resolves. See the return codes. - * - * The cache entry we return will get its 'inuse' counter increased when this - * function is used. You MUST call Curl_resolv_unlink() later (when you are - * done using this struct) to decrease the reference counter again. - * - * Return codes: - * - * CURLRESOLV_ERROR (-1) = error, no pointer - * CURLRESOLV_RESOLVED (0) = OK, pointer provided - * CURLRESOLV_PENDING (1) = waiting for response, no pointer - */ - -enum resolve_t Curl_resolv(struct Curl_easy *data, - const char *hostname, - int port, - bool allowDOH, - struct Curl_dns_entry **entry) -{ - struct Curl_dns_entry *dns = NULL; - CURLcode result; - enum resolve_t rc = CURLRESOLV_ERROR; /* default to failure */ - struct connectdata *conn = data->conn; - /* We should intentionally error and not resolve .onion TLDs */ - size_t hostname_len = strlen(hostname); - if(hostname_len >= 7 && - (curl_strequal(&hostname[hostname_len - 6], ".onion") || - curl_strequal(&hostname[hostname_len - 7], ".onion."))) { - failf(data, "Not resolving .onion address (RFC 7686)"); - return CURLRESOLV_ERROR; - } - *entry = NULL; -#ifndef CURL_DISABLE_DOH - conn->bits.doh = FALSE; /* default is not */ -#else - (void)allowDOH; -#endif - - if(data->share) - Curl_share_lock(data, CURL_LOCK_DATA_DNS, CURL_LOCK_ACCESS_SINGLE); - - dns = fetch_addr(data, hostname, port); - - if(dns) { - infof(data, "Hostname %s was found in DNS cache", hostname); - dns->refcount++; /* we use it! */ - rc = CURLRESOLV_RESOLVED; - } - - if(data->share) - Curl_share_unlock(data, CURL_LOCK_DATA_DNS); - - if(!dns) { - /* The entry was not in the cache. Resolve it to IP address */ - - struct Curl_addrinfo *addr = NULL; - int respwait = 0; -#if !defined(CURL_DISABLE_DOH) || !defined(USE_RESOLVE_ON_IPS) - struct in_addr in; -#endif -#ifndef CURL_DISABLE_DOH -#ifndef USE_RESOLVE_ON_IPS - const -#endif - bool ipnum = FALSE; -#endif - - /* notify the resolver start callback */ - if(data->set.resolver_start) { - int st; - Curl_set_in_callback(data, TRUE); - st = data->set.resolver_start( -#ifdef USE_CURL_ASYNC - data->state.async.resolver, -#else - NULL, -#endif - NULL, - data->set.resolver_start_client); - Curl_set_in_callback(data, FALSE); - if(st) - return CURLRESOLV_ERROR; - } - -#ifndef USE_RESOLVE_ON_IPS - /* First check if this is an IPv4 address string */ - if(Curl_inet_pton(AF_INET, hostname, &in) > 0) { - /* This is a dotted IP address 123.123.123.123-style */ - addr = Curl_ip2addr(AF_INET, &in, hostname, port); - if(!addr) - return CURLRESOLV_ERROR; - } -#ifdef USE_IPV6 - else { - struct in6_addr in6; - /* check if this is an IPv6 address string */ - if(Curl_inet_pton(AF_INET6, hostname, &in6) > 0) { - /* This is an IPv6 address literal */ - addr = Curl_ip2addr(AF_INET6, &in6, hostname, port); - if(!addr) - return CURLRESOLV_ERROR; - } - } -#endif /* USE_IPV6 */ - -#else /* if USE_RESOLVE_ON_IPS */ -#ifndef CURL_DISABLE_DOH - /* First check if this is an IPv4 address string */ - if(Curl_inet_pton(AF_INET, hostname, &in) > 0) - /* This is a dotted IP address 123.123.123.123-style */ - ipnum = TRUE; -#ifdef USE_IPV6 - else { - struct in6_addr in6; - /* check if this is an IPv6 address string */ - if(Curl_inet_pton(AF_INET6, hostname, &in6) > 0) - /* This is an IPv6 address literal */ - ipnum = TRUE; - } -#endif /* USE_IPV6 */ -#endif /* CURL_DISABLE_DOH */ - -#endif /* !USE_RESOLVE_ON_IPS */ - - if(!addr) { - if(conn->ip_version == CURL_IPRESOLVE_V6 && !Curl_ipv6works(data)) - return CURLRESOLV_ERROR; - - if(strcasecompare(hostname, "localhost") || - strcasecompare(hostname, "localhost.") || - tailmatch(hostname, ".localhost") || - tailmatch(hostname, ".localhost.")) - addr = get_localhost(port, hostname); -#ifndef CURL_DISABLE_DOH - else if(allowDOH && data->set.doh && !ipnum) - addr = Curl_doh(data, hostname, port, &respwait); -#endif - else { - /* Check what IP specifics the app has requested and if we can provide - * it. If not, bail out. */ - if(!Curl_ipvalid(data, conn)) - return CURLRESOLV_ERROR; - /* If Curl_getaddrinfo() returns NULL, 'respwait' might be set to a - non-zero value indicating that we need to wait for the response to - the resolve call */ - addr = Curl_getaddrinfo(data, hostname, port, &respwait); - } - } - if(!addr) { - if(respwait) { - /* the response to our resolve call will come asynchronously at - a later time, good or bad */ - /* First, check that we have not received the info by now */ - result = Curl_resolv_check(data, &dns); - if(result) /* error detected */ - return CURLRESOLV_ERROR; - if(dns) - rc = CURLRESOLV_RESOLVED; /* pointer provided */ - else - rc = CURLRESOLV_PENDING; /* no info yet */ - } - } - else { - if(data->share) - Curl_share_lock(data, CURL_LOCK_DATA_DNS, CURL_LOCK_ACCESS_SINGLE); - - /* we got a response, store it in the cache */ - dns = Curl_cache_addr(data, addr, hostname, 0, port, FALSE); - - if(data->share) - Curl_share_unlock(data, CURL_LOCK_DATA_DNS); - - if(!dns) - /* returned failure, bail out nicely */ - Curl_freeaddrinfo(addr); - else { - rc = CURLRESOLV_RESOLVED; - show_resolve_info(data, dns); - } - } - } - - *entry = dns; - - return rc; -} - -#ifdef USE_ALARM_TIMEOUT -/* - * This signal handler jumps back into the main libcurl code and continues - * execution. This effectively causes the remainder of the application to run - * within a signal handler which is nonportable and could lead to problems. - */ -CURL_NORETURN static -void alarmfunc(int sig) -{ - (void)sig; - siglongjmp(curl_jmpenv, 1); -} -#endif /* USE_ALARM_TIMEOUT */ - -/* - * Curl_resolv_timeout() is the same as Curl_resolv() but specifies a - * timeout. This function might return immediately if we are using asynch - * resolves. See the return codes. - * - * The cache entry we return will get its 'inuse' counter increased when this - * function is used. You MUST call Curl_resolv_unlink() later (when you are - * done using this struct) to decrease the reference counter again. - * - * If built with a synchronous resolver and use of signals is not - * disabled by the application, then a nonzero timeout will cause a - * timeout after the specified number of milliseconds. Otherwise, timeout - * is ignored. - * - * Return codes: - * - * CURLRESOLV_TIMEDOUT(-2) = warning, time too short or previous alarm expired - * CURLRESOLV_ERROR (-1) = error, no pointer - * CURLRESOLV_RESOLVED (0) = OK, pointer provided - * CURLRESOLV_PENDING (1) = waiting for response, no pointer - */ - -enum resolve_t Curl_resolv_timeout(struct Curl_easy *data, - const char *hostname, - int port, - struct Curl_dns_entry **entry, - timediff_t timeoutms) -{ -#ifdef USE_ALARM_TIMEOUT -#ifdef HAVE_SIGACTION - struct sigaction keep_sigact; /* store the old struct here */ - volatile bool keep_copysig = FALSE; /* whether old sigact has been saved */ - struct sigaction sigact; -#else -#ifdef HAVE_SIGNAL - void (*keep_sigact)(int); /* store the old handler here */ -#endif /* HAVE_SIGNAL */ -#endif /* HAVE_SIGACTION */ - volatile long timeout; - volatile unsigned int prev_alarm = 0; -#endif /* USE_ALARM_TIMEOUT */ - enum resolve_t rc; - - *entry = NULL; - - if(timeoutms < 0) - /* got an already expired timeout */ - return CURLRESOLV_TIMEDOUT; - -#ifdef USE_ALARM_TIMEOUT - if(data->set.no_signal) - /* Ignore the timeout when signals are disabled */ - timeout = 0; - else - timeout = (timeoutms > LONG_MAX) ? LONG_MAX : (long)timeoutms; - - if(!timeout) - /* USE_ALARM_TIMEOUT defined, but no timeout actually requested */ - return Curl_resolv(data, hostname, port, TRUE, entry); - - if(timeout < 1000) { - /* The alarm() function only provides integer second resolution, so if - we want to wait less than one second we must bail out already now. */ - failf(data, - "remaining timeout of %ld too small to resolve via SIGALRM method", - timeout); - return CURLRESOLV_TIMEDOUT; - } - /* This allows us to time-out from the name resolver, as the timeout - will generate a signal and we will siglongjmp() from that here. - This technique has problems (see alarmfunc). - This should be the last thing we do before calling Curl_resolv(), - as otherwise we would have to worry about variables that get modified - before we invoke Curl_resolv() (and thus use "volatile"). */ - curl_simple_lock_lock(&curl_jmpenv_lock); - - if(sigsetjmp(curl_jmpenv, 1)) { - /* this is coming from a siglongjmp() after an alarm signal */ - failf(data, "name lookup timed out"); - rc = CURLRESOLV_ERROR; - goto clean_up; - } - else { - /************************************************************* - * Set signal handler to catch SIGALRM - * Store the old value to be able to set it back later! - *************************************************************/ -#ifdef HAVE_SIGACTION - sigaction(SIGALRM, NULL, &sigact); - keep_sigact = sigact; - keep_copysig = TRUE; /* yes, we have a copy */ - sigact.sa_handler = alarmfunc; -#ifdef SA_RESTART - /* HP-UX does not have SA_RESTART but defaults to that behavior! */ - sigact.sa_flags &= ~SA_RESTART; -#endif - /* now set the new struct */ - sigaction(SIGALRM, &sigact, NULL); -#else /* HAVE_SIGACTION */ - /* no sigaction(), revert to the much lamer signal() */ -#ifdef HAVE_SIGNAL - keep_sigact = signal(SIGALRM, alarmfunc); -#endif -#endif /* HAVE_SIGACTION */ - - /* alarm() makes a signal get sent when the timeout fires off, and that - will abort system calls */ - prev_alarm = alarm(curlx_sltoui(timeout/1000L)); - } - -#else -#ifndef CURLRES_ASYNCH - if(timeoutms) - infof(data, "timeout on name lookup is not supported"); -#else - (void)timeoutms; /* timeoutms not used with an async resolver */ -#endif -#endif /* USE_ALARM_TIMEOUT */ - - /* Perform the actual name resolution. This might be interrupted by an - * alarm if it takes too long. - */ - rc = Curl_resolv(data, hostname, port, TRUE, entry); - -#ifdef USE_ALARM_TIMEOUT -clean_up: - - if(!prev_alarm) - /* deactivate a possibly active alarm before uninstalling the handler */ - alarm(0); - -#ifdef HAVE_SIGACTION - if(keep_copysig) { - /* we got a struct as it looked before, now put that one back nice - and clean */ - sigaction(SIGALRM, &keep_sigact, NULL); /* put it back */ - } -#else -#ifdef HAVE_SIGNAL - /* restore the previous SIGALRM handler */ - signal(SIGALRM, keep_sigact); -#endif -#endif /* HAVE_SIGACTION */ - - curl_simple_lock_unlock(&curl_jmpenv_lock); - - /* switch back the alarm() to either zero or to what it was before minus - the time we spent until now! */ - if(prev_alarm) { - /* there was an alarm() set before us, now put it back */ - timediff_t elapsed_secs = Curl_timediff(Curl_now(), - data->conn->created) / 1000; - - /* the alarm period is counted in even number of seconds */ - unsigned long alarm_set = (unsigned long)(prev_alarm - elapsed_secs); - - if(!alarm_set || - ((alarm_set >= 0x80000000) && (prev_alarm < 0x80000000)) ) { - /* if the alarm time-left reached zero or turned "negative" (counted - with unsigned values), we should fire off a SIGALRM here, but we - will not, and zero would be to switch it off so we never set it to - less than 1! */ - alarm(1); - rc = CURLRESOLV_TIMEDOUT; - failf(data, "Previous alarm fired off"); - } - else - alarm((unsigned int)alarm_set); - } -#endif /* USE_ALARM_TIMEOUT */ - - return rc; -} - -/* - * Curl_resolv_unlink() releases a reference to the given cached DNS entry. - * When the reference count reaches 0, the entry is destroyed. It is important - * that only one unlink is made for each Curl_resolv() call. - * - * May be called with 'data' == NULL for global cache. - */ -void Curl_resolv_unlink(struct Curl_easy *data, struct Curl_dns_entry **pdns) -{ - struct Curl_dns_entry *dns = *pdns; - *pdns = NULL; - if(data && data->share) - Curl_share_lock(data, CURL_LOCK_DATA_DNS, CURL_LOCK_ACCESS_SINGLE); - - hostcache_unlink_entry(dns); - - if(data && data->share) - Curl_share_unlock(data, CURL_LOCK_DATA_DNS); -} - -/* - * File-internal: release cache dns entry reference, free if inuse drops to 0 - */ -static void hostcache_unlink_entry(void *entry) -{ - struct Curl_dns_entry *dns = (struct Curl_dns_entry *) entry; - DEBUGASSERT(dns && (dns->refcount > 0)); - - dns->refcount--; - if(dns->refcount == 0) { - Curl_freeaddrinfo(dns->addr); -#ifdef USE_HTTPSRR - if(dns->hinfo) { - if(dns->hinfo->target) - free(dns->hinfo->target); - if(dns->hinfo->alpns) - free(dns->hinfo->alpns); - if(dns->hinfo->ipv4hints) - free(dns->hinfo->ipv4hints); - if(dns->hinfo->echconfiglist) - free(dns->hinfo->echconfiglist); - if(dns->hinfo->ipv6hints) - free(dns->hinfo->ipv6hints); - if(dns->hinfo->val) - free(dns->hinfo->val); - free(dns->hinfo); - } -#endif - free(dns); - } -} - -/* - * Curl_init_dnscache() inits a new DNS cache. - */ -void Curl_init_dnscache(struct Curl_hash *hash, size_t size) -{ - Curl_hash_init(hash, size, Curl_hash_str, Curl_str_key_compare, - hostcache_unlink_entry); -} - -/* - * Curl_hostcache_clean() - * - * This _can_ be called with 'data' == NULL but then of course no locking - * can be done! - */ - -void Curl_hostcache_clean(struct Curl_easy *data, - struct Curl_hash *hash) -{ - if(data && data->share) - Curl_share_lock(data, CURL_LOCK_DATA_DNS, CURL_LOCK_ACCESS_SINGLE); - - Curl_hash_clean(hash); - - if(data && data->share) - Curl_share_unlock(data, CURL_LOCK_DATA_DNS); -} - - -CURLcode Curl_loadhostpairs(struct Curl_easy *data) -{ - struct curl_slist *hostp; - char *host_end; - - /* Default is no wildcard found */ - data->state.wildcard_resolve = FALSE; - - for(hostp = data->state.resolve; hostp; hostp = hostp->next) { - char entry_id[MAX_HOSTCACHE_LEN]; - if(!hostp->data) - continue; - if(hostp->data[0] == '-') { - unsigned long num = 0; - size_t entry_len; - size_t hlen = 0; - host_end = strchr(&hostp->data[1], ':'); - - if(host_end) { - hlen = host_end - &hostp->data[1]; - num = strtoul(++host_end, NULL, 10); - if(!hlen || (num > 0xffff)) - host_end = NULL; - } - if(!host_end) { - infof(data, "Bad syntax CURLOPT_RESOLVE removal entry '%s'", - hostp->data); - continue; - } - /* Create an entry id, based upon the hostname and port */ - entry_len = create_hostcache_id(&hostp->data[1], hlen, (int)num, - entry_id, sizeof(entry_id)); - if(data->share) - Curl_share_lock(data, CURL_LOCK_DATA_DNS, CURL_LOCK_ACCESS_SINGLE); - - /* delete entry, ignore if it did not exist */ - Curl_hash_delete(data->dns.hostcache, entry_id, entry_len + 1); - - if(data->share) - Curl_share_unlock(data, CURL_LOCK_DATA_DNS); - } - else { - struct Curl_dns_entry *dns; - struct Curl_addrinfo *head = NULL, *tail = NULL; - size_t entry_len; - char address[64]; -#if !defined(CURL_DISABLE_VERBOSE_STRINGS) - char *addresses = NULL; -#endif - char *addr_begin; - char *addr_end; - char *port_ptr; - int port = 0; - char *end_ptr; - bool permanent = TRUE; - unsigned long tmp_port; - bool error = TRUE; - char *host_begin = hostp->data; - size_t hlen = 0; - - if(host_begin[0] == '+') { - host_begin++; - permanent = FALSE; - } - host_end = strchr(host_begin, ':'); - if(!host_end) - goto err; - hlen = host_end - host_begin; - - port_ptr = host_end + 1; - tmp_port = strtoul(port_ptr, &end_ptr, 10); - if(tmp_port > USHRT_MAX || end_ptr == port_ptr || *end_ptr != ':') - goto err; - - port = (int)tmp_port; -#if !defined(CURL_DISABLE_VERBOSE_STRINGS) - addresses = end_ptr + 1; -#endif - - while(*end_ptr) { - size_t alen; - struct Curl_addrinfo *ai; - - addr_begin = end_ptr + 1; - addr_end = strchr(addr_begin, ','); - if(!addr_end) - addr_end = addr_begin + strlen(addr_begin); - end_ptr = addr_end; - - /* allow IP(v6) address within [brackets] */ - if(*addr_begin == '[') { - if(addr_end == addr_begin || *(addr_end - 1) != ']') - goto err; - ++addr_begin; - --addr_end; - } - - alen = addr_end - addr_begin; - if(!alen) - continue; - - if(alen >= sizeof(address)) - goto err; - - memcpy(address, addr_begin, alen); - address[alen] = '\0'; - -#ifndef USE_IPV6 - if(strchr(address, ':')) { - infof(data, "Ignoring resolve address '%s', missing IPv6 support.", - address); - continue; - } -#endif - - ai = Curl_str2addr(address, port); - if(!ai) { - infof(data, "Resolve address '%s' found illegal", address); - goto err; - } - - if(tail) { - tail->ai_next = ai; - tail = tail->ai_next; - } - else { - head = tail = ai; - } - } - - if(!head) - goto err; - - error = FALSE; -err: - if(error) { - failf(data, "Couldn't parse CURLOPT_RESOLVE entry '%s'", - hostp->data); - Curl_freeaddrinfo(head); - return CURLE_SETOPT_OPTION_SYNTAX; - } - - /* Create an entry id, based upon the hostname and port */ - entry_len = create_hostcache_id(host_begin, hlen, port, - entry_id, sizeof(entry_id)); - - if(data->share) - Curl_share_lock(data, CURL_LOCK_DATA_DNS, CURL_LOCK_ACCESS_SINGLE); - - /* See if it is already in our dns cache */ - dns = Curl_hash_pick(data->dns.hostcache, entry_id, entry_len + 1); - - if(dns) { - infof(data, "RESOLVE %.*s:%d - old addresses discarded", - (int)hlen, host_begin, port); - /* delete old entry, there are two reasons for this - 1. old entry may have different addresses. - 2. even if entry with correct addresses is already in the cache, - but if it is close to expire, then by the time next http - request is made, it can get expired and pruned because old - entry is not necessarily marked as permanent. - 3. when adding a non-permanent entry, we want it to remove and - replace an existing permanent entry. - 4. when adding a non-permanent entry, we want it to get a "fresh" - timeout that starts _now_. */ - - Curl_hash_delete(data->dns.hostcache, entry_id, entry_len + 1); - } - - /* put this new host in the cache */ - dns = Curl_cache_addr(data, head, host_begin, hlen, port, permanent); - if(dns) { - /* release the returned reference; the cache itself will keep the - * entry alive: */ - dns->refcount--; - } - - if(data->share) - Curl_share_unlock(data, CURL_LOCK_DATA_DNS); - - if(!dns) { - Curl_freeaddrinfo(head); - return CURLE_OUT_OF_MEMORY; - } -#ifndef CURL_DISABLE_VERBOSE_STRINGS - infof(data, "Added %.*s:%d:%s to DNS cache%s", - (int)hlen, host_begin, port, addresses, - permanent ? "" : " (non-permanent)"); -#endif - - /* Wildcard hostname */ - if((hlen == 1) && (host_begin[0] == '*')) { - infof(data, "RESOLVE *:%d using wildcard", port); - data->state.wildcard_resolve = TRUE; - } - } - } - data->state.resolve = NULL; /* dealt with now */ - - return CURLE_OK; -} - -#ifndef CURL_DISABLE_VERBOSE_STRINGS -static void show_resolve_info(struct Curl_easy *data, - struct Curl_dns_entry *dns) -{ - struct Curl_addrinfo *a; - CURLcode result = CURLE_OK; -#ifdef CURLRES_IPV6 - struct dynbuf out[2]; -#else - struct dynbuf out[1]; -#endif - DEBUGASSERT(data); - DEBUGASSERT(dns); - - if(!data->set.verbose || - /* ignore no name or numerical IP addresses */ - !dns->hostname[0] || Curl_host_is_ipnum(dns->hostname)) - return; - - a = dns->addr; - - infof(data, "Host %s:%d was resolved.", - (dns->hostname[0] ? dns->hostname : "(none)"), dns->hostport); - - Curl_dyn_init(&out[0], 1024); -#ifdef CURLRES_IPV6 - Curl_dyn_init(&out[1], 1024); -#endif - - while(a) { - if( -#ifdef CURLRES_IPV6 - a->ai_family == PF_INET6 || -#endif - a->ai_family == PF_INET) { - char buf[MAX_IPADR_LEN]; - struct dynbuf *d = &out[(a->ai_family != PF_INET)]; - Curl_printable_address(a, buf, sizeof(buf)); - if(Curl_dyn_len(d)) - result = Curl_dyn_addn(d, ", ", 2); - if(!result) - result = Curl_dyn_add(d, buf); - if(result) { - infof(data, "too many IP, cannot show"); - goto fail; - } - } - a = a->ai_next; - } - -#ifdef CURLRES_IPV6 - infof(data, "IPv6: %s", - (Curl_dyn_len(&out[1]) ? Curl_dyn_ptr(&out[1]) : "(none)")); -#endif - infof(data, "IPv4: %s", - (Curl_dyn_len(&out[0]) ? Curl_dyn_ptr(&out[0]) : "(none)")); - -fail: - Curl_dyn_free(&out[0]); -#ifdef CURLRES_IPV6 - Curl_dyn_free(&out[1]); -#endif -} -#endif - -CURLcode Curl_resolv_check(struct Curl_easy *data, - struct Curl_dns_entry **dns) -{ - CURLcode result; -#if defined(CURL_DISABLE_DOH) && !defined(CURLRES_ASYNCH) - (void)data; - (void)dns; -#endif -#ifndef CURL_DISABLE_DOH - if(data->conn->bits.doh) { - result = Curl_doh_is_resolved(data, dns); - } - else -#endif - result = Curl_resolver_is_resolved(data, dns); - if(*dns) - show_resolve_info(data, *dns); - return result; -} - -int Curl_resolv_getsock(struct Curl_easy *data, - curl_socket_t *socks) -{ -#ifdef CURLRES_ASYNCH -#ifndef CURL_DISABLE_DOH - if(data->conn->bits.doh) - /* nothing to wait for during DoH resolve, those handles have their own - sockets */ - return GETSOCK_BLANK; -#endif - return Curl_resolver_getsock(data, socks); -#else - (void)data; - (void)socks; - return GETSOCK_BLANK; -#endif -} - -/* Call this function after Curl_connect() has returned async=TRUE and - then a successful name resolve has been received. - - Note: this function disconnects and frees the conn data in case of - resolve failure */ -CURLcode Curl_once_resolved(struct Curl_easy *data, bool *protocol_done) -{ - CURLcode result; - struct connectdata *conn = data->conn; - -#ifdef USE_CURL_ASYNC - if(data->state.async.dns) { - conn->dns_entry = data->state.async.dns; - data->state.async.dns = NULL; - } -#endif - - result = Curl_setup_conn(data, protocol_done); - - if(result) { - Curl_detach_connection(data); - Curl_cpool_disconnect(data, conn, TRUE); - } - return result; -} - -/* - * Curl_resolver_error() calls failf() with the appropriate message after a - * resolve error - */ - -#ifdef USE_CURL_ASYNC -CURLcode Curl_resolver_error(struct Curl_easy *data) -{ - const char *host_or_proxy; - CURLcode result; - -#ifndef CURL_DISABLE_PROXY - struct connectdata *conn = data->conn; - if(conn->bits.httpproxy) { - host_or_proxy = "proxy"; - result = CURLE_COULDNT_RESOLVE_PROXY; - } - else -#endif - { - host_or_proxy = "host"; - result = CURLE_COULDNT_RESOLVE_HOST; - } - - failf(data, "Could not resolve %s: %s", host_or_proxy, - data->state.async.hostname); - - return result; -} -#endif /* USE_CURL_ASYNC */ diff --git a/registry/native/c/curl-upstream-overlay/lib/hostip.h b/registry/native/c/curl-upstream-overlay/lib/hostip.h deleted file mode 100644 index c6a49718d6..0000000000 --- a/registry/native/c/curl-upstream-overlay/lib/hostip.h +++ /dev/null @@ -1,270 +0,0 @@ -#ifndef HEADER_CURL_HOSTIP_H -#define HEADER_CURL_HOSTIP_H -/*************************************************************************** - * _ _ ____ _ - * Project ___| | | | _ \| | - * / __| | | | |_) | | - * | (__| |_| | _ <| |___ - * \___|\___/|_| \_\_____| - * - * Copyright (C) Daniel Stenberg, , et al. - * - * This software is licensed as described in the file COPYING, which - * you should have received as part of this distribution. The terms - * are also available at https://curl.se/docs/copyright.html. - * - * You may opt to use, copy, modify, merge, publish, distribute and/or sell - * copies of the Software, and permit persons to whom the Software is - * furnished to do so, under the terms of the COPYING file. - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY - * KIND, either express or implied. - * - * SPDX-License-Identifier: curl - * - ***************************************************************************/ - -#include "curl_setup.h" -#include "hash.h" -#include "curl_addrinfo.h" -#include "timeval.h" /* for timediff_t */ -#include "asyn.h" - -/* setjmp.h requires WASM exception handling — not available on WASI */ -#ifndef __wasi__ -#include -#endif - -#ifdef USE_HTTPSRR -# include -#endif - -/* Allocate enough memory to hold the full name information structs and - * everything. OSF1 is known to require at least 8872 bytes. The buffer - * required for storing all possible aliases and IP numbers is according to - * Stevens' Unix Network Programming 2nd edition, p. 304: 8192 bytes! - */ -#define CURL_HOSTENT_SIZE 9000 - -#define CURL_TIMEOUT_RESOLVE 300 /* when using asynch methods, we allow this - many seconds for a name resolve */ - -#define CURL_ASYNC_SUCCESS CURLE_OK - -struct addrinfo; -struct hostent; -struct Curl_easy; -struct connectdata; - -/* - * Curl_global_host_cache_init() initializes and sets up a global DNS cache. - * Global DNS cache is general badness. Do not use. This will be removed in - * a future version. Use the share interface instead! - * - * Returns a struct Curl_hash pointer on success, NULL on failure. - */ -struct Curl_hash *Curl_global_host_cache_init(void); - -#ifdef USE_HTTPSRR - -#define CURL_MAXLEN_host_name 253 - -struct Curl_https_rrinfo { - size_t len; /* raw encoded length */ - unsigned char *val; /* raw encoded octets */ - /* - * fields from HTTPS RR, with the mandatory fields - * first (priority, target), then the others in the - * order of the keytag numbers defined at - * https://datatracker.ietf.org/doc/html/rfc9460#section-14.3.2 - */ - uint16_t priority; - char *target; - char *alpns; /* keytag = 1 */ - bool no_def_alpn; /* keytag = 2 */ - /* - * we do not support ports (keytag = 3) as we do not support - * port-switching yet - */ - unsigned char *ipv4hints; /* keytag = 4 */ - size_t ipv4hints_len; - unsigned char *echconfiglist; /* keytag = 5 */ - size_t echconfiglist_len; - unsigned char *ipv6hints; /* keytag = 6 */ - size_t ipv6hints_len; -}; -#endif - -struct Curl_dns_entry { - struct Curl_addrinfo *addr; -#ifdef USE_HTTPSRR - struct Curl_https_rrinfo *hinfo; -#endif - /* timestamp == 0 -- permanent CURLOPT_RESOLVE entry (does not time out) */ - time_t timestamp; - /* reference counter, entry is freed on reaching 0 */ - size_t refcount; - /* hostname port number that resolved to addr. */ - int hostport; - /* hostname that resolved to addr. may be NULL (Unix domain sockets). */ - char hostname[1]; -}; - -bool Curl_host_is_ipnum(const char *hostname); - -/* - * Curl_resolv() returns an entry with the info for the specified host - * and port. - * - * The returned data *MUST* be "released" with Curl_resolv_unlink() after - * use, or we will leak memory! - */ -/* return codes */ -enum resolve_t { - CURLRESOLV_TIMEDOUT = -2, - CURLRESOLV_ERROR = -1, - CURLRESOLV_RESOLVED = 0, - CURLRESOLV_PENDING = 1 -}; -enum resolve_t Curl_resolv(struct Curl_easy *data, - const char *hostname, - int port, - bool allowDOH, - struct Curl_dns_entry **dnsentry); -enum resolve_t Curl_resolv_timeout(struct Curl_easy *data, - const char *hostname, int port, - struct Curl_dns_entry **dnsentry, - timediff_t timeoutms); - -#ifdef USE_IPV6 -/* - * Curl_ipv6works() returns TRUE if IPv6 seems to work. - */ -bool Curl_ipv6works(struct Curl_easy *data); -#else -#define Curl_ipv6works(x) FALSE -#endif - -/* - * Curl_ipvalid() checks what CURL_IPRESOLVE_* requirements that might've - * been set and returns TRUE if they are OK. - */ -bool Curl_ipvalid(struct Curl_easy *data, struct connectdata *conn); - - -/* - * Curl_getaddrinfo() is the generic low-level name resolve API within this - * source file. There are several versions of this function - for different - * name resolve layers (selected at build-time). They all take this same set - * of arguments - */ -struct Curl_addrinfo *Curl_getaddrinfo(struct Curl_easy *data, - const char *hostname, - int port, - int *waitp); - - -/* unlink a dns entry, potentially shared with a cache */ -void Curl_resolv_unlink(struct Curl_easy *data, - struct Curl_dns_entry **pdns); - -/* init a new dns cache */ -void Curl_init_dnscache(struct Curl_hash *hash, size_t hashsize); - -/* prune old entries from the DNS cache */ -void Curl_hostcache_prune(struct Curl_easy *data); - -/* IPv4 threadsafe resolve function used for synch and asynch builds */ -struct Curl_addrinfo *Curl_ipv4_resolve_r(const char *hostname, int port); - -CURLcode Curl_once_resolved(struct Curl_easy *data, bool *protocol_connect); - -/* - * Curl_addrinfo_callback() is used when we build with any asynch specialty. - * Handles end of async request processing. Inserts ai into hostcache when - * status is CURL_ASYNC_SUCCESS. Twiddles fields in conn to indicate async - * request completed whether successful or failed. - */ -CURLcode Curl_addrinfo_callback(struct Curl_easy *data, - int status, - struct Curl_addrinfo *ai); - -/* - * Curl_printable_address() returns a printable version of the 1st address - * given in the 'ip' argument. The result will be stored in the buf that is - * bufsize bytes big. - */ -void Curl_printable_address(const struct Curl_addrinfo *ip, - char *buf, size_t bufsize); - -/* - * Curl_fetch_addr() fetches a 'Curl_dns_entry' already in the DNS cache. - * - * Returns the Curl_dns_entry entry pointer or NULL if not in the cache. - * - * The returned data *MUST* be "released" with Curl_resolv_unlink() after - * use, or we will leak memory! - */ -struct Curl_dns_entry * -Curl_fetch_addr(struct Curl_easy *data, - const char *hostname, - int port); - -/* - * Curl_cache_addr() stores a 'Curl_addrinfo' struct in the DNS cache. - * @param permanent iff TRUE, entry will never become stale - * Returns the Curl_dns_entry entry pointer or NULL if the storage failed. - */ -struct Curl_dns_entry * -Curl_cache_addr(struct Curl_easy *data, struct Curl_addrinfo *addr, - const char *hostname, size_t hostlen, int port, - bool permanent); - -#ifndef INADDR_NONE -#define CURL_INADDR_NONE (in_addr_t) ~0 -#else -#define CURL_INADDR_NONE INADDR_NONE -#endif - -/* - * Function provided by the resolver backend to set DNS servers to use. - */ -CURLcode Curl_set_dns_servers(struct Curl_easy *data, char *servers); - -/* - * Function provided by the resolver backend to set - * outgoing interface to use for DNS requests - */ -CURLcode Curl_set_dns_interface(struct Curl_easy *data, - const char *interf); - -/* - * Function provided by the resolver backend to set - * local IPv4 address to use as source address for DNS requests - */ -CURLcode Curl_set_dns_local_ip4(struct Curl_easy *data, - const char *local_ip4); - -/* - * Function provided by the resolver backend to set - * local IPv6 address to use as source address for DNS requests - */ -CURLcode Curl_set_dns_local_ip6(struct Curl_easy *data, - const char *local_ip6); - -/* - * Clean off entries from the cache - */ -void Curl_hostcache_clean(struct Curl_easy *data, struct Curl_hash *hash); - -/* - * Populate the cache with specified entries from CURLOPT_RESOLVE. - */ -CURLcode Curl_loadhostpairs(struct Curl_easy *data); -CURLcode Curl_resolv_check(struct Curl_easy *data, - struct Curl_dns_entry **dns); -int Curl_resolv_getsock(struct Curl_easy *data, - curl_socket_t *socks); - -CURLcode Curl_resolver_error(struct Curl_easy *data); -#endif /* HEADER_CURL_HOSTIP_H */ diff --git a/registry/native/c/curl-upstream-overlay/lib/vtls/wasi_tls.c b/registry/native/c/curl-upstream-overlay/lib/vtls/wasi_tls.c deleted file mode 100644 index a6a59e1ee7..0000000000 --- a/registry/native/c/curl-upstream-overlay/lib/vtls/wasi_tls.c +++ /dev/null @@ -1,187 +0,0 @@ -/* - * WASI TLS backend for libcurl - * - * Uses the host_net WASM import net_tls_connect() to upgrade a TCP - * socket to TLS. The host runtime (Node.js tls.connect()) handles all - * cryptographic operations transparently — after the upgrade, regular - * send()/recv() carry encrypted traffic. This backend therefore just - * calls the host import during connect and passes data through to the - * socket layer for send/recv. - */ - -#include "curl_setup.h" - -#ifdef USE_WASI_TLS - -#include - -#include "urldata.h" -#include "sendf.h" -#include "vtls.h" -#include "vtls_int.h" -#include "connect.h" -#include "select.h" -#include "curl_printf.h" - -/* The last #include files should be: */ -#include "curl_memory.h" -#include "memdebug.h" - -/* WASM import: upgrade a TCP socket FD to TLS. - * flags: 0 = verify peer certificate (default) - * 1 = skip peer certificate verification (-k / --insecure) - * Returns 0 on success, errno on failure. */ -__attribute__((import_module("host_net"), import_name("net_tls_connect"))) -extern int __wasi_net_tls_connect(int fd, const char *hostname_ptr, - int hostname_len, int flags); - -/* Per-connection backend data (minimal — host handles all TLS state) */ -struct wasi_tls_backend_data { - int dummy; /* struct must be non-empty */ -}; - -static size_t wasi_tls_version(char *buffer, size_t size) -{ - return msnprintf(buffer, size, "WASI-TLS/host"); -} - -static CURLcode wasi_tls_connect_blocking(struct Curl_cfilter *cf, - struct Curl_easy *data) -{ - struct ssl_connect_data *connssl = cf->ctx; - curl_socket_t sockfd = Curl_conn_cf_get_socket(cf, data); - const char *hostname = connssl->peer.hostname; - int flags = 0; - - /* Check if peer verification is disabled (curl -k / --insecure) */ - struct ssl_primary_config *conn_config = Curl_ssl_cf_get_primary_config(cf); - if(conn_config && !conn_config->verifypeer) { - flags = 1; - } - - int ret = __wasi_net_tls_connect((int)sockfd, hostname, - (int)strlen(hostname), flags); - if(ret != 0) { - failf(data, "WASI TLS: host TLS connect failed (errno %d)", ret); - return CURLE_SSL_CONNECT_ERROR; - } - - connssl->state = ssl_connection_complete; - return CURLE_OK; -} - -static CURLcode wasi_tls_connect_nonblocking(struct Curl_cfilter *cf, - struct Curl_easy *data, - bool *done) -{ - /* Our host TLS connect is synchronous — just call blocking version */ - CURLcode result = wasi_tls_connect_blocking(cf, data); - *done = (result == CURLE_OK); - return result; -} - -/* recv: pass through to the socket layer — host handles decryption */ -static ssize_t wasi_tls_recv(struct Curl_cfilter *cf, - struct Curl_easy *data, - char *buf, size_t len, CURLcode *err) -{ - return Curl_conn_cf_recv(cf->next, data, buf, len, err); -} - -/* send: pass through to the socket layer — host handles encryption */ -static ssize_t wasi_tls_send(struct Curl_cfilter *cf, - struct Curl_easy *data, - const void *buf, size_t len, CURLcode *err) -{ - return Curl_conn_cf_send(cf->next, data, buf, len, FALSE, err); -} - -static CURLcode wasi_tls_shutdown(struct Curl_cfilter *cf, - struct Curl_easy *data, - bool send_shutdown, bool *done) -{ - (void)cf; - (void)data; - (void)send_shutdown; - *done = TRUE; - return CURLE_OK; -} - -static bool wasi_tls_data_pending(struct Curl_cfilter *cf, - const struct Curl_easy *data) -{ - (void)cf; - (void)data; - return FALSE; -} - -/* data may be NULL */ -static CURLcode wasi_tls_random(struct Curl_easy *data, - unsigned char *entropy, size_t length) -{ - size_t offset = 0; - (void)data; - - while(offset < length) { - size_t chunk = length - offset; - if(chunk > 256) - chunk = 256; - - if(getentropy(entropy + offset, chunk) != 0) - return CURLE_FAILED_INIT; - - offset += chunk; - } - - return CURLE_OK; -} - -static void wasi_tls_close(struct Curl_cfilter *cf, struct Curl_easy *data) -{ - (void)cf; - (void)data; - /* Host-side TLS is cleaned up when the socket is closed */ -} - -static void *wasi_tls_get_internals(struct ssl_connect_data *connssl, - CURLINFO info) -{ - (void)connssl; - (void)info; - return NULL; -} - -const struct Curl_ssl Curl_ssl_wasi_tls = { - { CURLSSLBACKEND_NONE, "wasi-tls" }, /* info */ - - 0, /* supports — no special features */ - - sizeof(struct wasi_tls_backend_data), - - Curl_none_init, /* init */ - Curl_none_cleanup, /* cleanup */ - wasi_tls_version, /* version */ - Curl_none_check_cxn, /* check_cxn */ - wasi_tls_shutdown, /* shutdown */ - wasi_tls_data_pending, /* data_pending */ - wasi_tls_random, /* random */ - Curl_none_cert_status_request, /* cert_status_request */ - wasi_tls_connect_blocking, /* connect */ - wasi_tls_connect_nonblocking, /* connect_nonblocking */ - Curl_ssl_adjust_pollset, /* adjust_pollset */ - wasi_tls_get_internals, /* get_internals */ - wasi_tls_close, /* close */ - Curl_none_close_all, /* close_all */ - Curl_none_set_engine, /* set_engine */ - Curl_none_set_engine_default, /* set_engine_default */ - Curl_none_engines_list, /* engines_list */ - Curl_none_false_start, /* false_start */ - NULL, /* sha256sum */ - NULL, /* associate_connection */ - NULL, /* disassociate_connection */ - wasi_tls_recv, /* recv decrypted data */ - wasi_tls_send, /* send data to encrypt */ - NULL, /* get_channel_binding */ -}; - -#endif /* USE_WASI_TLS */ diff --git a/registry/native/c/curl-upstream-overlay/lib/vtls/wasi_tls.h b/registry/native/c/curl-upstream-overlay/lib/vtls/wasi_tls.h deleted file mode 100644 index d75fdf1d93..0000000000 --- a/registry/native/c/curl-upstream-overlay/lib/vtls/wasi_tls.h +++ /dev/null @@ -1,18 +0,0 @@ -#ifndef HEADER_CURL_WASI_TLS_H -#define HEADER_CURL_WASI_TLS_H -/* - * WASI TLS backend for libcurl - * - * Delegates TLS to the host runtime via the host_net WASM import - * net_tls_connect(). After the host upgrades the socket, regular - * send()/recv() transparently carry encrypted traffic, so the - * backend's recv_plain/send_plain are simple pass-throughs. - */ - -#include "curl_setup.h" - -#ifdef USE_WASI_TLS -extern const struct Curl_ssl Curl_ssl_wasi_tls; -#endif - -#endif /* HEADER_CURL_WASI_TLS_H */ diff --git a/registry/native/c/programs/http_get.c b/registry/native/c/programs/http_get.c deleted file mode 100644 index 0e7dc466ea..0000000000 --- a/registry/native/c/programs/http_get.c +++ /dev/null @@ -1,95 +0,0 @@ -/* http_get.c — connect to an HTTP server, send GET request, print response body */ -#include -#include -#include -#include -#include -#include -#include - -int main(int argc, char *argv[]) { - if (argc < 2) { - fprintf(stderr, "usage: http_get [path] [output_file]\n"); - return 1; - } - - int port = atoi(argv[1]); - const char *path = argc >= 3 ? argv[2] : "/"; - const char *output_file = argc >= 4 ? argv[3] : NULL; - - int fd = socket(AF_INET, SOCK_STREAM, 0); - if (fd < 0) { - perror("socket"); - return 1; - } - - struct sockaddr_in addr; - memset(&addr, 0, sizeof(addr)); - addr.sin_family = AF_INET; - addr.sin_port = htons((uint16_t)port); - inet_pton(AF_INET, "127.0.0.1", &addr.sin_addr); - - if (connect(fd, (struct sockaddr *)&addr, sizeof(addr)) < 0) { - perror("connect"); - close(fd); - return 1; - } - - char request[1024]; - int request_len = snprintf( - request, - sizeof(request), - "GET %s HTTP/1.0\r\nHost: localhost\r\n\r\n", - path - ); - if (request_len < 0 || (size_t)request_len >= sizeof(request)) { - fprintf(stderr, "request path too long\n"); - close(fd); - return 1; - } - - ssize_t sent = send(fd, request, (size_t)request_len, 0); - if (sent < 0) { - perror("send"); - close(fd); - return 1; - } - - /* Read full response */ - char response[4096]; - size_t total = 0; - ssize_t n; - while ((n = recv(fd, response + total, sizeof(response) - total - 1, 0)) > 0) { - total += (size_t)n; - } - response[total] = '\0'; - - close(fd); - - /* Find body after \r\n\r\n */ - const char *body = strstr(response, "\r\n\r\n"); - if (body) { - body += 4; - if (output_file) { - FILE *out = fopen(output_file, "wb"); - if (!out) { - perror("fopen"); - return 1; - } - size_t body_len = total - (size_t)(body - response); - if (fwrite(body, 1, body_len, out) != body_len) { - perror("fwrite"); - fclose(out); - return 1; - } - fclose(out); - } else { - printf("body: %s\n", body); - } - } else { - printf("body: (no separator found)\n"); - return 1; - } - - return 0; -} diff --git a/registry/native/c/programs/spawn_child.c b/registry/native/c/programs/spawn_child.c deleted file mode 100644 index b174aad1b5..0000000000 --- a/registry/native/c/programs/spawn_child.c +++ /dev/null @@ -1,53 +0,0 @@ -/* spawn_child.c -- posix_spawn 'echo hello', waitpid, print child stdout */ -#include -#include -#include -#include - -#include "posix_spawn_compat.h" - -extern char **environ; - -int main(void) { - int pipefd[2]; - if (pipe(pipefd) != 0) { - perror("pipe"); - return 1; - } - - /* Redirect child stdout to pipe write end */ - posix_spawn_file_actions_t fa; - posix_spawn_file_actions_init(&fa); - posix_spawn_file_actions_adddup2(&fa, pipefd[1], STDOUT_FILENO); - posix_spawn_file_actions_addclose(&fa, pipefd[0]); - posix_spawn_file_actions_addclose(&fa, pipefd[1]); - - char *argv[] = {"echo", "hello", NULL}; - pid_t child; - int err = posix_spawnp(&child, "echo", &fa, NULL, argv, environ); - posix_spawn_file_actions_destroy(&fa); - - if (err != 0) { - fprintf(stderr, "posix_spawn failed: %d\n", err); - return 1; - } - - close(pipefd[1]); - - char buf[256]; - ssize_t n = read(pipefd[0], buf, sizeof(buf) - 1); - close(pipefd[0]); - - int status; - waitpid(child, &status, 0); - - if (n > 0) { - buf[n] = '\0'; - printf("child_stdout: %s", buf); - } else { - printf("child_stdout: (empty)\n"); - } - printf("child_exit: %d\n", WIFEXITED(status) ? WEXITSTATUS(status) : -1); - - return 0; -} diff --git a/registry/native/c/programs/sqlite3_cli.c b/registry/native/c/programs/sqlite3_cli.c deleted file mode 100644 index ba878b956f..0000000000 --- a/registry/native/c/programs/sqlite3_cli.c +++ /dev/null @@ -1,558 +0,0 @@ -/* sqlite3_cli — SQLite CLI for WasmVM - * - * Full sqlite3 CLI supporting: - * sqlite3 :memory: — interactive/piped in-memory DB - * sqlite3 /path/to/db.sqlite — file-based DB via WASI VFS - * echo 'SELECT 1;' | sqlite3 — stdin pipe mode (defaults to :memory:) - * sqlite3 :memory: "SELECT 1;" — SQL from command line argument - * - * Meta-commands: .dump, .schema, .tables, .quit, .help, .headers, .mode - * - * Compiled with -DSQLITE_OS_OTHER — uses a custom VFS wrapping WASI file I/O. - */ -#include -#include -#include -#include -#include -#include -#include -#include "sqlite3.h" - -/* ── WASI VFS ────────────────────────────────────────────────────────── */ - -/* Minimal VFS implementation using POSIX file I/O (supported by WASI) */ - -typedef struct WasiFile { - sqlite3_file base; - int fd; -} WasiFile; - -static int wasiClose(sqlite3_file *pFile) { - WasiFile *p = (WasiFile *)pFile; - if (p->fd >= 0) close(p->fd); - p->fd = -1; - return SQLITE_OK; -} - -static int wasiRead(sqlite3_file *pFile, void *zBuf, int iAmt, sqlite3_int64 iOfst) { - WasiFile *p = (WasiFile *)pFile; - if (lseek(p->fd, (off_t)iOfst, SEEK_SET) != (off_t)iOfst) - return SQLITE_IOERR_READ; - ssize_t got = read(p->fd, zBuf, iAmt); - if (got == iAmt) return SQLITE_OK; - if (got >= 0) { - memset((char *)zBuf + got, 0, iAmt - got); - return SQLITE_IOERR_SHORT_READ; - } - return SQLITE_IOERR_READ; -} - -static int wasiWrite(sqlite3_file *pFile, const void *zBuf, int iAmt, sqlite3_int64 iOfst) { - WasiFile *p = (WasiFile *)pFile; - if (lseek(p->fd, (off_t)iOfst, SEEK_SET) != (off_t)iOfst) - return SQLITE_IOERR_WRITE; - ssize_t wrote = write(p->fd, zBuf, iAmt); - if (wrote == iAmt) return SQLITE_OK; - return SQLITE_IOERR_WRITE; -} - -static int wasiTruncate(sqlite3_file *pFile, sqlite3_int64 size) { - WasiFile *p = (WasiFile *)pFile; - if (ftruncate(p->fd, (off_t)size) != 0) return SQLITE_IOERR_TRUNCATE; - return SQLITE_OK; -} - -static int wasiSync(sqlite3_file *pFile, int flags) { - (void)flags; - WasiFile *p = (WasiFile *)pFile; - if (fsync(p->fd) != 0) return SQLITE_IOERR_FSYNC; - return SQLITE_OK; -} - -static int wasiFileSize(sqlite3_file *pFile, sqlite3_int64 *pSize) { - WasiFile *p = (WasiFile *)pFile; - struct stat st; - if (fstat(p->fd, &st) != 0) return SQLITE_IOERR_FSTAT; - *pSize = (sqlite3_int64)st.st_size; - return SQLITE_OK; -} - -/* No locking in WASM — single process */ -static int wasiLock(sqlite3_file *f, int l) { (void)f; (void)l; return SQLITE_OK; } -static int wasiUnlock(sqlite3_file *f, int l) { (void)f; (void)l; return SQLITE_OK; } -static int wasiCheckReservedLock(sqlite3_file *f, int *pResOut) { - (void)f; *pResOut = 0; return SQLITE_OK; -} -static int wasiFileControl(sqlite3_file *f, int op, void *pArg) { - (void)f; (void)op; (void)pArg; return SQLITE_NOTFOUND; -} -static int wasiSectorSize(sqlite3_file *f) { (void)f; return 512; } -static int wasiDeviceCharacteristics(sqlite3_file *f) { (void)f; return 0; } - -static const sqlite3_io_methods wasiIoMethods __attribute__((used)) = { - 1, /* iVersion */ - wasiClose, - wasiRead, - wasiWrite, - wasiTruncate, - wasiSync, - wasiFileSize, - wasiLock, - wasiUnlock, - wasiCheckReservedLock, - wasiFileControl, - wasiSectorSize, - wasiDeviceCharacteristics, - /* v2+ methods */ - 0, 0, 0, 0 -}; - -static int wasiOpen(sqlite3_vfs *pVfs, sqlite3_filename zName, sqlite3_file *pFile, - int flags, int *pOutFlags) { - (void)pVfs; - WasiFile *p = (WasiFile *)pFile; - p->fd = -1; - - if (!zName) { - /* Temp file — use in-memory only */ - return SQLITE_IOERR; - } - - int oflags = 0; - if (flags & SQLITE_OPEN_CREATE) oflags |= O_CREAT; - if (flags & SQLITE_OPEN_READWRITE) oflags |= O_RDWR; - else oflags |= O_RDONLY; - - p->fd = open(zName, oflags, 0644); - if (p->fd < 0) return SQLITE_CANTOPEN; - - if (pOutFlags) *pOutFlags = flags; - p->base.pMethods = &wasiIoMethods; - return SQLITE_OK; -} - -static int wasiDelete(sqlite3_vfs *pVfs, const char *zName, int syncDir) { - (void)pVfs; (void)syncDir; - if (unlink(zName) != 0 && errno != ENOENT) return SQLITE_IOERR_DELETE; - return SQLITE_OK; -} - -static int wasiAccess(sqlite3_vfs *pVfs, const char *zName, int flags, int *pResOut) { - (void)pVfs; - struct stat st; - *pResOut = (stat(zName, &st) == 0) ? 1 : 0; - (void)flags; - return SQLITE_OK; -} - -static int wasiFullPathname(sqlite3_vfs *pVfs, const char *zName, int nOut, char *zOut) { - (void)pVfs; - if (zName[0] == '/') { - snprintf(zOut, nOut, "%s", zName); - } else { - /* Relative path — prefix with / since VFS root is / */ - snprintf(zOut, nOut, "/%s", zName); - } - return SQLITE_OK; -} - -static int wasiRandomness(sqlite3_vfs *pVfs, int nByte, char *zOut) { - (void)pVfs; - int fd = open("/dev/urandom", O_RDONLY); - if (fd >= 0) { - int total = 0; - while (total < nByte) { - ssize_t read_len = read(fd, zOut + total, (size_t)(nByte - total)); - if (read_len <= 0) { - break; - } - total += (int)read_len; - } - close(fd); - if (total == nByte) { - return nByte; - } - nByte = total; - } - - /* Fallback only if urandom is unexpectedly unavailable in the runtime. */ - for (int i = 0; i < nByte; i++) zOut[i] = (char)(i * 37 + 17); - return nByte; -} - -static int wasiSleep(sqlite3_vfs *pVfs, int microseconds) { - (void)pVfs; (void)microseconds; - return 0; -} - -static int wasiCurrentTime(sqlite3_vfs *pVfs, double *pTime) { - (void)pVfs; - /* Julian day for Unix epoch 0 */ - *pTime = 2440587.5; - return SQLITE_OK; -} - -static int wasiGetLastError(sqlite3_vfs *pVfs, int nBuf, char *zBuf) { - (void)pVfs; - if (nBuf > 0) zBuf[0] = 0; - return 0; -} - -static sqlite3_vfs wasiVfs __attribute__((used)) = { - 1, /* iVersion */ - sizeof(WasiFile), /* szOsFile */ - 512, /* mxPathname */ - 0, /* pNext */ - "wasi", /* zName */ - 0, /* pAppData */ - wasiOpen, - wasiDelete, - wasiAccess, - wasiFullPathname, - 0, 0, 0, 0, /* xDlOpen, xDlError, xDlSym, xDlClose */ - wasiRandomness, - wasiSleep, - wasiCurrentTime, - wasiGetLastError, - /* v2+ */ - 0, 0, 0, 0 -}; - -#ifdef SQLITE_OS_OTHER -int sqlite3_os_init(void) { - return sqlite3_vfs_register(&wasiVfs, 1); -} - -int sqlite3_os_end(void) { - return SQLITE_OK; -} -#endif /* SQLITE_OS_OTHER */ - -/* ── CLI ─────────────────────────────────────────────────────────────── */ - -static int headers_enabled = 0; -static int headers_printed = 0; -static char output_sep_buf[8]; -static char *output_separator = output_sep_buf; - -/* Callback for sqlite3_exec — prints rows in column format */ -static int print_row(void *unused, int ncols, char **values, char **names) { - (void)unused; - if (headers_enabled && !headers_printed) { - for (int i = 0; i < ncols; i++) { - if (i > 0) printf("%s", output_separator); - printf("%s", names[i]); - } - printf("\n"); - headers_printed = 1; - } - for (int i = 0; i < ncols; i++) { - if (i > 0) printf("%s", output_separator); - printf("%s", values[i] ? values[i] : ""); - } - printf("\n"); - return 0; -} - -/* Execute SQL and report errors */ -static int exec_sql(sqlite3 *db, const char *sql) { - char *err = NULL; - headers_printed = 0; /* Reset per query */ - int rc = sqlite3_exec(db, sql, print_row, NULL, &err); - if (rc != SQLITE_OK) { - fprintf(stderr, "Error: %s\n", err ? err : sqlite3_errmsg(db)); - sqlite3_free(err); - return 1; - } - return 0; -} - -/* Handle dot-commands */ -static int handle_meta(sqlite3 *db, const char *line) { - while (*line == ' ' || *line == '\t') line++; - - if (strcmp(line, ".quit") == 0 || strcmp(line, ".exit") == 0) { - return -1; /* Signal exit */ - } - if (strcmp(line, ".help") == 0) { - printf(".dump Dump database as SQL\n"); - printf(".headers on|off Toggle column headers\n"); - printf(".help Show this help\n"); - printf(".mode MODE Set output mode (list, csv)\n"); - printf(".quit Exit\n"); - printf(".schema Show CREATE statements\n"); - printf(".tables List tables\n"); - return 0; - } - if (strcmp(line, ".tables") == 0) { - return exec_sql(db, - "SELECT name FROM sqlite_master WHERE type='table' ORDER BY 1;"); - } - if (strcmp(line, ".schema") == 0) { - return exec_sql(db, - "SELECT sql||';' FROM sqlite_master WHERE sql IS NOT NULL ORDER BY 1;"); - } - if (strcmp(line, ".dump") == 0) { - printf("BEGIN TRANSACTION;\n"); - /* Schema */ - exec_sql(db, - "SELECT sql||';' FROM sqlite_master WHERE sql IS NOT NULL " - "ORDER BY CASE WHEN type='table' THEN 0 ELSE 1 END, name;"); - /* Data — generate INSERT statements for each table */ - sqlite3_stmt *stmt; - int rc = sqlite3_prepare_v2(db, - "SELECT name FROM sqlite_master WHERE type='table' ORDER BY 1;", - -1, &stmt, NULL); - if (rc == SQLITE_OK) { - while (sqlite3_step(stmt) == SQLITE_ROW) { - const char *table = (const char *)sqlite3_column_text(stmt, 0); - char query[1024]; - snprintf(query, sizeof(query), "SELECT * FROM \"%s\"", table); - - sqlite3_stmt *data_stmt; - rc = sqlite3_prepare_v2(db, query, -1, &data_stmt, NULL); - if (rc == SQLITE_OK) { - int ncols = sqlite3_column_count(data_stmt); - while (sqlite3_step(data_stmt) == SQLITE_ROW) { - printf("INSERT INTO \"%s\" VALUES(", table); - for (int i = 0; i < ncols; i++) { - if (i > 0) printf(","); - int type = sqlite3_column_type(data_stmt, i); - if (type == SQLITE_NULL) { - printf("NULL"); - } else if (type == SQLITE_INTEGER) { - printf("%lld", sqlite3_column_int64(data_stmt, i)); - } else if (type == SQLITE_FLOAT) { - printf("%g", sqlite3_column_double(data_stmt, i)); - } else { - const char *text = (const char *)sqlite3_column_text(data_stmt, i); - printf("'"); - for (const char *c = text; *c; c++) { - if (*c == '\'') printf("''"); - else putchar(*c); - } - printf("'"); - } - } - printf(");\n"); - } - sqlite3_finalize(data_stmt); - } - } - sqlite3_finalize(stmt); - } - printf("COMMIT;\n"); - return 0; - } - if (strncmp(line, ".headers ", 9) == 0) { - const char *arg = line + 9; - while (*arg == ' ') arg++; - if (strcmp(arg, "on") == 0) headers_enabled = 1; - else headers_enabled = 0; - return 0; - } - if (strncmp(line, ".mode ", 6) == 0) { - const char *arg = line + 6; - while (*arg == ' ') arg++; - if (strcmp(arg, "csv") == 0) strcpy(output_sep_buf, ","); - else strcpy(output_sep_buf, "|"); - return 0; - } - - fprintf(stderr, "Error: unknown command or invalid arguments: \"%s\"\n", line); - return 0; -} - -/* Read all data from stream into a buffer */ -static char *read_all(FILE *in, size_t *out_len) { - size_t cap = 65536; - size_t len = 0; - char *buf = (char *)malloc(cap); - if (!buf) return NULL; - - int c; - while ((c = fgetc(in)) != EOF) { - if (len + 1 >= cap) { - cap *= 2; - char *nb = (char *)realloc(buf, cap); - if (!nb) { free(buf); return NULL; } - buf = nb; - } - buf[len++] = (char)c; - } - buf[len] = '\0'; - if (out_len) *out_len = len; - return buf; -} - -/* Process a line of input — either meta-command or SQL accumulation */ -static int process_line(sqlite3 *db, const char *line, size_t len, - char *sql_buf, int *sql_len, int sql_cap) { - /* Meta-commands start with . */ - if (line[0] == '.' && *sql_len == 0) { - /* Need a mutable copy for handle_meta */ - char meta_buf[1024]; - size_t ml = len < sizeof(meta_buf) - 1 ? len : sizeof(meta_buf) - 1; - memcpy(meta_buf, line, ml); - meta_buf[ml] = '\0'; - int rc = handle_meta(db, meta_buf); - if (rc < 0) return -1; /* .quit */ - return rc; - } - - /* Accumulate SQL */ - if (*sql_len + (int)len + 2 < sql_cap) { - if (*sql_len > 0) sql_buf[(*sql_len)++] = '\n'; - memcpy(sql_buf + *sql_len, line, len); - *sql_len += (int)len; - sql_buf[*sql_len] = '\0'; - } - - /* Check if statement is complete */ - if (sqlite3_complete(sql_buf)) { - int rc = exec_sql(db, sql_buf); - *sql_len = 0; - return rc; - } - return 0; -} - -/* Read and execute SQL from stdin */ -static int process_input(sqlite3 *db, FILE *in) { - size_t total = 0; - char *data = read_all(in, &total); - if (!data || total == 0) { - free(data); - return 0; - } - - char sql_buf[65536]; - int sql_len = 0; - int had_error = 0; - - /* Process input line by line */ - const char *p = data; - const char *end = data + total; - while (p < end) { - /* Find end of line */ - const char *eol = p; - while (eol < end && *eol != '\n' && *eol != '\r') eol++; - size_t len = eol - p; - - /* Skip empty lines */ - if (len > 0) { - int rc = process_line(db, p, len, sql_buf, &sql_len, (int)sizeof(sql_buf)); - if (rc < 0) break; /* .quit */ - if (rc > 0) had_error = 1; - } - - /* Skip past newline */ - p = eol; - if (p < end && *p == '\r') p++; - if (p < end && *p == '\n') p++; - } - - /* Execute any remaining SQL */ - if (sql_len > 0) { - if (exec_sql(db, sql_buf) != 0) - had_error = 1; - } - - free(data); - return had_error; -} - -static void print_usage(void) { - fprintf(stderr, "Usage: sqlite3 [OPTIONS] [FILENAME] [SQL]\n"); - fprintf(stderr, " FILENAME is the name of an SQLite database. A new database is\n"); - fprintf(stderr, " created if the file does not exist. Use \":memory:\" for in-memory.\n"); - fprintf(stderr, " SQL is an optional SQL statement to execute.\n"); - fprintf(stderr, "Options:\n"); - fprintf(stderr, " -header Turn headers on\n"); - fprintf(stderr, " -csv Set output mode to CSV\n"); - fprintf(stderr, " -separator S Set output separator (default \"|\")\n"); - fprintf(stderr, " -help Show this help\n"); - fprintf(stderr, " -version Show SQLite version\n"); -} - -int main(int argc, char **argv) { - const char *db_path = ":memory:"; - const char *sql_arg = NULL; - int positional = 0; - int i; - - /* Initialize separator at runtime — static initializers for arrays - * may not work correctly in optimized WASM builds. */ - output_sep_buf[0] = '|'; - output_sep_buf[1] = '\0'; - - /* Parse options */ - for (i = 1; i < argc; i++) { - if (argv[i][0] == '-') { - if (strcmp(argv[i], "-help") == 0 || strcmp(argv[i], "--help") == 0) { - print_usage(); - return 0; - } - if (strcmp(argv[i], "-version") == 0 || strcmp(argv[i], "--version") == 0) { - printf("%s\n", sqlite3_libversion()); - return 0; - } - if (strcmp(argv[i], "-header") == 0 || strcmp(argv[i], "-headers") == 0) { - headers_enabled = 1; - continue; - } - if (strcmp(argv[i], "-csv") == 0) { - strcpy(output_sep_buf, ","); - continue; - } - if (strcmp(argv[i], "-separator") == 0 && i + 1 < argc) { - strncpy(output_sep_buf, argv[++i], sizeof(output_sep_buf) - 1); - output_sep_buf[sizeof(output_sep_buf) - 1] = '\0'; - continue; - } - fprintf(stderr, "Error: unknown option: %s\n", argv[i]); - print_usage(); - return 1; - } - /* Positional args: [FILENAME] [SQL] */ - if (positional == 0) { - db_path = argv[i]; - positional++; - } else if (positional == 1) { - sql_arg = argv[i]; - positional++; - } else { - fprintf(stderr, "Error: too many arguments\n"); - print_usage(); - return 1; - } - } - - /* Open database */ - sqlite3 *db; - int rc = sqlite3_open(db_path, &db); - if (rc != SQLITE_OK) { - fprintf(stderr, "Error: unable to open database \"%s\": %s\n", - db_path, sqlite3_errmsg(db)); - sqlite3_close(db); - return 1; - } - - int exit_code = 0; - - if (sql_arg) { - /* Execute SQL from command line */ - exit_code = exec_sql(db, sql_arg); - } else { - /* Read from stdin */ - exit_code = process_input(db, stdin); - } - - /* Flush output */ - fflush(stdout); - fflush(stderr); - /* Skip sqlite3_close + atexit handlers — use _Exit to avoid WASM indirect - * call table traps during wasi-libc/SQLite cleanup with wasm-opt. */ - _Exit(exit_code); -} diff --git a/registry/native/c/programs/unzip.c b/registry/native/c/programs/unzip.c deleted file mode 100644 index 383e714205..0000000000 --- a/registry/native/c/programs/unzip.c +++ /dev/null @@ -1,669 +0,0 @@ -/* unzip.c — Extract ZIP archives using zlib/minizip - * - * Usage: unzip archive.zip (extract all to cwd) - * unzip -d outdir archive.zip (extract to directory) - * unzip -l archive.zip (list contents) - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include "ioapi.h" -#include "unzip.h" - -#define MAX_PATH_LEN 4096 -#define WRITE_BUF_SIZE 8192 - -/* Cap per-entry allocation in the fallback parser. Hostile central directory - * records can claim sizes up to 4 GiB; refuse anything above this bound. */ -#define MAX_UNCOMPRESSED_SIZE (256u * 1024u * 1024u) - -typedef struct { - FILE *file; - char *filename; - char mode[4]; - long position; - long size; -} unzip_file_stream; - -static voidpf ZCALLBACK unzip_open_file(voidpf opaque, const char *filename, int mode) { - unzip_file_stream *stream = NULL; - const char *mode_fopen = NULL; - (void)opaque; - - if ((mode & ZLIB_FILEFUNC_MODE_READWRITEFILTER) == ZLIB_FILEFUNC_MODE_READ) - mode_fopen = "rb"; - else if (mode & ZLIB_FILEFUNC_MODE_EXISTING) - mode_fopen = "r+b"; - else if (mode & ZLIB_FILEFUNC_MODE_CREATE) - mode_fopen = "wb"; - - if (filename == NULL || mode_fopen == NULL) - return NULL; - - stream = (unzip_file_stream *)calloc(1, sizeof(unzip_file_stream)); - if (!stream) - return NULL; - - stream->filename = (char *)malloc(strlen(filename) + 1); - if (!stream->filename) { - free(stream); - return NULL; - } - strcpy(stream->filename, filename); - strncpy(stream->mode, mode_fopen, sizeof(stream->mode) - 1); - stream->mode[sizeof(stream->mode) - 1] = '\0'; - - stream->file = fopen(filename, mode_fopen); - if (!stream->file) { - free(stream->filename); - free(stream); - return NULL; - } - struct stat st; - stream->size = stat(filename, &st) == 0 ? (long)st.st_size : 0; - stream->position = 0; - return stream; -} - -static uLong ZCALLBACK unzip_read_file(voidpf opaque, voidpf stream, void *buf, uLong size) { - unzip_file_stream *file_stream = (unzip_file_stream *)stream; - uLong got; - (void)opaque; - got = (uLong)fread(buf, 1, (size_t)size, file_stream->file); - file_stream->position += (long)got; - return got; -} - -static uLong ZCALLBACK unzip_write_file(voidpf opaque, voidpf stream, const void *buf, uLong size) { - unzip_file_stream *file_stream = (unzip_file_stream *)stream; - uLong wrote; - (void)opaque; - wrote = (uLong)fwrite(buf, 1, (size_t)size, file_stream->file); - file_stream->position += (long)wrote; - if (file_stream->position > file_stream->size) - file_stream->size = file_stream->position; - return wrote; -} - -static long ZCALLBACK unzip_tell_file(voidpf opaque, voidpf stream) { - unzip_file_stream *file_stream = (unzip_file_stream *)stream; - (void)opaque; - return file_stream->position; -} - -static long ZCALLBACK unzip_seek_file(voidpf opaque, voidpf stream, uLong offset, int origin) { - int fseek_origin = 0; - long seek_offset = (long)offset; - unzip_file_stream *file_stream = (unzip_file_stream *)stream; - (void)opaque; - - switch (origin) { - case ZLIB_FILEFUNC_SEEK_CUR: - seek_offset = file_stream->position + (long)offset; - fseek_origin = SEEK_SET; - break; - case ZLIB_FILEFUNC_SEEK_END: - seek_offset = file_stream->size + (long)offset; - fseek_origin = SEEK_SET; - break; - case ZLIB_FILEFUNC_SEEK_SET: - fseek_origin = SEEK_SET; - break; - default: - return -1; - } - - fclose(file_stream->file); - file_stream->file = fopen(file_stream->filename, file_stream->mode); - if (!file_stream->file) - return -1; - - if (fseek(file_stream->file, seek_offset, fseek_origin) != 0) - return -1; - clearerr(file_stream->file); - file_stream->position = seek_offset; - return 0; -} - -static int ZCALLBACK unzip_close_file(voidpf opaque, voidpf stream) { - unzip_file_stream *file_stream = (unzip_file_stream *)stream; - int ret; - (void)opaque; - ret = fclose(file_stream->file); - free(file_stream->filename); - free(file_stream); - return ret; -} - -static int ZCALLBACK unzip_error_file(voidpf opaque, voidpf stream) { - unzip_file_stream *file_stream = (unzip_file_stream *)stream; - (void)opaque; - return ferror(file_stream->file); -} - -static unzFile open_archive(const char *archive) { - zlib_filefunc_def filefunc = { - .zopen_file = unzip_open_file, - .zread_file = unzip_read_file, - .zwrite_file = unzip_write_file, - .ztell_file = unzip_tell_file, - .zseek_file = unzip_seek_file, - .zclose_file = unzip_close_file, - .zerror_file = unzip_error_file, - .opaque = NULL, - }; - return unzOpen2(archive, &filefunc); -} - -/* Ensure all parent directories of path exist */ -static int mkdirs(const char *path) { - char tmp[MAX_PATH_LEN]; - size_t len = strlen(path); - if (len >= sizeof(tmp)) return -1; - memcpy(tmp, path, len + 1); - - for (size_t i = 1; i < len; i++) { - if (tmp[i] == '/') { - tmp[i] = '\0'; - if (mkdir(tmp, 0755) != 0 && errno != EEXIST) - return -1; - tmp[i] = '/'; - } - } - return 0; -} - -static uint16_t read_le16(const unsigned char *p) { - return (uint16_t)p[0] | ((uint16_t)p[1] << 8); -} - -static uint32_t read_le32(const unsigned char *p) { - return (uint32_t)p[0] | ((uint32_t)p[1] << 8) | - ((uint32_t)p[2] << 16) | ((uint32_t)p[3] << 24); -} - -static int read_archive_bytes(const char *archive, unsigned char **out, size_t *out_len) { - FILE *f = fopen(archive, "rb"); - long size; - unsigned char *data; - if (!f) - return -1; - if (fseek(f, 0, SEEK_END) != 0) { - fclose(f); - return -1; - } - size = ftell(f); - if (size < 0 || fseek(f, 0, SEEK_SET) != 0) { - fclose(f); - return -1; - } - data = (unsigned char *)malloc((size_t)size); - if (!data) { - fclose(f); - return -1; - } - if (fread(data, 1, (size_t)size, f) != (size_t)size) { - free(data); - fclose(f); - return -1; - } - fclose(f); - *out = data; - *out_len = (size_t)size; - return 0; -} - -static int find_eocd(const unsigned char *data, size_t len, size_t *eocd_offset) { - size_t min = len > 0xffff + 22 ? len - (0xffff + 22) : 0; - if (len < 22) - return -1; - for (size_t pos = len - 22; pos + 4 <= len && pos >= min; pos--) { - if (read_le32(data + pos) == 0x06054b50) { - *eocd_offset = pos; - return 0; - } - if (pos == 0) - break; - } - return -1; -} - -static const char *entry_output_name(const char *name, size_t name_len) { - const char *end = name + name_len; - while (name < end && *name == '/') - name++; - return name; -} - -static int inflate_raw_entry(const unsigned char *src, size_t src_len, unsigned char *dst, size_t dst_len) { - z_stream stream; - memset(&stream, 0, sizeof(stream)); - stream.next_in = (Bytef *)src; - stream.avail_in = (uInt)src_len; - stream.next_out = dst; - stream.avail_out = (uInt)dst_len; - if (inflateInit2(&stream, -MAX_WBITS) != Z_OK) - return -1; - int result = inflate(&stream, Z_FINISH); - inflateEnd(&stream); - return result == Z_STREAM_END && stream.total_out == dst_len ? 0 : -1; -} - -static int simple_archive_entries(const unsigned char *data, size_t len, size_t *cd_offset, uint16_t *entry_count) { - size_t eocd; - if (len < 22 || find_eocd(data, len, &eocd) != 0 || eocd > len - 22) - return -1; - *entry_count = read_le16(data + eocd + 10); - *cd_offset = read_le32(data + eocd + 16); - return *cd_offset < len ? 0 : -1; -} - -static int simple_list_archive(const char *archive) { - unsigned char *data = NULL; - size_t len = 0; - size_t pos; - uint16_t entries; - unsigned long total_size = 0; - if (read_archive_bytes(archive, &data, &len) != 0 || - simple_archive_entries(data, len, &pos, &entries) != 0) { - free(data); - return 1; - } - - printf(" Length Name\n"); - printf("--------- ----\n"); - for (uint16_t i = 0; i < entries; i++) { - uint16_t name_len; - uint16_t extra_len; - uint16_t comment_len; - uint32_t uncompressed_size; - if (len < 46 || pos > len - 46 || read_le32(data + pos) != 0x02014b50) { - free(data); - return 1; - } - uncompressed_size = read_le32(data + pos + 24); - name_len = read_le16(data + pos + 28); - extra_len = read_le16(data + pos + 30); - comment_len = read_le16(data + pos + 32); - size_t header_len = 46 + (size_t)name_len + (size_t)extra_len + (size_t)comment_len; - if (header_len > len - pos) { - free(data); - return 1; - } - printf("%9lu %.*s\n", (unsigned long)uncompressed_size, name_len, data + pos + 46); - total_size += uncompressed_size; - pos += header_len; - } - printf("--------- ----\n"); - printf("%9lu %u file(s)\n", total_size, entries); - free(data); - return 0; -} - -static int simple_extract_archive(const char *archive, const char *outdir) { - unsigned char *data = NULL; - size_t len = 0; - size_t pos; - uint16_t entries; - int errors = 0; - if (read_archive_bytes(archive, &data, &len) != 0 || - simple_archive_entries(data, len, &pos, &entries) != 0) { - free(data); - return 1; - } - - if (outdir && mkdir(outdir, 0755) != 0 && errno != EEXIST) { - fprintf(stderr, "unzip: cannot create directory '%s': %s\n", outdir, strerror(errno)); - free(data); - return 1; - } - - for (uint16_t i = 0; i < entries; i++) { - uint16_t method; - uint16_t name_len; - uint16_t extra_len; - uint16_t comment_len; - uint16_t local_name_len; - uint16_t local_extra_len; - uint32_t compressed_size; - uint32_t uncompressed_size; - uint32_t local_offset; - size_t file_data_offset; - const char *name; - const char *safe_name; - char outpath[MAX_PATH_LEN]; - unsigned char *out = NULL; - - if (len < 46 || pos > len - 46 || read_le32(data + pos) != 0x02014b50) { - errors++; - break; - } - method = read_le16(data + pos + 10); - compressed_size = read_le32(data + pos + 20); - uncompressed_size = read_le32(data + pos + 24); - name_len = read_le16(data + pos + 28); - extra_len = read_le16(data + pos + 30); - comment_len = read_le16(data + pos + 32); - local_offset = read_le32(data + pos + 42); - size_t header_len = 46 + (size_t)name_len + (size_t)extra_len + (size_t)comment_len; - if (header_len > len - pos || (size_t)local_offset > len - 30) { - errors++; - break; - } - - name = (const char *)(data + pos + 46); - safe_name = entry_output_name(name, name_len); - size_t safe_len = (size_t)name_len - (size_t)(safe_name - name); - pos += header_len; - if (safe_len == 0) - continue; - snprintf(outpath, sizeof(outpath), "%s%s%.*s", - outdir ? outdir : "", outdir ? "/" : "", (int)safe_len, safe_name); - - size_t out_len = strlen(outpath); - if (out_len > 0 && outpath[out_len - 1] == '/') { - if (mkdir(outpath, 0755) != 0 && errno != EEXIST) - errors++; - continue; - } - if (mkdirs(outpath) != 0) { - errors++; - continue; - } - - if (read_le32(data + local_offset) != 0x04034b50) { - errors++; - continue; - } - local_name_len = read_le16(data + local_offset + 26); - local_extra_len = read_le16(data + local_offset + 28); - size_t local_header_len = 30 + (size_t)local_name_len + (size_t)local_extra_len; - if (local_header_len > len - (size_t)local_offset) { - errors++; - continue; - } - file_data_offset = (size_t)local_offset + local_header_len; - if ((size_t)compressed_size > len - file_data_offset) { - errors++; - continue; - } - - if (uncompressed_size > MAX_UNCOMPRESSED_SIZE) { - fprintf(stderr, "unzip: entry '%.*s' too large (%lu bytes)\n", - (int)safe_len, safe_name, (unsigned long)uncompressed_size); - errors++; - continue; - } - out = (unsigned char *)malloc(uncompressed_size > 0 ? uncompressed_size : 1); - if (!out) { - errors++; - continue; - } - if (method == 0) { - if (compressed_size != uncompressed_size) { - errors++; - free(out); - continue; - } - memcpy(out, data + file_data_offset, uncompressed_size); - } else if (method == Z_DEFLATED) { - if (inflate_raw_entry(data + file_data_offset, compressed_size, out, uncompressed_size) != 0) { - errors++; - free(out); - continue; - } - } else { - fprintf(stderr, "unzip: unsupported compression method %u for '%.*s'\n", method, name_len, name); - errors++; - free(out); - continue; - } - - int fd = open(outpath, O_WRONLY | O_CREAT | O_TRUNC, 0644); - if (fd < 0) { - fprintf(stderr, "unzip: cannot create '%s': %s\n", outpath, strerror(errno)); - errors++; - free(out); - continue; - } - size_t written = 0; - while (written < uncompressed_size) { - ssize_t n = write(fd, out + written, uncompressed_size - written); - if (n <= 0) { - errors++; - break; - } - written += (size_t)n; - } - close(fd); - free(out); - } - - free(data); - if (errors > 0) { - fprintf(stderr, "unzip: completed with %d error(s)\n", errors); - return 1; - } - return 0; -} - -/* List archive contents */ -static int list_archive(const char *archive) { - unzFile uf = open_archive(archive); - if (!uf) { - return simple_list_archive(archive); - } - - unz_global_info gi; - if (unzGetGlobalInfo(uf, &gi) != UNZ_OK) { - fprintf(stderr, "unzip: cannot read archive info\n"); - unzClose(uf); - return 1; - } - - printf(" Length Name\n"); - printf("--------- ----\n"); - - unsigned long total_size = 0; - for (uLong i = 0; i < gi.number_entry; i++) { - char filename[MAX_PATH_LEN]; - unz_file_info fi; - if (unzGetCurrentFileInfo(uf, &fi, filename, sizeof(filename), - NULL, 0, NULL, 0) != UNZ_OK) { - fprintf(stderr, "unzip: error reading file info\n"); - unzClose(uf); - return 1; - } - - printf("%9lu %s\n", fi.uncompressed_size, filename); - total_size += fi.uncompressed_size; - - if (i + 1 < gi.number_entry) { - if (unzGoToNextFile(uf) != UNZ_OK) { - fprintf(stderr, "unzip: error iterating archive\n"); - unzClose(uf); - return 1; - } - } - } - - printf("--------- ----\n"); - printf("%9lu %lu file(s)\n", total_size, gi.number_entry); - - unzClose(uf); - return 0; -} - -/* Extract a single file from the archive */ -static int extract_current_file(unzFile uf, const char *outdir) { - char filename[MAX_PATH_LEN]; - unz_file_info fi; - if (unzGetCurrentFileInfo(uf, &fi, filename, sizeof(filename), - NULL, 0, NULL, 0) != UNZ_OK) { - fprintf(stderr, "unzip: error reading file info\n"); - return -1; - } - - /* Build output path */ - char outpath[MAX_PATH_LEN]; - if (outdir) { - snprintf(outpath, sizeof(outpath), "%s/%s", outdir, filename); - } else { - snprintf(outpath, sizeof(outpath), "%s", filename); - } - - /* Directory entry (trailing slash) */ - size_t namelen = strlen(outpath); - if (namelen > 0 && outpath[namelen - 1] == '/') { - if (mkdir(outpath, 0755) != 0 && errno != EEXIST) { - fprintf(stderr, "unzip: cannot create directory '%s': %s\n", - outpath, strerror(errno)); - return -1; - } - return 0; - } - - /* Ensure parent directory exists */ - if (mkdirs(outpath) != 0) { - fprintf(stderr, "unzip: cannot create parent directories for '%s'\n", outpath); - return -1; - } - - if (unzOpenCurrentFile(uf) != UNZ_OK) { - fprintf(stderr, "unzip: cannot open '%s' in archive\n", filename); - return -1; - } - - FILE *fout = fopen(outpath, "wb"); - if (!fout) { - fprintf(stderr, "unzip: cannot create '%s': %s\n", outpath, strerror(errno)); - unzCloseCurrentFile(uf); - return -1; - } - - unsigned char buf[WRITE_BUF_SIZE]; - int err = UNZ_OK; - int bytes; - while ((bytes = unzReadCurrentFile(uf, buf, sizeof(buf))) > 0) { - if (fwrite(buf, 1, (size_t)bytes, fout) != (size_t)bytes) { - fprintf(stderr, "unzip: error writing '%s'\n", outpath); - err = -1; - break; - } - } - if (bytes < 0) { - fprintf(stderr, "unzip: error reading '%s' from archive\n", filename); - err = -1; - } - - fclose(fout); - unzCloseCurrentFile(uf); - return err; -} - -/* Extract all files from the archive */ -static int extract_archive(const char *archive, const char *outdir) { - unzFile uf = open_archive(archive); - if (!uf) { - return simple_extract_archive(archive, outdir); - } - - /* Create output directory if specified */ - if (outdir) { - if (mkdir(outdir, 0755) != 0 && errno != EEXIST) { - fprintf(stderr, "unzip: cannot create directory '%s': %s\n", - outdir, strerror(errno)); - unzClose(uf); - return 1; - } - } - - unz_global_info gi; - if (unzGetGlobalInfo(uf, &gi) != UNZ_OK) { - fprintf(stderr, "unzip: cannot read archive info\n"); - unzClose(uf); - return 1; - } - - int errors = 0; - for (uLong i = 0; i < gi.number_entry; i++) { - if (extract_current_file(uf, outdir) != 0) - errors++; - - if (i + 1 < gi.number_entry) { - if (unzGoToNextFile(uf) != UNZ_OK) { - fprintf(stderr, "unzip: error iterating archive\n"); - unzClose(uf); - return 1; - } - } - } - - unzClose(uf); - - if (errors > 0) { - fprintf(stderr, "unzip: completed with %d error(s)\n", errors); - return 1; - } - return 0; -} - -static void print_usage(void) { - fprintf(stderr, "Usage: unzip [-l] [-d dir] archive.zip\n"); - fprintf(stderr, " -l List archive contents\n"); - fprintf(stderr, " -d dir Extract to directory\n"); -} - -int main(int argc, char *argv[]) { - if (argc < 2) { - print_usage(); - return 1; - } - - int list_mode = 0; - const char *outdir = NULL; - const char *archive = NULL; - int i = 1; - - /* Parse flags */ - while (i < argc && argv[i][0] == '-') { - if (strcmp(argv[i], "-l") == 0) { - list_mode = 1; - i++; - } else if (strcmp(argv[i], "-d") == 0) { - if (i + 1 >= argc) { - fprintf(stderr, "unzip: -d requires a directory argument\n"); - return 1; - } - outdir = argv[i + 1]; - i += 2; - } else if (strcmp(argv[i], "--") == 0) { - i++; - break; - } else { - fprintf(stderr, "unzip: unknown option '%s'\n", argv[i]); - print_usage(); - return 1; - } - } - - if (i >= argc) { - fprintf(stderr, "unzip: no archive specified\n"); - print_usage(); - return 1; - } - - archive = argv[i]; - - if (list_mode) { - return list_archive(archive); - } else { - return extract_archive(archive, outdir); - } -} diff --git a/registry/native/c/programs/wget.c b/registry/native/c/programs/wget.c deleted file mode 100644 index e0e0ca02fd..0000000000 --- a/registry/native/c/programs/wget.c +++ /dev/null @@ -1,174 +0,0 @@ -/* - * wget.c - minimal wget implementation built on libcurl - * - * Supports common wget options for HTTP/HTTPS downloads: - * URL Download file to current directory (basename from URL) - * -O FILE Write output to specific file ("-" for stdout) - * -q Quiet mode (suppress progress/messages) - * -L Follow redirects (enabled by default, like real wget) - * --no-check-certificate Skip TLS certificate verification - */ - -#include -#include -#include -#include - -/* Write callback: write to FILE* */ -static size_t write_callback(char *ptr, size_t size, size_t nmemb, - void *userdata) { - FILE *out = (FILE *)userdata; - return fwrite(ptr, size, nmemb, out); -} - -/* Extract filename from URL path, fallback to "index.html" */ -static const char *basename_from_url(const char *url) { - /* Skip scheme */ - const char *p = strstr(url, "://"); - if (p) p += 3; else p = url; - - /* Find last '/' in path (before query string) */ - const char *last_slash = NULL; - const char *q = p; - while (*q && *q != '?' && *q != '#') { - if (*q == '/') last_slash = q; - q++; - } - - if (last_slash && last_slash[1] && last_slash[1] != '?' && last_slash[1] != '#') { - /* Extract filename between last slash and query/end */ - const char *start = last_slash + 1; - size_t len = 0; - while (start[len] && start[len] != '?' && start[len] != '#') len++; - if (len > 0) { - static char buf[256]; - if (len >= sizeof(buf)) len = sizeof(buf) - 1; - memcpy(buf, start, len); - buf[len] = '\0'; - return buf; - } - } - - return "index.html"; -} - -int main(int argc, char *argv[]) { - const char *url = NULL; - const char *output_file = NULL; - int quiet = 0; - int no_check_cert = 0; - int output_to_stdout = 0; - - /* Parse arguments */ - for (int i = 1; i < argc; i++) { - if (strcmp(argv[i], "-O") == 0 && i + 1 < argc) { - output_file = argv[++i]; - if (strcmp(output_file, "-") == 0) { - output_to_stdout = 1; - output_file = NULL; - } - } else if (strcmp(argv[i], "-q") == 0) { - quiet = 1; - } else if (strcmp(argv[i], "-L") == 0) { - /* Follow redirects — already default, accept silently */ - } else if (strcmp(argv[i], "--no-check-certificate") == 0) { - no_check_cert = 1; - } else if (argv[i][0] != '-') { - url = argv[i]; - } else { - /* Unknown option — skip silently for forward compat */ - } - } - - if (!url) { - fprintf(stderr, "wget: missing URL\nUsage: wget [OPTION]... [URL]...\n"); - return 1; - } - - CURLcode res; - curl_global_init(CURL_GLOBAL_DEFAULT); - - CURL *curl = curl_easy_init(); - if (!curl) { - fprintf(stderr, "wget: failed to initialize\n"); - curl_global_cleanup(); - return 1; - } - - /* Determine output destination */ - FILE *out = NULL; - const char *dest_name = NULL; - - if (output_to_stdout) { - out = stdout; - } else { - dest_name = output_file ? output_file : basename_from_url(url); - out = fopen(dest_name, "wb"); - if (!out) { - fprintf(stderr, "wget: cannot open '%s' for writing\n", dest_name); - curl_easy_cleanup(curl); - curl_global_cleanup(); - return 1; - } - } - - curl_easy_setopt(curl, CURLOPT_URL, url); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, out); - - /* wget follows redirects by default */ - curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); - curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 20L); - - /* Suppress progress meter */ - curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1L); - - /* TLS: skip certificate verification */ - if (no_check_cert) { - curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L); - curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L); - } - - /* Perform request */ - res = curl_easy_perform(curl); - - int exit_code = 0; - - if (res != CURLE_OK) { - if (!quiet) { - fprintf(stderr, "wget: failed: %s\n", curl_easy_strerror(res)); - } - exit_code = 1; - /* Remove partial download on failure (unless stdout) */ - if (!output_to_stdout && dest_name) { - fclose(out); - out = NULL; - remove(dest_name); - } - } else { - /* Check HTTP response code */ - long http_code = 0; - curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code); - if (http_code >= 400) { - if (!quiet) { - fprintf(stderr, "wget: server returned HTTP %ld\n", http_code); - } - exit_code = 8; /* wget uses exit code 8 for server errors */ - /* Remove error response file (unless stdout) */ - if (!output_to_stdout && dest_name) { - fclose(out); - out = NULL; - remove(dest_name); - } - } - } - - /* Cleanup */ - curl_easy_cleanup(curl); - if (out && out != stdout) { - fclose(out); - } - curl_global_cleanup(); - - return exit_code; -} diff --git a/registry/native/c/programs/zip.c b/registry/native/c/programs/zip.c deleted file mode 100644 index d8eca7a5bd..0000000000 --- a/registry/native/c/programs/zip.c +++ /dev/null @@ -1,203 +0,0 @@ -/* zip.c — Create ZIP archives using zlib/minizip - * - * Usage: zip [-r] archive.zip file1 [file2 ...] - * -r Recurse into directories - */ - -#include -#include -#include -#include -#include -#include -#include "zip.h" - -#define MAX_PATH_LEN 4096 -#define READ_BUF_SIZE 8192 - -/* Add a single file to the zip archive */ -static int add_file_to_zip(zipFile zf, const char *filepath, const char *archivepath) { - FILE *fin = fopen(filepath, "rb"); - if (!fin) { - fprintf(stderr, "zip: cannot open '%s': ", filepath); - perror(""); - return -1; - } - - zip_fileinfo zi; - memset(&zi, 0, sizeof(zi)); - - /* Get file modification time */ - struct stat st; - if (stat(filepath, &st) == 0) { - struct tm *lt = localtime(&st.st_mtime); - if (lt) { - zi.tmz_date.tm_sec = lt->tm_sec; - zi.tmz_date.tm_min = lt->tm_min; - zi.tmz_date.tm_hour = lt->tm_hour; - zi.tmz_date.tm_mday = lt->tm_mday; - zi.tmz_date.tm_mon = lt->tm_mon; - zi.tmz_date.tm_year = lt->tm_year; - } - } - - int err = zipOpenNewFileInZip(zf, archivepath, &zi, - NULL, 0, NULL, 0, NULL, - Z_DEFLATED, Z_DEFAULT_COMPRESSION); - if (err != ZIP_OK) { - fprintf(stderr, "zip: error opening '%s' in archive\n", archivepath); - fclose(fin); - return -1; - } - - unsigned char buf[READ_BUF_SIZE]; - size_t n; - while ((n = fread(buf, 1, sizeof(buf), fin)) > 0) { - if (zipWriteInFileInZip(zf, buf, (unsigned int)n) != ZIP_OK) { - fprintf(stderr, "zip: error writing '%s' to archive\n", archivepath); - fclose(fin); - zipCloseFileInZip(zf); - return -1; - } - } - - fclose(fin); - zipCloseFileInZip(zf); - return 0; -} - -/* Recursively add directory contents to zip */ -static int add_dir_to_zip(zipFile zf, const char *dirpath, const char *archivebase) { - DIR *d = opendir(dirpath); - if (!d) { - fprintf(stderr, "zip: cannot open directory '%s': ", dirpath); - perror(""); - return -1; - } - - /* Add directory entry (trailing slash) */ - char direntry[MAX_PATH_LEN]; - snprintf(direntry, sizeof(direntry), "%s/", archivebase); - zip_fileinfo zi; - memset(&zi, 0, sizeof(zi)); - zipOpenNewFileInZip(zf, direntry, &zi, NULL, 0, NULL, 0, NULL, 0, 0); - zipCloseFileInZip(zf); - - struct dirent *entry; - int err = 0; - while ((entry = readdir(d)) != NULL) { - if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0) - continue; - - char fullpath[MAX_PATH_LEN]; - char arcpath[MAX_PATH_LEN]; - snprintf(fullpath, sizeof(fullpath), "%s/%s", dirpath, entry->d_name); - snprintf(arcpath, sizeof(arcpath), "%s/%s", archivebase, entry->d_name); - - struct stat st; - if (stat(fullpath, &st) != 0) { - fprintf(stderr, "zip: cannot stat '%s': ", fullpath); - perror(""); - err = -1; - continue; - } - - if (S_ISDIR(st.st_mode)) { - if (add_dir_to_zip(zf, fullpath, arcpath) != 0) - err = -1; - } else { - if (add_file_to_zip(zf, fullpath, arcpath) != 0) - err = -1; - } - } - - closedir(d); - return err; -} - -static void print_usage(void) { - fprintf(stderr, "Usage: zip [-r] archive.zip file1 [file2 ...]\n"); - fprintf(stderr, " -r Recurse into directories\n"); -} - -int main(int argc, char *argv[]) { - if (argc < 3) { - print_usage(); - return 1; - } - - int recursive = 0; - int arg_start = 1; - - /* Parse flags */ - while (arg_start < argc && argv[arg_start][0] == '-') { - if (strcmp(argv[arg_start], "-r") == 0) { - recursive = 1; - arg_start++; - } else if (strcmp(argv[arg_start], "--") == 0) { - arg_start++; - break; - } else { - fprintf(stderr, "zip: unknown option '%s'\n", argv[arg_start]); - print_usage(); - return 1; - } - } - - if (argc - arg_start < 2) { - print_usage(); - return 1; - } - - const char *archive = argv[arg_start]; - arg_start++; - - zipFile zf = zipOpen(archive, APPEND_STATUS_CREATE); - if (!zf) { - fprintf(stderr, "zip: cannot create '%s'\n", archive); - return 1; - } - - int errors = 0; - for (int i = arg_start; i < argc; i++) { - const char *path = argv[i]; - - /* Strip trailing slashes for consistent archive paths */ - char cleanpath[MAX_PATH_LEN]; - strncpy(cleanpath, path, sizeof(cleanpath) - 1); - cleanpath[sizeof(cleanpath) - 1] = '\0'; - size_t len = strlen(cleanpath); - while (len > 1 && cleanpath[len - 1] == '/') - cleanpath[--len] = '\0'; - - struct stat st; - if (stat(cleanpath, &st) != 0) { - fprintf(stderr, "zip: cannot stat '%s': ", cleanpath); - perror(""); - errors++; - continue; - } - - if (S_ISDIR(st.st_mode)) { - if (!recursive) { - fprintf(stderr, "zip: '%s' is a directory (use -r to recurse)\n", cleanpath); - errors++; - continue; - } - if (add_dir_to_zip(zf, cleanpath, cleanpath) != 0) - errors++; - } else { - if (add_file_to_zip(zf, cleanpath, cleanpath) != 0) - errors++; - } - } - - zipClose(zf, NULL); - - if (errors > 0) { - fprintf(stderr, "zip: completed with %d error(s)\n", errors); - return 1; - } - - return 0; -} diff --git a/registry/native/c/scripts/build-curl-upstream.sh b/registry/native/c/scripts/build-curl-upstream.sh deleted file mode 100644 index d2ffe82c2e..0000000000 --- a/registry/native/c/scripts/build-curl-upstream.sh +++ /dev/null @@ -1,251 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -usage() { - cat <<'EOF' -Usage: build-curl-upstream.sh \ - --version \ - --tag \ - --url \ - --cache-dir \ - --build-dir \ - --overlay-dir \ - --cc \ - --ar \ - --ranlib \ - --output -EOF -} - -VERSION="" -TAG="" -URL="" -CACHE_DIR="" -BUILD_DIR="" -OVERLAY_DIR="" -CC_CMD="" -AR_CMD="" -RANLIB_CMD="" -OUTPUT="" - -while [[ $# -gt 0 ]]; do - case "$1" in - --version) - VERSION="$2" - shift 2 - ;; - --tag) - TAG="$2" - shift 2 - ;; - --url) - URL="$2" - shift 2 - ;; - --cache-dir) - CACHE_DIR="$2" - shift 2 - ;; - --build-dir) - BUILD_DIR="$2" - shift 2 - ;; - --overlay-dir) - OVERLAY_DIR="$2" - shift 2 - ;; - --cc) - CC_CMD="$2" - shift 2 - ;; - --ar) - AR_CMD="$2" - shift 2 - ;; - --ranlib) - RANLIB_CMD="$2" - shift 2 - ;; - --output) - OUTPUT="$2" - shift 2 - ;; - *) - echo "Unknown argument: $1" >&2 - usage >&2 - exit 1 - ;; - esac -done - -if [[ -z "$VERSION" || -z "$TAG" || -z "$URL" || -z "$CACHE_DIR" || -z "$BUILD_DIR" || -z "$OVERLAY_DIR" || -z "$CC_CMD" || -z "$AR_CMD" || -z "$RANLIB_CMD" || -z "$OUTPUT" ]]; then - usage >&2 - exit 1 -fi - -fetch() { - local url="$1" - local out="$2" - if command -v curl >/dev/null 2>&1; then - curl -fSL "$url" -o "$out" - elif command -v wget >/dev/null 2>&1; then - wget -q "$url" -O "$out" - else - echo "Neither curl nor wget is available to fetch $url" >&2 - exit 1 - fi -} - -mkdir -p "$CACHE_DIR" -rm -rf "$BUILD_DIR" -mkdir -p "$BUILD_DIR" - -TARBALL="$CACHE_DIR/curl-${VERSION}.tar.xz" -if [[ ! -f "$TARBALL" ]]; then - echo "Fetching upstream curl ${VERSION} release tarball..." - fetch "$URL" "$TARBALL" -fi - -echo "Extracting upstream curl ${VERSION}..." -tar -xf "$TARBALL" -C "$BUILD_DIR" - -SRC_DIR="$BUILD_DIR/curl-${VERSION}" -if [[ ! -d "$SRC_DIR" ]]; then - echo "Expected extracted source at $SRC_DIR" >&2 - exit 1 -fi - -echo "Applying secure-exec overlay..." -while IFS= read -r -d '' file; do - rel="${file#$OVERLAY_DIR/}" - mkdir -p "$SRC_DIR/$(dirname "$rel")" - cp "$file" "$SRC_DIR/$rel" -done < <(find "$OVERLAY_DIR" -type f -print0) - -pushd "$SRC_DIR" >/dev/null - -echo "Patching WASI-incompatible signal/setjmp includes..." -python3 - <<'PY' -from pathlib import Path - -replacements = { - "lib/hostip.h": [ - ( - '#include \n', - '#ifndef __wasi__\n#include \n#endif\n', - ), - ], - "lib/hostip.c": [ - ( - '#include \n#include \n', - '#ifndef __wasi__\n#include \n#include \n#endif\n', - ), - ], - "lib/transfer.c": [ - ( - '#include \n', - '#ifndef __wasi__\n#include \n#endif\n', - ), - ], - "src/tool_main.c": [ - ( - '#include \n', - '#ifndef __wasi__\n#include \n#endif\n', - ), - ], -} - -for rel_path, edits in replacements.items(): - path = Path(rel_path) - updated = path.read_text() - for old, new in edits: - text = updated - if new in text: - continue - if old not in text: - raise SystemExit(f"Expected to patch {rel_path}, but no replacement matched") - updated = text.replace(old, new) - path.write_text(updated) -PY - -echo "Configuring upstream curl for wasm32-wasip1..." -CC="$CC_CMD" \ -AR="$AR_CMD" \ -RANLIB="$RANLIB_CMD" \ -CFLAGS="-O2 -flto" \ -./configure \ - --host=wasm32-unknown-wasi \ - --disable-shared \ - --disable-threaded-resolver \ - --disable-ldap \ - --without-zlib \ - --without-brotli \ - --without-zstd \ - --without-libpsl \ - --without-ca-bundle \ - --without-ca-path \ - --without-ssl - -echo "Enabling the secure-exec WASI TLS backend in generated curl config..." -python3 - <<'PY' -from pathlib import Path - -config = Path("lib/curl_config.h") -text = config.read_text() - -updates = { - "#define CURL_DISABLE_HSTS 1": "/* #undef CURL_DISABLE_HSTS */", -} -for old, new in updates.items(): - text = text.replace(old, new) - -if "#define USE_WASI_TLS 1" not in text: - text += "\n#define USE_WASI_TLS 1\n" -if "#define CURL_DISABLE_OPENSSL_AUTO_LOAD_CONFIG 1" not in text: - text += "#define CURL_DISABLE_OPENSSL_AUTO_LOAD_CONFIG 1\n" - -config.write_text(text) -PY - -echo "Patching generated lib/Makefile to compile wasi_tls.c..." -cat >> lib/Makefile <<'EOF' - -am_libcurl_la_OBJECTS += vtls/libcurl_la-wasi_tls.lo -libcurl_la_LIBADD += vtls/libcurl_la-wasi_tls.lo -libcurl.la: vtls/libcurl_la-wasi_tls.lo - -vtls/libcurl_la-wasi_tls.lo: vtls/$(am__dirstamp) vtls/$(DEPDIR)/$(am__dirstamp) vtls/wasi_tls.c - $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libcurl_la_CPPFLAGS) $(CPPFLAGS) $(libcurl_la_CFLAGS) $(CFLAGS) -MT vtls/libcurl_la-wasi_tls.lo -MD -MP -MF vtls/$(DEPDIR)/libcurl_la-wasi_tls.Tpo -c -o vtls/libcurl_la-wasi_tls.lo `test -f 'vtls/wasi_tls.c' || echo '$(srcdir)/'`vtls/wasi_tls.c - $(AM_V_at)$(am__mv) vtls/$(DEPDIR)/libcurl_la-wasi_tls.Tpo vtls/$(DEPDIR)/libcurl_la-wasi_tls.Plo -EOF - -echo "Building upstream libcurl..." -make -C lib libcurl.la - -echo "Building upstream curl tool..." -make -C src curl - -BIN="" -for candidate in "src/.libs/curl" "src/curl" "src/curl.wasm"; do - if [[ -f "$candidate" ]]; then - BIN="$candidate" - break - fi -done - -if [[ -z "$BIN" ]]; then - echo "Unable to locate built curl binary in src/" >&2 - exit 1 -fi - -mkdir -p "$(dirname "$OUTPUT")" -if command -v wasm-opt >/dev/null 2>&1; then - echo "Optimizing curl WASM binary..." - wasm-opt -O3 --strip-debug --all-features "$BIN" -o "$OUTPUT" -else - cp "$BIN" "$OUTPUT" -fi - -popd >/dev/null - -echo "Built upstream curl at $OUTPUT" diff --git a/registry/native/c/vim/posix_stubs.c b/registry/native/c/vim/posix_stubs.c deleted file mode 100644 index f669d17bcf..0000000000 --- a/registry/native/c/vim/posix_stubs.c +++ /dev/null @@ -1,30 +0,0 @@ -/* posix_stubs.c — stubs for full-OS libc gaps vim references but that the VM - * does not implement (group database, etc.). Safe no-op behavior. */ -#include -#include -struct group *getgrgid(gid_t g) { (void)g; return NULL; } -struct group *getgrnam(const char *n) { (void)n; return NULL; } -struct group *getgrent(void) { return NULL; } -void setgrent(void) {} -void endgrent(void) {} - -/* --- additional full-OS libc gaps --- */ -#include -mode_t umask(mode_t mask) { (void)mask; return 0; } - -#include -struct itimerval; -int setitimer(int w, const struct itimerval *n, struct itimerval *o) { (void)w; (void)n; (void)o; return 0; } -int getitimer(int w, struct itimerval *o) { (void)w; (void)o; return 0; } - -/* --- process/signal stubs (not used by core editing) --- */ -#include -pid_t fork(void) { errno = ENOSYS; return -1; } -int execvp(const char *f, char *const a[]) { (void)f; (void)a; errno = ENOSYS; return -1; } -int raise(int s) { (void)s; return -1; } -/* signal sentinel symbols (this libc takes their address for SIG_IGN/SIG_ERR) */ -void __SIG_IGN(int s) { (void)s; } -void __SIG_ERR(int s) { (void)s; } -void __SIG_DFL(int s) { (void)s; } -int sigprocmask(int how, const void *set, void *old) { (void)how; (void)set; (void)old; return 0; } -int sigpending(void *set) { (void)set; return 0; } diff --git a/registry/native/crates/commands/arch/Cargo.toml b/registry/native/crates/commands/arch/Cargo.toml deleted file mode 100644 index 92f2e7757d..0000000000 --- a/registry/native/crates/commands/arch/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-arch" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "arch standalone binary for secure-exec VM" - -[[bin]] -name = "arch" -path = "src/main.rs" - -[dependencies] -uu_arch = "0.7.0" diff --git a/registry/native/crates/commands/awk/Cargo.toml b/registry/native/crates/commands/awk/Cargo.toml deleted file mode 100644 index 31ccd2a2bd..0000000000 --- a/registry/native/crates/commands/awk/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-awk" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "awk standalone binary for secure-exec VM" - -[[bin]] -name = "awk" -path = "src/main.rs" - -[dependencies] -secureexec-awk = { path = "../../libs/awk" } diff --git a/registry/native/crates/commands/b2sum/Cargo.toml b/registry/native/crates/commands/b2sum/Cargo.toml deleted file mode 100644 index 8219c1fd4b..0000000000 --- a/registry/native/crates/commands/b2sum/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-b2sum" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "b2sum standalone binary for secure-exec VM" - -[[bin]] -name = "b2sum" -path = "src/main.rs" - -[dependencies] -uu_b2sum = "0.7.0" diff --git a/registry/native/crates/commands/base32/Cargo.toml b/registry/native/crates/commands/base32/Cargo.toml deleted file mode 100644 index 3e719bf184..0000000000 --- a/registry/native/crates/commands/base32/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-base32" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "base32 standalone binary for secure-exec VM" - -[[bin]] -name = "base32" -path = "src/main.rs" - -[dependencies] -uu_base32 = "0.7.0" diff --git a/registry/native/crates/commands/base64/Cargo.toml b/registry/native/crates/commands/base64/Cargo.toml deleted file mode 100644 index bad7a503e1..0000000000 --- a/registry/native/crates/commands/base64/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-base64" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "base64 standalone binary for secure-exec VM" - -[[bin]] -name = "base64" -path = "src/main.rs" - -[dependencies] -uu_base64 = "0.7.0" diff --git a/registry/native/crates/commands/basename/Cargo.toml b/registry/native/crates/commands/basename/Cargo.toml deleted file mode 100644 index 8511991cdc..0000000000 --- a/registry/native/crates/commands/basename/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-basename" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "basename standalone binary for secure-exec VM" - -[[bin]] -name = "basename" -path = "src/main.rs" - -[dependencies] -uu_basename = "0.7.0" diff --git a/registry/native/crates/commands/basenc/Cargo.toml b/registry/native/crates/commands/basenc/Cargo.toml deleted file mode 100644 index ad2c8e5a92..0000000000 --- a/registry/native/crates/commands/basenc/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-basenc" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "basenc standalone binary for secure-exec VM" - -[[bin]] -name = "basenc" -path = "src/main.rs" - -[dependencies] -uu_basenc = "0.7.0" diff --git a/registry/native/crates/commands/cat/Cargo.toml b/registry/native/crates/commands/cat/Cargo.toml deleted file mode 100644 index 16a9d7218e..0000000000 --- a/registry/native/crates/commands/cat/Cargo.toml +++ /dev/null @@ -1,16 +0,0 @@ -[package] -name = "cmd-cat" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "cat standalone binary for secure-exec VM" - -[[bin]] -name = "cat" -path = "src/main.rs" - -[dependencies] -uu_cat = "0.7.0" - -[target.'cfg(any(target_os = "linux", target_os = "android"))'.dev-dependencies] -uucore = { version = "0.7.0", features = ["pipes"] } diff --git a/registry/native/crates/commands/chmod/Cargo.toml b/registry/native/crates/commands/chmod/Cargo.toml deleted file mode 100644 index 3a1da3b8c1..0000000000 --- a/registry/native/crates/commands/chmod/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-chmod" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "chmod standalone binary for secure-exec VM" - -[[bin]] -name = "chmod" -path = "src/main.rs" - -[dependencies] -uu_chmod = "0.7.0" diff --git a/registry/native/crates/commands/cksum/Cargo.toml b/registry/native/crates/commands/cksum/Cargo.toml deleted file mode 100644 index 1beec6b3a4..0000000000 --- a/registry/native/crates/commands/cksum/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-cksum" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "cksum standalone binary for secure-exec VM" - -[[bin]] -name = "cksum" -path = "src/main.rs" - -[dependencies] -uu_cksum = "0.7.0" diff --git a/registry/native/crates/commands/codex-exec/Cargo.toml b/registry/native/crates/commands/codex-exec/Cargo.toml deleted file mode 100644 index c2859ea274..0000000000 --- a/registry/native/crates/commands/codex-exec/Cargo.toml +++ /dev/null @@ -1,17 +0,0 @@ -[package] -name = "cmd-codex-exec" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "codex-exec command binary for secure-exec VM" - -[[bin]] -name = "codex-exec" -path = "src/main.rs" - -[dependencies] -wasi-http = { package = "secureexec-wasi-http", path = "../../libs/wasi-http" } - -# WASI stub crates for future codex-core dependencies that don't support wasm32-wasip1. -codex-network-proxy = "0.0.0" -codex-otel = "0.0.0" diff --git a/registry/native/crates/commands/codex/Cargo.toml b/registry/native/crates/commands/codex/Cargo.toml deleted file mode 100644 index 41f6109024..0000000000 --- a/registry/native/crates/commands/codex/Cargo.toml +++ /dev/null @@ -1,33 +0,0 @@ -[package] -name = "cmd-codex" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "codex standalone binary for secure-exec VM" - -[[bin]] -name = "codex" -path = "src/main.rs" - -# The full codex-exec dependency from rivet-dev/codex (wasi-support branch) -# will be wired in when the vendoring blocker is resolved. -# -# The fork at github.com/rivet-dev/codex already has cfg(target_os = "wasi") -# gates on codex-core (portable-pty, network-proxy, sandbox deps) and -# codex-exec (all platform deps gated behind non-WASI). -# -# wasi-spawn provides the WasiChild abstraction that will replace -# tokio::process::Command in codex-core when compiled for WASI. - -[dependencies] -wasi-spawn = { package = "secureexec-wasi-spawn", path = "../../libs/wasi-spawn" } -wasi-http = { package = "secureexec-wasi-http", path = "../../libs/wasi-http" } - -# TUI dependencies — ratatui for rendering, crossterm for terminal backend -# crossterm is patched for WASI support (see patches/crates/crossterm/) -ratatui = { version = "0.29", default-features = false, features = ["crossterm"] } - -# WASI stub crates for codex-core dependencies that don't support wasm32-wasip1. -# These provide zero-size/no-op implementations so codex-core compiles for WASI. -codex-network-proxy = "0.0.0" -codex-otel = "0.0.0" diff --git a/registry/native/crates/commands/codex/src/main.rs b/registry/native/crates/commands/codex/src/main.rs deleted file mode 100644 index 5375dfa82e..0000000000 --- a/registry/native/crates/commands/codex/src/main.rs +++ /dev/null @@ -1,337 +0,0 @@ -/// Codex TUI for secure-exec VM. -/// -/// Full terminal UI using ratatui + crossterm backend, rendering through -/// the WasmVM PTY. This is the interactive entry point — for headless -/// (scriptable) usage, see codex-exec. -/// -/// Uses wasi-spawn for process spawning via host_process FFI and -/// wasi-http for HTTP/HTTPS requests via host_net TCP/TLS imports. -/// -/// crossterm is patched for WASI support (see patches/crates/crossterm/): -/// - Terminal raw mode tracked in-process (host PTY handles discipline) -/// - Terminal size from COLUMNS/LINES env vars -/// - Event source reads stdin directly and parses ANSI escape sequences -/// - IsTty returns true (PTY slave FDs are terminals) -use std::io; - -use ratatui::{ - backend::CrosstermBackend, - crossterm::{ - event::{self, Event, KeyCode, KeyEvent, KeyModifiers}, - execute, - terminal::{self, EnterAlternateScreen, LeaveAlternateScreen}, - }, - layout::{Constraint, Layout}, - style::{Color, Modifier, Style}, - text::{Line, Span, Text}, - widgets::{Block, Borders, Paragraph, Wrap}, - Frame, Terminal, -}; - -// Validate WASI stub crates compile by referencing key types -use codex_network_proxy::NetworkProxy; -use codex_otel::SessionTelemetry; - -const VERSION: &str = env!("CARGO_PKG_VERSION"); -const MAX_INPUT_CHARS: usize = 8192; -const MAX_MESSAGES: usize = 200; - -fn main() { - let args: Vec = std::env::args().collect(); - - // Handle --help - if args.iter().any(|a| a == "--help" || a == "-h") { - print_help(); - return; - } - - // Handle --version - if args.iter().any(|a| a == "--version" || a == "-V") { - println!("codex {}", VERSION); - return; - } - - // Handle --model flag (accept but note it's stored for future use) - let model = args - .windows(2) - .find(|w| w[0] == "--model") - .map(|w| w[1].clone()); - - // Built-in HTTP test subcommand (validates wasi-http integration) - if args.get(1).map(|s| s.as_str()) == Some("--http-test") { - return http_test(&args[2..]); - } - - // Stub validation subcommand (validates WASI stub crates) - if args.get(1).map(|s| s.as_str()) == Some("--stub-test") { - return stub_test(); - } - - // Run the TUI - if let Err(e) = run_tui(model.as_deref()) { - eprintln!("codex: TUI error: {}", e); - std::process::exit(1); - } -} - -/// Main TUI event loop using ratatui + crossterm. -fn run_tui(model: Option<&str>) -> io::Result<()> { - let _terminal_guard = TerminalGuard::enter()?; - - let stdout = io::stdout(); - let backend = CrosstermBackend::new(stdout); - let mut terminal = Terminal::new(backend)?; - - // App state - let mut input = String::new(); - let mut messages: Vec = Vec::new(); - let mut should_quit = false; - - // Draw initial frame - terminal.draw(|f| draw_ui(f, &input, &messages, model))?; - - // Event loop - while !should_quit { - match event::read()? { - Event::Key(key) => { - handle_key_event(key, &mut input, &mut messages, &mut should_quit); - terminal.draw(|f| draw_ui(f, &input, &messages, model))?; - } - Event::Resize(_, _) => { - terminal.draw(|f| draw_ui(f, &input, &messages, model))?; - } - _ => {} - } - } - - Ok(()) -} - -/// Handle a key event, updating app state. -fn handle_key_event( - key: KeyEvent, - input: &mut String, - messages: &mut Vec, - should_quit: &mut bool, -) { - match (key.code, key.modifiers) { - // Ctrl+C quits - (KeyCode::Char('c'), m) if m.contains(KeyModifiers::CONTROL) => { - *should_quit = true; - } - // 'q' on empty input quits - (KeyCode::Char('q'), KeyModifiers::NONE) if input.is_empty() => { - *should_quit = true; - } - // Enter submits the input - (KeyCode::Enter, _) => { - if !input.is_empty() { - let prompt = input.clone(); - push_message(messages, format!("> {}", prompt)); - push_message( - messages, - "codex: agent loop is under development".to_string(), - ); - push_message( - messages, - format!("codex: prompt received ({} chars)", prompt.len()), - ); - input.clear(); - } - } - // Backspace deletes last char - (KeyCode::Backspace, _) => { - input.pop(); - } - // Esc clears input - (KeyCode::Esc, _) => { - input.clear(); - } - // Regular character input - (KeyCode::Char(c), _) => { - if input.chars().count() < MAX_INPUT_CHARS { - input.push(c); - } - } - _ => {} - } -} - -fn push_message(messages: &mut Vec, message: String) { - messages.push(message); - if messages.len() > MAX_MESSAGES { - messages.drain(..messages.len() - MAX_MESSAGES); - } -} - -struct TerminalGuard { - raw_mode_enabled: bool, - alternate_screen_enabled: bool, -} - -impl TerminalGuard { - fn enter() -> io::Result { - terminal::enable_raw_mode()?; - - if let Err(error) = execute!(io::stdout(), EnterAlternateScreen) { - let _ = terminal::disable_raw_mode(); - return Err(error); - } - - Ok(Self { - raw_mode_enabled: true, - alternate_screen_enabled: true, - }) - } -} - -impl Drop for TerminalGuard { - fn drop(&mut self) { - if self.alternate_screen_enabled { - let _ = execute!(io::stdout(), LeaveAlternateScreen); - self.alternate_screen_enabled = false; - } - - if self.raw_mode_enabled { - let _ = terminal::disable_raw_mode(); - self.raw_mode_enabled = false; - } - } -} - -/// Draw the TUI layout. -fn draw_ui(f: &mut Frame, input: &str, messages: &[String], model: Option<&str>) { - let area = f.area(); - - let chunks = Layout::vertical([ - Constraint::Length(3), // Header - Constraint::Min(5), // Messages - Constraint::Length(3), // Input - ]) - .split(area); - - // Header - let model_text = model.unwrap_or("default"); - let header = Paragraph::new(Line::from(vec![ - Span::styled( - "Codex ", - Style::default() - .fg(Color::Cyan) - .add_modifier(Modifier::BOLD), - ), - Span::styled(VERSION, Style::default().fg(Color::DarkGray)), - Span::raw(" "), - Span::styled( - format!("model: {}", model_text), - Style::default().fg(Color::Yellow), - ), - Span::raw(" "), - Span::styled("WasmVM", Style::default().fg(Color::Green)), - ])) - .block(Block::default().borders(Borders::ALL).title(" codex ")); - f.render_widget(header, chunks[0]); - - // Messages area - let msg_lines: Vec = messages - .iter() - .map(|m| { - if m.starts_with("> ") { - Line::from(Span::styled(m.as_str(), Style::default().fg(Color::Cyan))) - } else { - Line::from(Span::raw(m.as_str())) - } - }) - .collect(); - - let welcome = if messages.is_empty() { - vec![ - Line::from(""), - Line::from(Span::styled( - "Welcome to Codex on WasmVM!", - Style::default() - .fg(Color::Cyan) - .add_modifier(Modifier::BOLD), - )), - Line::from(""), - Line::from("Type a prompt and press Enter to submit."), - Line::from("Press 'q' (on empty input) or Ctrl+C to exit."), - ] - } else { - msg_lines - }; - - let messages_widget = Paragraph::new(Text::from(welcome)) - .block(Block::default().borders(Borders::ALL).title(" messages ")) - .wrap(Wrap { trim: false }); - f.render_widget(messages_widget, chunks[1]); - - // Input area - let input_widget = Paragraph::new(Line::from(vec![ - Span::styled("❯ ", Style::default().fg(Color::Green)), - Span::raw(input), - ])) - .block(Block::default().borders(Borders::ALL).title(" input ")); - f.render_widget(input_widget, chunks[2]); -} - -fn print_help() { - println!( - "codex {} — interactive Codex TUI for secure-exec VM", - VERSION - ); - println!(); - println!("USAGE:"); - println!(" codex [OPTIONS]"); - println!(); - println!("OPTIONS:"); - println!(" -h, --help Print this help message"); - println!(" -V, --version Print version information"); - println!(" --model MODEL Select model for completions"); - println!(" --http-test URL Test HTTP client via host_net"); - println!(" --stub-test Validate WASI stub crates"); - println!(); - println!("DESCRIPTION:"); - println!(" Interactive TUI for the Codex agent, rendered through"); - println!(" the WasmVM PTY using ratatui + crossterm backend."); - println!(" For headless mode, use codex-exec instead."); -} - -fn stub_test() { - let proxy = NetworkProxy; - let mut env = std::collections::HashMap::new(); - proxy.apply_to_env(&mut env); - println!("network-proxy: NetworkProxy is zero-size, apply_to_env is no-op"); - - let telemetry = SessionTelemetry::new(); - telemetry.counter("test.counter", 1, &[]); - telemetry.histogram("test.histogram", 42, &[]); - println!("otel: SessionTelemetry metrics are no-ops"); - - let global = codex_otel::metrics::global(); - assert!(global.is_none(), "global metrics should be None on WASI"); - println!("otel: global() returns None (no exporter on WASI)"); - - println!("stub-test: all stubs validated successfully"); -} - -fn http_test(args: &[String]) { - if args.is_empty() { - eprintln!("usage: codex --http-test "); - std::process::exit(1); - } - - let url = &args[0]; - match wasi_http::get(url) { - Ok(resp) => { - println!("status: {}", resp.status); - match resp.text() { - Ok(body) => println!("body: {}", body), - Err(e) => eprintln!("body decode error: {}", e), - } - } - Err(e) => { - eprintln!("http error: {}", e); - std::process::exit(1); - } - } -} diff --git a/registry/native/crates/commands/column/Cargo.toml b/registry/native/crates/commands/column/Cargo.toml deleted file mode 100644 index bca846e4db..0000000000 --- a/registry/native/crates/commands/column/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-column" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "column standalone binary for secure-exec VM" - -[[bin]] -name = "column" -path = "src/main.rs" - -[dependencies] -secureexec-column = { path = "../../libs/column" } diff --git a/registry/native/crates/commands/comm/Cargo.toml b/registry/native/crates/commands/comm/Cargo.toml deleted file mode 100644 index c748f132f1..0000000000 --- a/registry/native/crates/commands/comm/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-comm" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "comm standalone binary for secure-exec VM" - -[[bin]] -name = "comm" -path = "src/main.rs" - -[dependencies] -uu_comm = "0.7.0" diff --git a/registry/native/crates/commands/cp/Cargo.toml b/registry/native/crates/commands/cp/Cargo.toml deleted file mode 100644 index 851012ce6e..0000000000 --- a/registry/native/crates/commands/cp/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-cp" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "cp standalone binary for secure-exec VM" - -[[bin]] -name = "cp" -path = "src/main.rs" - -[dependencies] -uu_cp = "0.7.0" diff --git a/registry/native/crates/commands/curl/Cargo.toml b/registry/native/crates/commands/curl/Cargo.toml deleted file mode 100644 index 5d5a8a8763..0000000000 --- a/registry/native/crates/commands/curl/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-curl" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "Minimal curl-compatible HTTP client for secure-exec" - -[[bin]] -name = "curl" -path = "src/main.rs" - -[dependencies] -wasi-http = { package = "secureexec-wasi-http", path = "../../libs/wasi-http" } diff --git a/registry/native/crates/commands/cut/Cargo.toml b/registry/native/crates/commands/cut/Cargo.toml deleted file mode 100644 index 75e38c3add..0000000000 --- a/registry/native/crates/commands/cut/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-cut" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "cut standalone binary for secure-exec VM" - -[[bin]] -name = "cut" -path = "src/main.rs" - -[dependencies] -uu_cut = "0.7.0" diff --git a/registry/native/crates/commands/date/Cargo.toml b/registry/native/crates/commands/date/Cargo.toml deleted file mode 100644 index 54f324a982..0000000000 --- a/registry/native/crates/commands/date/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-date" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "date standalone binary for secure-exec VM" - -[[bin]] -name = "date" -path = "src/main.rs" - -[dependencies] -uu_date = "0.7.0" diff --git a/registry/native/crates/commands/dd/Cargo.toml b/registry/native/crates/commands/dd/Cargo.toml deleted file mode 100644 index 9f31bd6ea4..0000000000 --- a/registry/native/crates/commands/dd/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-dd" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "dd standalone binary for secure-exec VM" - -[[bin]] -name = "dd" -path = "src/main.rs" - -[dependencies] -uu_dd = "0.7.0" diff --git a/registry/native/crates/commands/diff/Cargo.toml b/registry/native/crates/commands/diff/Cargo.toml deleted file mode 100644 index ea9f00c8a2..0000000000 --- a/registry/native/crates/commands/diff/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-diff" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "diff standalone binary for secure-exec VM" - -[[bin]] -name = "diff" -path = "src/main.rs" - -[dependencies] -secureexec-diff = { path = "../../libs/diff" } diff --git a/registry/native/crates/commands/dircolors/Cargo.toml b/registry/native/crates/commands/dircolors/Cargo.toml deleted file mode 100644 index 2fb6f1a59d..0000000000 --- a/registry/native/crates/commands/dircolors/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-dircolors" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "dircolors standalone binary for secure-exec VM" - -[[bin]] -name = "dircolors" -path = "src/main.rs" - -[dependencies] -uu_dircolors = "0.7.0" diff --git a/registry/native/crates/commands/dirname/Cargo.toml b/registry/native/crates/commands/dirname/Cargo.toml deleted file mode 100644 index 26e18d7de7..0000000000 --- a/registry/native/crates/commands/dirname/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-dirname" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "dirname standalone binary for secure-exec VM" - -[[bin]] -name = "dirname" -path = "src/main.rs" - -[dependencies] -uu_dirname = "0.7.0" diff --git a/registry/native/crates/commands/du/Cargo.toml b/registry/native/crates/commands/du/Cargo.toml deleted file mode 100644 index b2101f3445..0000000000 --- a/registry/native/crates/commands/du/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-du" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "du standalone binary for secure-exec VM" - -[[bin]] -name = "du" -path = "src/main.rs" - -[dependencies] -secureexec-du = { path = "../../libs/du" } diff --git a/registry/native/crates/commands/echo/Cargo.toml b/registry/native/crates/commands/echo/Cargo.toml deleted file mode 100644 index 09e32de696..0000000000 --- a/registry/native/crates/commands/echo/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-echo" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "echo standalone binary for secure-exec VM" - -[[bin]] -name = "echo" -path = "src/main.rs" - -[dependencies] -uu_echo = "0.7.0" diff --git a/registry/native/crates/commands/env/Cargo.toml b/registry/native/crates/commands/env/Cargo.toml deleted file mode 100644 index f214d9ac47..0000000000 --- a/registry/native/crates/commands/env/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-env" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "env standalone binary for secure-exec VM" - -[[bin]] -name = "env" -path = "src/main.rs" - -[dependencies] -shims = { package = "secureexec-shims", path = "../../libs/shims" } diff --git a/registry/native/crates/commands/expand/Cargo.toml b/registry/native/crates/commands/expand/Cargo.toml deleted file mode 100644 index 9440c6bb9e..0000000000 --- a/registry/native/crates/commands/expand/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-expand" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "expand standalone binary for secure-exec VM" - -[[bin]] -name = "expand" -path = "src/main.rs" - -[dependencies] -uu_expand = "0.7.0" diff --git a/registry/native/crates/commands/expr/Cargo.toml b/registry/native/crates/commands/expr/Cargo.toml deleted file mode 100644 index 68acb90733..0000000000 --- a/registry/native/crates/commands/expr/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-expr" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "expr standalone binary for secure-exec VM" - -[[bin]] -name = "expr" -path = "src/main.rs" - -[dependencies] -secureexec-expr = { path = "../../libs/expr" } diff --git a/registry/native/crates/commands/factor/Cargo.toml b/registry/native/crates/commands/factor/Cargo.toml deleted file mode 100644 index 3ded9a0c20..0000000000 --- a/registry/native/crates/commands/factor/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-factor" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "factor standalone binary for secure-exec VM" - -[[bin]] -name = "factor" -path = "src/main.rs" - -[dependencies] -uu_factor = "0.7.0" diff --git a/registry/native/crates/commands/false/Cargo.toml b/registry/native/crates/commands/false/Cargo.toml deleted file mode 100644 index f30511a365..0000000000 --- a/registry/native/crates/commands/false/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-false" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "false standalone binary for secure-exec VM" - -[[bin]] -name = "false" -path = "src/main.rs" - -[dependencies] -uu_false = "0.7.0" diff --git a/registry/native/crates/commands/fd/Cargo.toml b/registry/native/crates/commands/fd/Cargo.toml deleted file mode 100644 index 12f78cdd30..0000000000 --- a/registry/native/crates/commands/fd/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-fd" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "fd standalone binary for secure-exec VM" - -[[bin]] -name = "fd" -path = "src/main.rs" - -[dependencies] -secureexec-fd = { path = "../../libs/fd" } diff --git a/registry/native/crates/commands/fd/src/main.rs b/registry/native/crates/commands/fd/src/main.rs deleted file mode 100644 index e74e129d80..0000000000 --- a/registry/native/crates/commands/fd/src/main.rs +++ /dev/null @@ -1,4 +0,0 @@ -fn main() { - let args: Vec = std::env::args_os().collect(); - std::process::exit(secureexec_fd::main(args)); -} diff --git a/registry/native/crates/commands/file/Cargo.toml b/registry/native/crates/commands/file/Cargo.toml deleted file mode 100644 index cabc723226..0000000000 --- a/registry/native/crates/commands/file/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-file" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "file standalone binary for secure-exec VM" - -[[bin]] -name = "file" -path = "src/main.rs" - -[dependencies] -secureexec-file-cmd = { path = "../../libs/file-cmd" } diff --git a/registry/native/crates/commands/find/Cargo.toml b/registry/native/crates/commands/find/Cargo.toml deleted file mode 100644 index 1b670a6fcf..0000000000 --- a/registry/native/crates/commands/find/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-find" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "find standalone binary for secure-exec VM" - -[[bin]] -name = "find" -path = "src/main.rs" - -[dependencies] -secureexec-find = { path = "../../libs/find" } diff --git a/registry/native/crates/commands/find/src/main.rs b/registry/native/crates/commands/find/src/main.rs deleted file mode 100644 index 5887b1a7a4..0000000000 --- a/registry/native/crates/commands/find/src/main.rs +++ /dev/null @@ -1,4 +0,0 @@ -fn main() { - let args: Vec = std::env::args_os().collect(); - std::process::exit(secureexec_find::main(args)); -} diff --git a/registry/native/crates/commands/fmt/Cargo.toml b/registry/native/crates/commands/fmt/Cargo.toml deleted file mode 100644 index cb2e5ae670..0000000000 --- a/registry/native/crates/commands/fmt/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-fmt" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "fmt standalone binary for secure-exec VM" - -[[bin]] -name = "fmt" -path = "src/main.rs" - -[dependencies] -uu_fmt = "0.7.0" diff --git a/registry/native/crates/commands/fold/Cargo.toml b/registry/native/crates/commands/fold/Cargo.toml deleted file mode 100644 index a8b067be43..0000000000 --- a/registry/native/crates/commands/fold/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-fold" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "fold standalone binary for secure-exec VM" - -[[bin]] -name = "fold" -path = "src/main.rs" - -[dependencies] -uu_fold = "0.7.0" diff --git a/registry/native/crates/commands/git/Cargo.toml b/registry/native/crates/commands/git/Cargo.toml deleted file mode 100644 index 3ae1427172..0000000000 --- a/registry/native/crates/commands/git/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-git" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "git standalone binary for secure-exec VM" - -[[bin]] -name = "git" -path = "src/main.rs" - -[dependencies] -secureexec-git = { path = "../../libs/git" } diff --git a/registry/native/crates/commands/git/src/main.rs b/registry/native/crates/commands/git/src/main.rs deleted file mode 100644 index 83886647f0..0000000000 --- a/registry/native/crates/commands/git/src/main.rs +++ /dev/null @@ -1,4 +0,0 @@ -fn main() { - let args: Vec = std::env::args_os().collect(); - std::process::exit(secureexec_git::main(args)); -} diff --git a/registry/native/crates/commands/grep/Cargo.toml b/registry/native/crates/commands/grep/Cargo.toml deleted file mode 100644 index fc3189a8d5..0000000000 --- a/registry/native/crates/commands/grep/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-grep" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "grep standalone binary for secure-exec VM" - -[[bin]] -name = "grep" -path = "src/main.rs" - -[dependencies] -secureexec-grep = { path = "../../libs/grep" } diff --git a/registry/native/crates/commands/grep/src/main.rs b/registry/native/crates/commands/grep/src/main.rs deleted file mode 100644 index ef8a789bdf..0000000000 --- a/registry/native/crates/commands/grep/src/main.rs +++ /dev/null @@ -1,13 +0,0 @@ -fn main() { - use std::io::Write; - - let args: Vec = std::env::args_os().collect(); - let mut code = secureexec_grep::main(args); - if let Err(error) = std::io::stdout().flush() { - eprintln!("Error flushing stdout: {error}"); - if code == 0 { - code = 2; - } - } - std::process::exit(code); -} diff --git a/registry/native/crates/commands/gzip/Cargo.toml b/registry/native/crates/commands/gzip/Cargo.toml deleted file mode 100644 index c6f1842eb4..0000000000 --- a/registry/native/crates/commands/gzip/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-gzip" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "gzip standalone binary for secure-exec VM" - -[[bin]] -name = "gzip" -path = "src/main.rs" - -[dependencies] -secureexec-gzip = { path = "../../libs/gzip" } diff --git a/registry/native/crates/commands/head/Cargo.toml b/registry/native/crates/commands/head/Cargo.toml deleted file mode 100644 index 49d35a6e1b..0000000000 --- a/registry/native/crates/commands/head/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-head" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "head standalone binary for secure-exec VM" - -[[bin]] -name = "head" -path = "src/main.rs" - -[dependencies] -uu_head = "0.7.0" diff --git a/registry/native/crates/commands/join/Cargo.toml b/registry/native/crates/commands/join/Cargo.toml deleted file mode 100644 index 408d1ceb4d..0000000000 --- a/registry/native/crates/commands/join/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-join" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "join standalone binary for secure-exec VM" - -[[bin]] -name = "join" -path = "src/main.rs" - -[dependencies] -uu_join = "0.7.0" diff --git a/registry/native/crates/commands/jq/Cargo.toml b/registry/native/crates/commands/jq/Cargo.toml deleted file mode 100644 index 31e46ae09f..0000000000 --- a/registry/native/crates/commands/jq/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-jq" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "jq standalone binary for secure-exec VM" - -[[bin]] -name = "jq" -path = "src/main.rs" - -[dependencies] -secureexec-jq = { path = "../../libs/jq" } diff --git a/registry/native/crates/commands/link/Cargo.toml b/registry/native/crates/commands/link/Cargo.toml deleted file mode 100644 index 79d9091369..0000000000 --- a/registry/native/crates/commands/link/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-link" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "link standalone binary for secure-exec VM" - -[[bin]] -name = "link" -path = "src/main.rs" - -[dependencies] -uu_link = "0.7.0" diff --git a/registry/native/crates/commands/ln/Cargo.toml b/registry/native/crates/commands/ln/Cargo.toml deleted file mode 100644 index ebb02bfe6e..0000000000 --- a/registry/native/crates/commands/ln/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-ln" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "ln standalone binary for secure-exec VM" - -[[bin]] -name = "ln" -path = "src/main.rs" - -[dependencies] -uu_ln = "0.7.0" diff --git a/registry/native/crates/commands/logname/Cargo.toml b/registry/native/crates/commands/logname/Cargo.toml deleted file mode 100644 index 6f7bbf5b76..0000000000 --- a/registry/native/crates/commands/logname/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-logname" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "logname standalone binary for secure-exec VM" - -[[bin]] -name = "logname" -path = "src/main.rs" - -[dependencies] -uu_logname = "0.7.0" diff --git a/registry/native/crates/commands/ls/Cargo.toml b/registry/native/crates/commands/ls/Cargo.toml deleted file mode 100644 index 025bfdfa03..0000000000 --- a/registry/native/crates/commands/ls/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-ls" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "ls standalone binary for secure-exec VM" - -[[bin]] -name = "ls" -path = "src/main.rs" - -[dependencies] -uu_ls = "0.7.0" diff --git a/registry/native/crates/commands/md5sum/Cargo.toml b/registry/native/crates/commands/md5sum/Cargo.toml deleted file mode 100644 index bd499cdc50..0000000000 --- a/registry/native/crates/commands/md5sum/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-md5sum" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "md5sum standalone binary for secure-exec VM" - -[[bin]] -name = "md5sum" -path = "src/main.rs" - -[dependencies] -uu_md5sum = "0.7.0" diff --git a/registry/native/crates/commands/mkdir/Cargo.toml b/registry/native/crates/commands/mkdir/Cargo.toml deleted file mode 100644 index 2e2cc2de94..0000000000 --- a/registry/native/crates/commands/mkdir/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-mkdir" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "mkdir standalone binary for secure-exec VM" - -[[bin]] -name = "mkdir" -path = "src/main.rs" - -[dependencies] -uu_mkdir = "0.7.0" diff --git a/registry/native/crates/commands/mktemp/Cargo.toml b/registry/native/crates/commands/mktemp/Cargo.toml deleted file mode 100644 index faa0c28810..0000000000 --- a/registry/native/crates/commands/mktemp/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-mktemp" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "mktemp standalone binary for secure-exec VM" - -[[bin]] -name = "mktemp" -path = "src/main.rs" - -[dependencies] -uu_mktemp = "0.7.0" diff --git a/registry/native/crates/commands/mv/Cargo.toml b/registry/native/crates/commands/mv/Cargo.toml deleted file mode 100644 index 3e52df17b6..0000000000 --- a/registry/native/crates/commands/mv/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-mv" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "mv standalone binary for secure-exec VM" - -[[bin]] -name = "mv" -path = "src/main.rs" - -[dependencies] -uu_mv = "0.7.0" diff --git a/registry/native/crates/commands/nice/Cargo.toml b/registry/native/crates/commands/nice/Cargo.toml deleted file mode 100644 index 5f00ac9f4b..0000000000 --- a/registry/native/crates/commands/nice/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-nice" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "nice standalone binary for secure-exec VM" - -[[bin]] -name = "nice" -path = "src/main.rs" - -[dependencies] -shims = { package = "secureexec-shims", path = "../../libs/shims" } diff --git a/registry/native/crates/commands/nl/Cargo.toml b/registry/native/crates/commands/nl/Cargo.toml deleted file mode 100644 index d1858d3be7..0000000000 --- a/registry/native/crates/commands/nl/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-nl" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "nl standalone binary for secure-exec VM" - -[[bin]] -name = "nl" -path = "src/main.rs" - -[dependencies] -uu_nl = "0.7.0" diff --git a/registry/native/crates/commands/nohup/Cargo.toml b/registry/native/crates/commands/nohup/Cargo.toml deleted file mode 100644 index ee2b24fa34..0000000000 --- a/registry/native/crates/commands/nohup/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-nohup" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "nohup standalone binary for secure-exec VM" - -[[bin]] -name = "nohup" -path = "src/main.rs" - -[dependencies] -shims = { package = "secureexec-shims", path = "../../libs/shims" } diff --git a/registry/native/crates/commands/nproc/Cargo.toml b/registry/native/crates/commands/nproc/Cargo.toml deleted file mode 100644 index 9f05c612cc..0000000000 --- a/registry/native/crates/commands/nproc/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-nproc" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "nproc standalone binary for secure-exec VM" - -[[bin]] -name = "nproc" -path = "src/main.rs" - -[dependencies] -uu_nproc = "0.7.0" diff --git a/registry/native/crates/commands/numfmt/Cargo.toml b/registry/native/crates/commands/numfmt/Cargo.toml deleted file mode 100644 index 18fae1f3a5..0000000000 --- a/registry/native/crates/commands/numfmt/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-numfmt" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "numfmt standalone binary for secure-exec VM" - -[[bin]] -name = "numfmt" -path = "src/main.rs" - -[dependencies] -uu_numfmt = "0.7.0" diff --git a/registry/native/crates/commands/od/Cargo.toml b/registry/native/crates/commands/od/Cargo.toml deleted file mode 100644 index 6e28c2db60..0000000000 --- a/registry/native/crates/commands/od/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-od" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "od standalone binary for secure-exec VM" - -[[bin]] -name = "od" -path = "src/main.rs" - -[dependencies] -uu_od = "0.7.0" diff --git a/registry/native/crates/commands/paste/Cargo.toml b/registry/native/crates/commands/paste/Cargo.toml deleted file mode 100644 index 87fb6f9c84..0000000000 --- a/registry/native/crates/commands/paste/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-paste" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "paste standalone binary for secure-exec VM" - -[[bin]] -name = "paste" -path = "src/main.rs" - -[dependencies] -uu_paste = "0.7.0" diff --git a/registry/native/crates/commands/pathchk/Cargo.toml b/registry/native/crates/commands/pathchk/Cargo.toml deleted file mode 100644 index a5bd0d8c9a..0000000000 --- a/registry/native/crates/commands/pathchk/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-pathchk" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "pathchk standalone binary for secure-exec VM" - -[[bin]] -name = "pathchk" -path = "src/main.rs" - -[dependencies] -uu_pathchk = "0.7.0" diff --git a/registry/native/crates/commands/printenv/Cargo.toml b/registry/native/crates/commands/printenv/Cargo.toml deleted file mode 100644 index 463418444e..0000000000 --- a/registry/native/crates/commands/printenv/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-printenv" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "printenv standalone binary for secure-exec VM" - -[[bin]] -name = "printenv" -path = "src/main.rs" - -[dependencies] -uu_printenv = "0.7.0" diff --git a/registry/native/crates/commands/printf/Cargo.toml b/registry/native/crates/commands/printf/Cargo.toml deleted file mode 100644 index 3ffe7fe58a..0000000000 --- a/registry/native/crates/commands/printf/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-printf" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "printf standalone binary for secure-exec VM" - -[[bin]] -name = "printf" -path = "src/main.rs" - -[dependencies] -uu_printf = "0.7.0" diff --git a/registry/native/crates/commands/ptx/Cargo.toml b/registry/native/crates/commands/ptx/Cargo.toml deleted file mode 100644 index 7176a63af2..0000000000 --- a/registry/native/crates/commands/ptx/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-ptx" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "ptx standalone binary for secure-exec VM" - -[[bin]] -name = "ptx" -path = "src/main.rs" - -[dependencies] -uu_ptx = "0.7.0" diff --git a/registry/native/crates/commands/pwd/Cargo.toml b/registry/native/crates/commands/pwd/Cargo.toml deleted file mode 100644 index d61413509a..0000000000 --- a/registry/native/crates/commands/pwd/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-pwd" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "pwd standalone binary for secure-exec VM" - -[[bin]] -name = "pwd" -path = "src/main.rs" - -[dependencies] -uu_pwd = "0.7.0" diff --git a/registry/native/crates/commands/readlink/Cargo.toml b/registry/native/crates/commands/readlink/Cargo.toml deleted file mode 100644 index e8c96fc77e..0000000000 --- a/registry/native/crates/commands/readlink/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-readlink" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "readlink standalone binary for secure-exec VM" - -[[bin]] -name = "readlink" -path = "src/main.rs" - -[dependencies] -uu_readlink = "0.7.0" diff --git a/registry/native/crates/commands/realpath/Cargo.toml b/registry/native/crates/commands/realpath/Cargo.toml deleted file mode 100644 index ad8fa957cf..0000000000 --- a/registry/native/crates/commands/realpath/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-realpath" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "realpath standalone binary for secure-exec VM" - -[[bin]] -name = "realpath" -path = "src/main.rs" - -[dependencies] -uu_realpath = "0.7.0" diff --git a/registry/native/crates/commands/rev/Cargo.toml b/registry/native/crates/commands/rev/Cargo.toml deleted file mode 100644 index d08020ac8b..0000000000 --- a/registry/native/crates/commands/rev/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-rev" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "rev standalone binary for secure-exec VM" - -[[bin]] -name = "rev" -path = "src/main.rs" - -[dependencies] -secureexec-rev = { path = "../../libs/rev" } diff --git a/registry/native/crates/commands/rg/Cargo.toml b/registry/native/crates/commands/rg/Cargo.toml deleted file mode 100644 index cd1ff4690e..0000000000 --- a/registry/native/crates/commands/rg/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-rg" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "rg (ripgrep) standalone binary for secure-exec VM" - -[[bin]] -name = "rg" -path = "src/main.rs" - -[dependencies] -secureexec-grep = { path = "../../libs/grep" } diff --git a/registry/native/crates/commands/rg/src/main.rs b/registry/native/crates/commands/rg/src/main.rs deleted file mode 100644 index e44c45f021..0000000000 --- a/registry/native/crates/commands/rg/src/main.rs +++ /dev/null @@ -1,4 +0,0 @@ -fn main() { - let args: Vec = std::env::args_os().collect(); - std::process::exit(secureexec_grep::rg(args)); -} diff --git a/registry/native/crates/commands/rm/Cargo.toml b/registry/native/crates/commands/rm/Cargo.toml deleted file mode 100644 index eb69989f9b..0000000000 --- a/registry/native/crates/commands/rm/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-rm" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "rm standalone binary for secure-exec VM" - -[[bin]] -name = "rm" -path = "src/main.rs" - -[dependencies] -uu_rm = "0.7.0" diff --git a/registry/native/crates/commands/rmdir/Cargo.toml b/registry/native/crates/commands/rmdir/Cargo.toml deleted file mode 100644 index 8773c2ec4c..0000000000 --- a/registry/native/crates/commands/rmdir/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-rmdir" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "rmdir standalone binary for secure-exec VM" - -[[bin]] -name = "rmdir" -path = "src/main.rs" - -[dependencies] -uu_rmdir = "0.7.0" diff --git a/registry/native/crates/commands/sed/Cargo.toml b/registry/native/crates/commands/sed/Cargo.toml deleted file mode 100644 index 9e006c3ba3..0000000000 --- a/registry/native/crates/commands/sed/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-sed" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "sed standalone binary for secure-exec VM" - -[[bin]] -name = "sed" -path = "src/main.rs" - -[dependencies] -sed = "0.1.1" diff --git a/registry/native/crates/commands/seq/Cargo.toml b/registry/native/crates/commands/seq/Cargo.toml deleted file mode 100644 index 83762f6eb9..0000000000 --- a/registry/native/crates/commands/seq/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-seq" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "seq standalone binary for secure-exec VM" - -[[bin]] -name = "seq" -path = "src/main.rs" - -[dependencies] -uu_seq = "0.7.0" diff --git a/registry/native/crates/commands/sh/Cargo.toml b/registry/native/crates/commands/sh/Cargo.toml deleted file mode 100644 index 3034c95973..0000000000 --- a/registry/native/crates/commands/sh/Cargo.toml +++ /dev/null @@ -1,18 +0,0 @@ -[package] -name = "cmd-sh" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "sh standalone binary for secure-exec VM" - -[[bin]] -name = "sh" -path = "src/main.rs" - -# `reedline` is enabled on ALL targets, including wasm: the vendored wasi -# patches supply what it needs (crossterm wasi backend, fd-lock, and -# brush-interactive's cfg(target_os = "wasi") alternatives to the -# tokio block_in_place paths), per the "behave like native Linux" rule. -# `minimal` stays available as an explicit opt-out backend. -[dependencies] -brush-shell = { version = "0.3.0", default-features = false, features = ["reedline", "minimal"] } diff --git a/registry/native/crates/commands/sha1sum/Cargo.toml b/registry/native/crates/commands/sha1sum/Cargo.toml deleted file mode 100644 index 8247916e83..0000000000 --- a/registry/native/crates/commands/sha1sum/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-sha1sum" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "sha1sum standalone binary for secure-exec VM" - -[[bin]] -name = "sha1sum" -path = "src/main.rs" - -[dependencies] -uu_sha1sum = "0.7.0" diff --git a/registry/native/crates/commands/sha224sum/Cargo.toml b/registry/native/crates/commands/sha224sum/Cargo.toml deleted file mode 100644 index 8f34c4db79..0000000000 --- a/registry/native/crates/commands/sha224sum/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-sha224sum" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "sha224sum standalone binary for secure-exec VM" - -[[bin]] -name = "sha224sum" -path = "src/main.rs" - -[dependencies] -uu_sha224sum = "0.7.0" diff --git a/registry/native/crates/commands/sha256sum/Cargo.toml b/registry/native/crates/commands/sha256sum/Cargo.toml deleted file mode 100644 index 8d4edba469..0000000000 --- a/registry/native/crates/commands/sha256sum/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-sha256sum" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "sha256sum standalone binary for secure-exec VM" - -[[bin]] -name = "sha256sum" -path = "src/main.rs" - -[dependencies] -uu_sha256sum = "0.7.0" diff --git a/registry/native/crates/commands/sha384sum/Cargo.toml b/registry/native/crates/commands/sha384sum/Cargo.toml deleted file mode 100644 index dc9fc1b53e..0000000000 --- a/registry/native/crates/commands/sha384sum/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-sha384sum" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "sha384sum standalone binary for secure-exec VM" - -[[bin]] -name = "sha384sum" -path = "src/main.rs" - -[dependencies] -uu_sha384sum = "0.7.0" diff --git a/registry/native/crates/commands/sha512sum/Cargo.toml b/registry/native/crates/commands/sha512sum/Cargo.toml deleted file mode 100644 index 6df4303347..0000000000 --- a/registry/native/crates/commands/sha512sum/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-sha512sum" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "sha512sum standalone binary for secure-exec VM" - -[[bin]] -name = "sha512sum" -path = "src/main.rs" - -[dependencies] -uu_sha512sum = "0.7.0" diff --git a/registry/native/crates/commands/shred/Cargo.toml b/registry/native/crates/commands/shred/Cargo.toml deleted file mode 100644 index 841e6f3057..0000000000 --- a/registry/native/crates/commands/shred/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-shred" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "shred standalone binary for secure-exec VM" - -[[bin]] -name = "shred" -path = "src/main.rs" - -[dependencies] -uu_shred = "0.7.0" diff --git a/registry/native/crates/commands/shuf/Cargo.toml b/registry/native/crates/commands/shuf/Cargo.toml deleted file mode 100644 index 0b9b499b18..0000000000 --- a/registry/native/crates/commands/shuf/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-shuf" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "shuf standalone binary for secure-exec VM" - -[[bin]] -name = "shuf" -path = "src/main.rs" - -[dependencies] -uu_shuf = "0.7.0" diff --git a/registry/native/crates/commands/sleep/Cargo.toml b/registry/native/crates/commands/sleep/Cargo.toml deleted file mode 100644 index 9e87d2649b..0000000000 --- a/registry/native/crates/commands/sleep/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-sleep" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "sleep standalone binary for secure-exec VM" - -[[bin]] -name = "sleep" -path = "src/main.rs" - -[dependencies] -secureexec-builtins = { path = "../../libs/builtins" } diff --git a/registry/native/crates/commands/sort/Cargo.toml b/registry/native/crates/commands/sort/Cargo.toml deleted file mode 100644 index 3ed6e1157f..0000000000 --- a/registry/native/crates/commands/sort/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-sort" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "sort standalone binary for secure-exec VM" - -[[bin]] -name = "sort" -path = "src/main.rs" - -[dependencies] -uu_sort = "0.7.0" diff --git a/registry/native/crates/commands/split/Cargo.toml b/registry/native/crates/commands/split/Cargo.toml deleted file mode 100644 index 76ca0362c6..0000000000 --- a/registry/native/crates/commands/split/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-split" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "split standalone binary for secure-exec VM" - -[[bin]] -name = "split" -path = "src/main.rs" - -[dependencies] -uu_split = "0.7.0" diff --git a/registry/native/crates/commands/stat/Cargo.toml b/registry/native/crates/commands/stat/Cargo.toml deleted file mode 100644 index d0587c9e33..0000000000 --- a/registry/native/crates/commands/stat/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-stat" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "stat standalone binary for secure-exec VM" - -[[bin]] -name = "stat" -path = "src/main.rs" - -[dependencies] -uu_stat = "0.7.0" diff --git a/registry/native/crates/commands/stdbuf/Cargo.toml b/registry/native/crates/commands/stdbuf/Cargo.toml deleted file mode 100644 index 96ecbb0a3f..0000000000 --- a/registry/native/crates/commands/stdbuf/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-stdbuf" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "stdbuf standalone binary for secure-exec VM" - -[[bin]] -name = "stdbuf" -path = "src/main.rs" - -[dependencies] -shims = { package = "secureexec-shims", path = "../../libs/shims" } diff --git a/registry/native/crates/commands/strings/Cargo.toml b/registry/native/crates/commands/strings/Cargo.toml deleted file mode 100644 index 7ce5f58a39..0000000000 --- a/registry/native/crates/commands/strings/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-strings" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "strings standalone binary for secure-exec VM" - -[[bin]] -name = "strings" -path = "src/main.rs" - -[dependencies] -secureexec-strings-cmd = { path = "../../libs/strings-cmd" } diff --git a/registry/native/crates/commands/sum/Cargo.toml b/registry/native/crates/commands/sum/Cargo.toml deleted file mode 100644 index 5fc71bfcdb..0000000000 --- a/registry/native/crates/commands/sum/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-sum" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "sum standalone binary for secure-exec VM" - -[[bin]] -name = "sum" -path = "src/main.rs" - -[dependencies] -uu_sum = "0.7.0" diff --git a/registry/native/crates/commands/tac/Cargo.toml b/registry/native/crates/commands/tac/Cargo.toml deleted file mode 100644 index c55250d7bd..0000000000 --- a/registry/native/crates/commands/tac/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-tac" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "tac standalone binary for secure-exec VM" - -[[bin]] -name = "tac" -path = "src/main.rs" - -[dependencies] -uu_tac = "0.7.0" diff --git a/registry/native/crates/commands/tail/Cargo.toml b/registry/native/crates/commands/tail/Cargo.toml deleted file mode 100644 index e47b9737ba..0000000000 --- a/registry/native/crates/commands/tail/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-tail" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "tail standalone binary for secure-exec VM" - -[[bin]] -name = "tail" -path = "src/main.rs" - -[dependencies] -uu_tail = "0.7.0" diff --git a/registry/native/crates/commands/tar/Cargo.toml b/registry/native/crates/commands/tar/Cargo.toml deleted file mode 100644 index d711b70205..0000000000 --- a/registry/native/crates/commands/tar/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-tar" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "tar standalone binary for secure-exec VM" - -[[bin]] -name = "tar" -path = "src/main.rs" - -[dependencies] -secureexec-tar = { path = "../../libs/tar" } diff --git a/registry/native/crates/commands/tee/Cargo.toml b/registry/native/crates/commands/tee/Cargo.toml deleted file mode 100644 index 94185e3cd8..0000000000 --- a/registry/native/crates/commands/tee/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-tee" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "tee standalone binary for secure-exec VM" - -[[bin]] -name = "tee" -path = "src/main.rs" - -[dependencies] -uu_tee = "0.7.0" diff --git a/registry/native/crates/commands/test/Cargo.toml b/registry/native/crates/commands/test/Cargo.toml deleted file mode 100644 index 9fe4f0f638..0000000000 --- a/registry/native/crates/commands/test/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-test" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "test/[ standalone binary for secure-exec VM" - -[[bin]] -name = "test" -path = "src/main.rs" - -[dependencies] -secureexec-builtins = { path = "../../libs/builtins" } diff --git a/registry/native/crates/commands/timeout/Cargo.toml b/registry/native/crates/commands/timeout/Cargo.toml deleted file mode 100644 index f872586c9f..0000000000 --- a/registry/native/crates/commands/timeout/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-timeout" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "timeout standalone binary for secure-exec VM" - -[[bin]] -name = "timeout" -path = "src/main.rs" - -[dependencies] -shims = { package = "secureexec-shims", path = "../../libs/shims" } diff --git a/registry/native/crates/commands/touch/Cargo.toml b/registry/native/crates/commands/touch/Cargo.toml deleted file mode 100644 index e88d117a95..0000000000 --- a/registry/native/crates/commands/touch/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-touch" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "touch standalone binary for secure-exec VM" - -[[bin]] -name = "touch" -path = "src/main.rs" - -[dependencies] -uu_touch = "0.7.0" diff --git a/registry/native/crates/commands/tr/Cargo.toml b/registry/native/crates/commands/tr/Cargo.toml deleted file mode 100644 index fcaa590eb7..0000000000 --- a/registry/native/crates/commands/tr/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-tr" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "tr standalone binary for secure-exec VM" - -[[bin]] -name = "tr" -path = "src/main.rs" - -[dependencies] -uu_tr = "0.7.0" diff --git a/registry/native/crates/commands/tree/Cargo.toml b/registry/native/crates/commands/tree/Cargo.toml deleted file mode 100644 index 73e0df78d5..0000000000 --- a/registry/native/crates/commands/tree/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-tree" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "tree standalone binary for secure-exec VM" - -[[bin]] -name = "tree" -path = "src/main.rs" - -[dependencies] -secureexec-tree = { path = "../../libs/tree" } diff --git a/registry/native/crates/commands/tree/src/main.rs b/registry/native/crates/commands/tree/src/main.rs deleted file mode 100644 index ee799a4287..0000000000 --- a/registry/native/crates/commands/tree/src/main.rs +++ /dev/null @@ -1,4 +0,0 @@ -fn main() { - let args: Vec = std::env::args_os().collect(); - std::process::exit(secureexec_tree::main(args)); -} diff --git a/registry/native/crates/commands/true/Cargo.toml b/registry/native/crates/commands/true/Cargo.toml deleted file mode 100644 index 95faf97085..0000000000 --- a/registry/native/crates/commands/true/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-true" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "true standalone binary for secure-exec VM" - -[[bin]] -name = "true" -path = "src/main.rs" - -[dependencies] -uu_true = "0.7.0" diff --git a/registry/native/crates/commands/truncate/Cargo.toml b/registry/native/crates/commands/truncate/Cargo.toml deleted file mode 100644 index 8a2ea8eef4..0000000000 --- a/registry/native/crates/commands/truncate/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-truncate" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "truncate standalone binary for secure-exec VM" - -[[bin]] -name = "truncate" -path = "src/main.rs" - -[dependencies] -uu_truncate = "0.7.0" diff --git a/registry/native/crates/commands/tsort/Cargo.toml b/registry/native/crates/commands/tsort/Cargo.toml deleted file mode 100644 index 9e17b0c29d..0000000000 --- a/registry/native/crates/commands/tsort/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-tsort" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "tsort standalone binary for secure-exec VM" - -[[bin]] -name = "tsort" -path = "src/main.rs" - -[dependencies] -uu_tsort = "0.7.0" diff --git a/registry/native/crates/commands/uname/Cargo.toml b/registry/native/crates/commands/uname/Cargo.toml deleted file mode 100644 index 4b3d810d0c..0000000000 --- a/registry/native/crates/commands/uname/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-uname" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "uname standalone binary for secure-exec VM" - -[[bin]] -name = "uname" -path = "src/main.rs" - -[dependencies] -uu_uname = "0.7.0" diff --git a/registry/native/crates/commands/unexpand/Cargo.toml b/registry/native/crates/commands/unexpand/Cargo.toml deleted file mode 100644 index 539079e2c1..0000000000 --- a/registry/native/crates/commands/unexpand/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-unexpand" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "unexpand standalone binary for secure-exec VM" - -[[bin]] -name = "unexpand" -path = "src/main.rs" - -[dependencies] -uu_unexpand = "0.7.0" diff --git a/registry/native/crates/commands/uniq/Cargo.toml b/registry/native/crates/commands/uniq/Cargo.toml deleted file mode 100644 index 39cef76b95..0000000000 --- a/registry/native/crates/commands/uniq/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-uniq" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "uniq standalone binary for secure-exec VM" - -[[bin]] -name = "uniq" -path = "src/main.rs" - -[dependencies] -uu_uniq = "0.7.0" diff --git a/registry/native/crates/commands/unlink/Cargo.toml b/registry/native/crates/commands/unlink/Cargo.toml deleted file mode 100644 index e1a4498758..0000000000 --- a/registry/native/crates/commands/unlink/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-unlink" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "unlink standalone binary for secure-exec VM" - -[[bin]] -name = "unlink" -path = "src/main.rs" - -[dependencies] -uu_unlink = "0.7.0" diff --git a/registry/native/crates/commands/wc/Cargo.toml b/registry/native/crates/commands/wc/Cargo.toml deleted file mode 100644 index d2d3a0a73b..0000000000 --- a/registry/native/crates/commands/wc/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-wc" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "wc standalone binary for secure-exec VM" - -[[bin]] -name = "wc" -path = "src/main.rs" - -[dependencies] -uu_wc = "0.7.0" diff --git a/registry/native/crates/commands/which/Cargo.toml b/registry/native/crates/commands/which/Cargo.toml deleted file mode 100644 index 0a6ae7d233..0000000000 --- a/registry/native/crates/commands/which/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-which" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "which standalone binary for secure-exec VM" - -[[bin]] -name = "which" -path = "src/main.rs" - -[dependencies] -shims = { package = "secureexec-shims", path = "../../libs/shims" } diff --git a/registry/native/crates/commands/whoami/Cargo.toml b/registry/native/crates/commands/whoami/Cargo.toml deleted file mode 100644 index 5aaf67d173..0000000000 --- a/registry/native/crates/commands/whoami/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-whoami" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "whoami standalone binary for secure-exec VM" - -[[bin]] -name = "whoami" -path = "src/main.rs" - -[dependencies] -secureexec-builtins = { path = "../../libs/builtins" } diff --git a/registry/native/crates/commands/xargs/Cargo.toml b/registry/native/crates/commands/xargs/Cargo.toml deleted file mode 100644 index 71a8b8cd6f..0000000000 --- a/registry/native/crates/commands/xargs/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-xargs" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "xargs standalone binary for secure-exec VM" - -[[bin]] -name = "xargs" -path = "src/main.rs" - -[dependencies] -shims = { package = "secureexec-shims", path = "../../libs/shims" } diff --git a/registry/native/crates/commands/xargs/src/main.rs b/registry/native/crates/commands/xargs/src/main.rs deleted file mode 100644 index 0ef4547696..0000000000 --- a/registry/native/crates/commands/xargs/src/main.rs +++ /dev/null @@ -1,4 +0,0 @@ -fn main() { - let args: Vec = std::env::args_os().collect(); - std::process::exit(shims::xargs::xargs(args)); -} diff --git a/registry/native/crates/commands/yes/Cargo.toml b/registry/native/crates/commands/yes/Cargo.toml deleted file mode 100644 index 4e6da247bd..0000000000 --- a/registry/native/crates/commands/yes/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-yes" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "yes standalone binary for secure-exec VM" - -[[bin]] -name = "yes" -path = "src/main.rs" - -[dependencies] -uu_yes = "0.7.0" diff --git a/registry/native/crates/commands/yq/Cargo.toml b/registry/native/crates/commands/yq/Cargo.toml deleted file mode 100644 index 77dadef52c..0000000000 --- a/registry/native/crates/commands/yq/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "cmd-yq" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "yq standalone binary for secure-exec VM" - -[[bin]] -name = "yq" -path = "src/main.rs" - -[dependencies] -secureexec-yq = { path = "../../libs/yq" } diff --git a/registry/native/crates/libs/awk/Cargo.toml b/registry/native/crates/libs/awk/Cargo.toml deleted file mode 100644 index 6de5ebc66f..0000000000 --- a/registry/native/crates/libs/awk/Cargo.toml +++ /dev/null @@ -1,9 +0,0 @@ -[package] -name = "secureexec-awk" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "awk implementation for secure-exec standalone binaries" - -[dependencies] -awk-rs = "0.1.0" diff --git a/registry/native/crates/libs/column/Cargo.toml b/registry/native/crates/libs/column/Cargo.toml deleted file mode 100644 index 0215159ee9..0000000000 --- a/registry/native/crates/libs/column/Cargo.toml +++ /dev/null @@ -1,8 +0,0 @@ -[package] -name = "secureexec-column" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "column implementation for secure-exec standalone binaries" - -[dependencies] diff --git a/registry/native/crates/libs/diff/Cargo.toml b/registry/native/crates/libs/diff/Cargo.toml deleted file mode 100644 index 614324b8ee..0000000000 --- a/registry/native/crates/libs/diff/Cargo.toml +++ /dev/null @@ -1,9 +0,0 @@ -[package] -name = "secureexec-diff" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "diff implementation for secure-exec standalone binaries" - -[dependencies] -similar = "2" diff --git a/registry/native/crates/libs/diff/src/lib.rs b/registry/native/crates/libs/diff/src/lib.rs deleted file mode 100644 index 04dbde97da..0000000000 --- a/registry/native/crates/libs/diff/src/lib.rs +++ /dev/null @@ -1,468 +0,0 @@ -//! diff -- compare files line by line using the `similar` crate - -use std::collections::HashSet; -use std::ffi::OsString; -use std::fs; -use std::io::{self, Write}; -use std::path::Path; - -use similar::{ChangeTag, TextDiff}; - -struct Options { - unified: bool, - context_fmt: bool, - context_lines: usize, - recursive: bool, - brief: bool, - ignore_case: bool, - ignore_whitespace: bool, - ignore_blank_lines: bool, -} - -impl Default for Options { - fn default() -> Self { - Self { - unified: false, - context_fmt: false, - context_lines: 3, - recursive: false, - brief: false, - ignore_case: false, - ignore_whitespace: false, - ignore_blank_lines: false, - } - } -} - -pub fn main(args: Vec) -> i32 { - let args: Vec = args - .iter() - .map(|a| a.to_string_lossy().to_string()) - .collect(); - let mut opts = Options::default(); - let mut files: Vec = Vec::new(); - let mut i = 1; - - while i < args.len() { - let arg = &args[i]; - match arg.as_str() { - "-u" | "--unified" => opts.unified = true, - "-c" | "--context" => opts.context_fmt = true, - "-r" | "-R" | "--recursive" => opts.recursive = true, - "-q" | "--brief" => opts.brief = true, - "-i" | "--ignore-case" => opts.ignore_case = true, - "-w" | "--ignore-all-space" => opts.ignore_whitespace = true, - "-B" | "--ignore-blank-lines" => opts.ignore_blank_lines = true, - s if s.starts_with("-U") => { - if let Ok(n) = s[2..].parse::() { - opts.unified = true; - opts.context_lines = n; - } - } - s if s.starts_with("-C") => { - if let Ok(n) = s[2..].parse::() { - opts.context_fmt = true; - opts.context_lines = n; - } - } - _ if !arg.starts_with('-') || arg == "-" => { - files.push(arg.clone()); - } - _ => { - eprintln!("diff: unknown option: {}", arg); - return 2; - } - } - i += 1; - } - - if files.len() != 2 { - eprintln!("diff: requires exactly two file or directory arguments"); - return 2; - } - - let path_a = Path::new(&files[0]); - let path_b = Path::new(&files[1]); - - let mut visited_dirs = HashSet::new(); - match diff_paths(path_a, path_b, &opts, &mut visited_dirs) { - Ok(has_diff) => { - if has_diff { - 1 - } else { - 0 - } - } - Err(e) => { - eprintln!("diff: {}", e); - 2 - } - } -} - -fn diff_paths( - path_a: &Path, - path_b: &Path, - opts: &Options, - visited_dirs: &mut HashSet<(std::path::PathBuf, std::path::PathBuf)>, -) -> Result { - let a_is_dir = path_a.is_dir(); - let b_is_dir = path_b.is_dir(); - - if a_is_dir && b_is_dir { - if opts.recursive { - diff_dirs(path_a, path_b, opts, visited_dirs) - } else { - Err(format!("{} is a directory", path_a.display())) - } - } else if a_is_dir || b_is_dir { - if a_is_dir { - let name = path_b.file_name().ok_or("invalid filename")?; - diff_files(&path_a.join(name), path_b, opts) - } else { - let name = path_a.file_name().ok_or("invalid filename")?; - diff_files(path_a, &path_b.join(name), opts) - } - } else { - diff_files(path_a, path_b, opts) - } -} - -fn diff_dirs( - dir_a: &Path, - dir_b: &Path, - opts: &Options, - visited_dirs: &mut HashSet<(std::path::PathBuf, std::path::PathBuf)>, -) -> Result { - let key = ( - fs::canonicalize(dir_a).map_err(|e| format!("{}: {}", dir_a.display(), e))?, - fs::canonicalize(dir_b).map_err(|e| format!("{}: {}", dir_b.display(), e))?, - ); - if !visited_dirs.insert(key) { - return Ok(false); - } - - let mut entries_a = list_dir(dir_a)?; - let mut entries_b = list_dir(dir_b)?; - entries_a.sort(); - entries_b.sort(); - - let mut all_names: Vec = Vec::new(); - for name in &entries_a { - all_names.push(name.clone()); - } - for name in &entries_b { - if !entries_a.contains(name) { - all_names.push(name.clone()); - } - } - all_names.sort(); - - let mut has_diff = false; - - for name in &all_names { - let pa = dir_a.join(name); - let pb = dir_b.join(name); - let a_exists = entries_a.contains(name); - let b_exists = entries_b.contains(name); - - if a_exists && !b_exists { - print_stdout_line(format_args!("Only in {}: {}", dir_a.display(), name))?; - has_diff = true; - } else if !a_exists && b_exists { - print_stdout_line(format_args!("Only in {}: {}", dir_b.display(), name))?; - has_diff = true; - } else { - match diff_paths(&pa, &pb, opts, visited_dirs) { - Ok(d) => { - if d { - has_diff = true; - } - } - Err(e) => { - return Err(e); - } - } - } - } - - Ok(has_diff) -} - -fn list_dir(dir: &Path) -> Result, String> { - let entries = fs::read_dir(dir).map_err(|e| format!("{}: {}", dir.display(), e))?; - let mut names = Vec::new(); - for entry in entries { - let entry = entry.map_err(|e| format!("{}: {}", dir.display(), e))?; - if let Some(name) = entry.file_name().to_str() { - names.push(name.to_string()); - } - } - Ok(names) -} - -fn preprocess(text: &str, opts: &Options) -> String { - let mut s = text.to_string(); - if opts.ignore_blank_lines { - let trailing = s.ends_with('\n'); - s = s - .lines() - .filter(|line| !line.trim().is_empty()) - .collect::>() - .join("\n"); - if trailing { - s.push('\n'); - } - } - if opts.ignore_case { - s = s.to_lowercase(); - } - if opts.ignore_whitespace { - let trailing = s.ends_with('\n'); - s = s - .lines() - .map(|line| { - let mut result = String::new(); - let mut in_ws = false; - for ch in line.chars() { - if ch.is_whitespace() { - if !in_ws { - result.push(' '); - in_ws = true; - } - } else { - result.push(ch); - in_ws = false; - } - } - let trimmed = result.trim(); - trimmed.to_string() - }) - .collect::>() - .join("\n"); - if trailing { - s.push('\n'); - } - } - s -} - -fn diff_files(path_a: &Path, path_b: &Path, opts: &Options) -> Result { - let text_a = read_file(path_a)?; - let text_b = read_file(path_b)?; - - // Check for differences (possibly with preprocessing) - let needs_pp = opts.ignore_case || opts.ignore_whitespace || opts.ignore_blank_lines; - let has_changes = if needs_pp { - preprocess(&text_a, opts) != preprocess(&text_b, opts) - } else { - text_a != text_b - }; - - if !has_changes { - return Ok(false); - } - - if opts.brief { - print_stdout_line(format_args!( - "Files {} and {} differ", - path_a.display(), - path_b.display() - ))?; - return Ok(true); - } - - // Diff the original text for output (all output logic inline to avoid lifetime issues) - let diff = TextDiff::from_lines(&text_a, &text_b); - let label_a = format!("{}", path_a.display()); - let label_b = format!("{}", path_b.display()); - let stdout = io::stdout(); - let mut out = stdout.lock(); - - if opts.unified { - writeln!(out, "--- {}", label_a).map_err(|e| format!("failed to write output: {e}"))?; - writeln!(out, "+++ {}", label_b).map_err(|e| format!("failed to write output: {e}"))?; - for hunk in diff - .unified_diff() - .context_radius(opts.context_lines) - .iter_hunks() - { - write!(out, "{}", hunk).map_err(|e| format!("failed to write output: {e}"))?; - } - } else if opts.context_fmt { - writeln!(out, "*** {}", label_a).map_err(|e| format!("failed to write output: {e}"))?; - writeln!(out, "--- {}", label_b).map_err(|e| format!("failed to write output: {e}"))?; - for hunk in diff - .unified_diff() - .context_radius(opts.context_lines) - .iter_hunks() - { - let mut old_lines: Vec<(ChangeTag, String)> = Vec::new(); - let mut new_lines: Vec<(ChangeTag, String)> = Vec::new(); - let mut old_start = 0usize; - let mut new_start = 0usize; - let mut first = true; - for change in hunk.iter_changes() { - if first { - old_start = change.old_index().unwrap_or(0) + 1; - new_start = change.new_index().unwrap_or(0) + 1; - first = false; - } - match change.tag() { - ChangeTag::Equal => { - old_lines.push((ChangeTag::Equal, change.value().to_string())); - new_lines.push((ChangeTag::Equal, change.value().to_string())); - } - ChangeTag::Delete => { - old_lines.push((ChangeTag::Delete, change.value().to_string())); - } - ChangeTag::Insert => { - new_lines.push((ChangeTag::Insert, change.value().to_string())); - } - } - } - let old_end = old_start + old_lines.len().saturating_sub(1); - let new_end = new_start + new_lines.len().saturating_sub(1); - writeln!(out, "***************").map_err(|e| format!("failed to write output: {e}"))?; - writeln!(out, "*** {},{} ****", old_start, old_end) - .map_err(|e| format!("failed to write output: {e}"))?; - for (tag, line) in &old_lines { - let prefix = match tag { - ChangeTag::Delete => "- ", - ChangeTag::Equal => " ", - _ => continue, - }; - write!(out, "{}{}", prefix, line) - .map_err(|e| format!("failed to write output: {e}"))?; - if !line.ends_with('\n') { - writeln!(out).map_err(|e| format!("failed to write output: {e}"))?; - } - } - writeln!(out, "--- {},{} ----", new_start, new_end) - .map_err(|e| format!("failed to write output: {e}"))?; - for (tag, line) in &new_lines { - let prefix = match tag { - ChangeTag::Insert => "+ ", - ChangeTag::Equal => " ", - _ => continue, - }; - write!(out, "{}{}", prefix, line) - .map_err(|e| format!("failed to write output: {e}"))?; - if !line.ends_with('\n') { - writeln!(out).map_err(|e| format!("failed to write output: {e}"))?; - } - } - } - } else { - // Normal diff format - let old_text: String = diff.old_slices().concat(); - let new_text: String = diff.new_slices().concat(); - let old_lines: Vec<&str> = old_text.lines().collect(); - let new_lines: Vec<&str> = new_text.lines().collect(); - - for op in diff.ops() { - match op { - similar::DiffOp::Equal { .. } => {} - similar::DiffOp::Delete { - old_index, - old_len, - new_index, - } => { - writeln!( - out, - "{}d{}", - format_range(*old_index + 1, *old_len), - new_index - ) - .map_err(|e| format!("failed to write output: {e}"))?; - for i in *old_index..*old_index + old_len { - if i < old_lines.len() { - writeln!(out, "< {}", old_lines[i]) - .map_err(|e| format!("failed to write output: {e}"))?; - } - } - } - similar::DiffOp::Insert { - old_index, - new_index, - new_len, - } => { - writeln!( - out, - "{}a{}", - old_index, - format_range(*new_index + 1, *new_len) - ) - .map_err(|e| format!("failed to write output: {e}"))?; - for i in *new_index..*new_index + new_len { - if i < new_lines.len() { - writeln!(out, "> {}", new_lines[i]) - .map_err(|e| format!("failed to write output: {e}"))?; - } - } - } - similar::DiffOp::Replace { - old_index, - old_len, - new_index, - new_len, - } => { - writeln!( - out, - "{}c{}", - format_range(*old_index + 1, *old_len), - format_range(*new_index + 1, *new_len) - ) - .map_err(|e| format!("failed to write output: {e}"))?; - for i in *old_index..*old_index + old_len { - if i < old_lines.len() { - writeln!(out, "< {}", old_lines[i]) - .map_err(|e| format!("failed to write output: {e}"))?; - } - } - writeln!(out, "---").map_err(|e| format!("failed to write output: {e}"))?; - for i in *new_index..*new_index + new_len { - if i < new_lines.len() { - writeln!(out, "> {}", new_lines[i]) - .map_err(|e| format!("failed to write output: {e}"))?; - } - } - } - } - } - } - - out.flush() - .map_err(|e| format!("failed to write output: {e}"))?; - - Ok(true) -} - -fn print_stdout_line(args: std::fmt::Arguments<'_>) -> Result<(), String> { - let stdout = io::stdout(); - let mut out = stdout.lock(); - writeln!(out, "{args}").map_err(|e| format!("failed to write output: {e}"))?; - out.flush() - .map_err(|e| format!("failed to write output: {e}")) -} - -fn read_file(path: &Path) -> Result { - if path.to_str() == Some("-") { - use std::io::Read; - let mut buf = String::new(); - io::stdin() - .read_to_string(&mut buf) - .map_err(|e| format!("stdin: {}", e))?; - Ok(buf) - } else { - fs::read_to_string(path).map_err(|e| format!("{}: {}", path.display(), e)) - } -} - -fn format_range(start: usize, len: usize) -> String { - if len == 1 { - format!("{}", start) - } else { - format!("{},{}", start, start + len - 1) - } -} diff --git a/registry/native/crates/libs/du/Cargo.toml b/registry/native/crates/libs/du/Cargo.toml deleted file mode 100644 index ad08de95a1..0000000000 --- a/registry/native/crates/libs/du/Cargo.toml +++ /dev/null @@ -1,8 +0,0 @@ -[package] -name = "secureexec-du" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "du implementation for secure-exec standalone binaries" - -[dependencies] diff --git a/registry/native/crates/libs/expr/Cargo.toml b/registry/native/crates/libs/expr/Cargo.toml deleted file mode 100644 index 8bd1f300fb..0000000000 --- a/registry/native/crates/libs/expr/Cargo.toml +++ /dev/null @@ -1,9 +0,0 @@ -[package] -name = "secureexec-expr" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "expr implementation for secure-exec standalone binaries" - -[dependencies] -regex = "1" diff --git a/registry/native/crates/libs/fd/Cargo.toml b/registry/native/crates/libs/fd/Cargo.toml deleted file mode 100644 index 2be330866f..0000000000 --- a/registry/native/crates/libs/fd/Cargo.toml +++ /dev/null @@ -1,9 +0,0 @@ -[package] -name = "secureexec-fd" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "fd-find compatible file finder for secure-exec standalone binaries" - -[dependencies] -regex = "1" diff --git a/registry/native/crates/libs/fd/src/lib.rs b/registry/native/crates/libs/fd/src/lib.rs deleted file mode 100644 index 8ab845214f..0000000000 --- a/registry/native/crates/libs/fd/src/lib.rs +++ /dev/null @@ -1,353 +0,0 @@ -//! Minimal fd-find compatible file finder for WASM. -//! -//! Uses std::fs for directory traversal (no walkdir/rayon dependency) -//! and regex for pattern matching. Covers common fd patterns: -//! fd PATTERN, fd -e EXT, fd -t f/d, fd -H (hidden), fd -I (no-ignore). - -use std::collections::HashSet; -use std::ffi::OsString; -use std::fs; -use std::io::{self, Write}; -use std::path::{Path, PathBuf}; - -use regex::Regex; - -/// Entry point for fd command. -pub fn main(args: Vec) -> i32 { - let str_args: Vec = args - .iter() - .skip(1) // skip argv[0] - .map(|a| a.to_string_lossy().to_string()) - .collect(); - - match run(&str_args) { - Ok(code) => code, - Err(msg) => { - eprintln!("[fd error]: {}", msg); - 1 - } - } -} - -#[derive(Debug, Clone, Copy, PartialEq)] -enum TypeFilter { - File, - Directory, - Symlink, -} - -enum ParsedArgs { - Search(Options), - Help, - Version, -} - -struct Options { - pattern: Option, - extensions: Vec, - type_filter: Option, - max_depth: Option, - search_paths: Vec, - show_hidden: bool, - _case_insensitive: bool, - full_path: bool, - absolute_path: bool, -} - -fn run(args: &[String]) -> Result { - let parsed = parse_args(args)?; - let stdout = io::stdout(); - let mut out = stdout.lock(); - - let ParsedArgs::Search(opts) = parsed else { - match parsed { - ParsedArgs::Help => print_help(&mut out), - ParsedArgs::Version => writeln!(out, "fd 0.1.0 (secure-exec)"), - ParsedArgs::Search(_) => unreachable!(), - } - .map_err(|e| format!("failed to write output: {e}"))?; - out.flush() - .map_err(|e| format!("failed to write output: {e}"))?; - return Ok(0); - }; - - let mut found = false; - - for search_path in &opts.search_paths { - let base = PathBuf::from(search_path); - let mut visited_dirs = HashSet::new(); - walk( - &base, - search_path, - 0, - &opts, - &mut found, - &mut out, - &mut visited_dirs, - )?; - } - - out.flush() - .map_err(|e| format!("failed to write output: {e}"))?; - - Ok(if found { 0 } else { 1 }) -} - -fn walk( - full_path: &Path, - base_path: &str, - depth: usize, - opts: &Options, - found: &mut bool, - out: &mut W, - visited_dirs: &mut HashSet, -) -> Result<(), String> { - if let Some(max) = opts.max_depth { - if depth > max { - return Ok(()); - } - } - - // fd skips the root search directory itself (depth 0); only matches children - if depth > 0 { - if matches_entry(full_path, base_path, opts) { - *found = true; - let display = if opts.absolute_path { - match fs::canonicalize(full_path) { - Ok(abs) => abs.to_string_lossy().to_string(), - Err(_) => full_path.to_string_lossy().to_string(), - } - } else { - full_path.to_string_lossy().to_string() - }; - writeln!(out, "{}", display).map_err(|e| format!("failed to write output: {e}"))?; - } - } - - // Recurse into directories - if full_path.is_dir() { - let canonical_path = - fs::canonicalize(full_path).map_err(|e| format!("{}: {}", full_path.display(), e))?; - if !visited_dirs.insert(canonical_path) { - return Ok(()); - } - - if let Some(max) = opts.max_depth { - if depth >= max { - return Ok(()); - } - } - - let entries = - fs::read_dir(full_path).map_err(|e| format!("{}: {}", full_path.display(), e))?; - - let mut sorted: Vec<_> = entries - .collect::, _>>() - .map_err(|e| format!("{}: {}", full_path.display(), e))?; - sorted.sort_by_key(|e| e.file_name()); - - for entry in sorted { - let name = entry.file_name().to_string_lossy().to_string(); - - // Skip hidden files/directories unless -H is set - if !opts.show_hidden && name.starts_with('.') { - continue; - } - - let child = entry.path(); - walk(&child, base_path, depth + 1, opts, found, out, visited_dirs)?; - } - } - - Ok(()) -} - -fn matches_entry(path: &Path, _base_path: &str, opts: &Options) -> bool { - // Type filter - if let Some(ref tf) = opts.type_filter { - let ok = match tf { - TypeFilter::File => path.is_file(), - TypeFilter::Directory => path.is_dir(), - TypeFilter::Symlink => path - .symlink_metadata() - .map(|m| m.file_type().is_symlink()) - .unwrap_or(false), - }; - if !ok { - return false; - } - } - - // Extension filter - if !opts.extensions.is_empty() { - let ext = path - .extension() - .map(|e| e.to_string_lossy().to_string()) - .unwrap_or_default(); - let ext_lower = ext.to_lowercase(); - if !opts - .extensions - .iter() - .any(|e| e.to_lowercase() == ext_lower) - { - return false; - } - } - - // Pattern match - if let Some(ref re) = opts.pattern { - let target = if opts.full_path { - path.to_string_lossy().to_string() - } else { - path.file_name() - .map(|n| n.to_string_lossy().to_string()) - .unwrap_or_default() - }; - if !re.is_match(&target) { - return false; - } - } - - true -} - -fn parse_args(args: &[String]) -> Result { - let mut pattern: Option = None; - let mut extensions: Vec = Vec::new(); - let mut type_filter: Option = None; - let mut max_depth: Option = None; - let mut search_paths: Vec = Vec::new(); - let mut show_hidden = false; - let mut case_insensitive = false; - let mut full_path = false; - let mut absolute_path = false; - let mut i = 0; - - while i < args.len() { - let arg = &args[i]; - match arg.as_str() { - "-h" | "--help" => { - return Ok(ParsedArgs::Help); - } - "-V" | "--version" => { - return Ok(ParsedArgs::Version); - } - "-H" | "--hidden" => { - show_hidden = true; - } - "-I" | "--no-ignore" => { - // No .gitignore support in WASM VFS — this is a no-op - } - "-i" | "--ignore-case" => { - case_insensitive = true; - } - "-s" | "--case-sensitive" => { - case_insensitive = false; - } - "-p" | "--full-path" => { - full_path = true; - } - "-a" | "--absolute-path" => { - absolute_path = true; - } - "-e" | "--extension" => { - i += 1; - if i >= args.len() { - return Err("-e/--extension requires an argument".to_string()); - } - extensions.push(args[i].clone()); - } - "-t" | "--type" => { - i += 1; - if i >= args.len() { - return Err("-t/--type requires an argument".to_string()); - } - type_filter = Some(match args[i].as_str() { - "f" | "file" => TypeFilter::File, - "d" | "directory" => TypeFilter::Directory, - "l" | "symlink" => TypeFilter::Symlink, - other => return Err(format!("unknown type filter: {}", other)), - }); - } - "-d" | "--max-depth" => { - i += 1; - if i >= args.len() { - return Err("-d/--max-depth requires an argument".to_string()); - } - max_depth = Some( - args[i] - .parse() - .map_err(|_| format!("invalid max-depth: {}", args[i]))?, - ); - } - arg if arg.starts_with('-') => { - return Err(format!("unknown option: {}", arg)); - } - _ => { - // First positional arg is pattern, rest are search paths - if pattern.is_none() { - pattern = Some(arg.clone()); - } else { - search_paths.push(arg.clone()); - } - } - } - i += 1; - } - - if search_paths.is_empty() { - search_paths.push(".".to_string()); - } - - // Compile pattern regex - let compiled_pattern = match pattern { - Some(pat) => { - let re_str = if case_insensitive { - format!("(?i){}", pat) - } else { - pat - }; - Some(Regex::new(&re_str).map_err(|e| format!("invalid pattern: {}", e))?) - } - None => None, - }; - - Ok(ParsedArgs::Search(Options { - pattern: compiled_pattern, - extensions, - type_filter, - max_depth, - search_paths, - show_hidden, - _case_insensitive: case_insensitive, - full_path, - absolute_path, - })) -} - -fn print_help(out: &mut W) -> io::Result<()> { - writeln!( - out, - "fd - a simple, fast file finder - -USAGE: - fd [OPTIONS] [PATTERN] [PATH...] - -ARGS: - Regex search pattern - ... Search paths (default: current directory) - -OPTIONS: - -H, --hidden Include hidden files/directories - -I, --no-ignore Do not respect .gitignore (no-op in sandbox) - -i, --ignore-case Case-insensitive search - -s, --case-sensitive Case-sensitive search (default) - -p, --full-path Match pattern against full path - -a, --absolute-path Show absolute paths - -e, --extension EXT Filter by file extension - -t, --type TYPE Filter by type: f(ile), d(irectory), l(ink) - -d, --max-depth N Maximum search depth - -h, --help Print help - -V, --version Print version" - ) -} diff --git a/registry/native/crates/libs/file-cmd/Cargo.toml b/registry/native/crates/libs/file-cmd/Cargo.toml deleted file mode 100644 index 7f14225fd9..0000000000 --- a/registry/native/crates/libs/file-cmd/Cargo.toml +++ /dev/null @@ -1,9 +0,0 @@ -[package] -name = "secureexec-file-cmd" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "file command implementation for secure-exec standalone binaries" - -[dependencies] -infer = "0.16" diff --git a/registry/native/crates/libs/file-cmd/src/lib.rs b/registry/native/crates/libs/file-cmd/src/lib.rs deleted file mode 100644 index 92a9faa647..0000000000 --- a/registry/native/crates/libs/file-cmd/src/lib.rs +++ /dev/null @@ -1,336 +0,0 @@ -//! file -- determine file type -//! -//! Magic byte detection using the `infer` crate, plus text/binary heuristic. -//! Supports -b (brief, no filename prefix), -i (output MIME type). - -use std::ffi::OsString; -use std::fs; -use std::io::{self, Read, Write}; - -const DETECTION_BYTES: usize = 8192; - -pub fn main(args: Vec) -> i32 { - let str_args: Vec = args - .iter() - .skip(1) - .map(|a| a.to_string_lossy().to_string()) - .collect(); - - let mut brief = false; - let mut mime = false; - let mut filenames: Vec = Vec::new(); - - let mut i = 0; - while i < str_args.len() { - match str_args[i].as_str() { - "-b" | "--brief" => brief = true, - "-i" | "--mime-type" | "--mime" => mime = true, - // Combined short flags - s if s.starts_with('-') && s.len() > 1 && !s.starts_with("--") => { - for ch in s[1..].chars() { - match ch { - 'b' => brief = true, - 'i' => mime = true, - _ => { - eprintln!("file: unknown option '{}'", s); - return 1; - } - } - } - } - "-" => { - // Read from stdin - filenames.push("-".to_string()); - } - s if s.starts_with('-') && s.len() > 1 => { - eprintln!("file: unknown option '{}'", s); - return 1; - } - _ => filenames.push(str_args[i].clone()), - } - i += 1; - } - - if filenames.is_empty() { - eprintln!("Usage: file [-bi] FILE..."); - return 1; - } - - let stdout = io::stdout(); - let mut out = stdout.lock(); - let mut exit_code = 0; - - for filename in &filenames { - let data = if filename == "-" { - match read_head_from_reader(io::stdin().lock(), DETECTION_BYTES) { - Ok(data) => data, - Err(e) => { - eprintln!("file: stdin: {}", e); - exit_code = 1; - continue; - } - } - } else { - // Check if path exists and what kind it is - match fs::metadata(filename) { - Ok(meta) => { - if meta.is_dir() { - if let Err(error) = print_result( - &mut out, - filename, - brief, - "directory", - if mime { "inode/directory" } else { "" }, - mime, - ) { - return output_error(error); - } - continue; - } - if meta.is_symlink() { - if let Err(error) = print_result( - &mut out, - filename, - brief, - "symbolic link", - if mime { "inode/symlink" } else { "" }, - mime, - ) { - return output_error(error); - } - continue; - } - if meta.len() == 0 { - if let Err(error) = print_result( - &mut out, - filename, - brief, - "empty", - if mime { "inode/x-empty" } else { "" }, - mime, - ) { - return output_error(error); - } - continue; - } - // Read file data (up to 8KB for detection) - match read_head(filename, DETECTION_BYTES) { - Ok(data) => data, - Err(e) => { - eprintln!("file: {}: {}", filename, e); - exit_code = 1; - continue; - } - } - } - Err(e) => { - eprintln!("file: {}: {}", filename, e); - exit_code = 1; - continue; - } - } - }; - - let (desc, mime_type) = identify(&data); - - if mime { - if let Err(error) = print_result(&mut out, filename, brief, &desc, &mime_type, true) { - return output_error(error); - } - } else { - if let Err(error) = print_result(&mut out, filename, brief, &desc, &mime_type, false) { - return output_error(error); - } - } - } - - if let Err(error) = out.flush() { - return output_error(error); - } - - exit_code -} - -fn output_error(error: io::Error) -> i32 { - eprintln!("file: failed to write output: {error}"); - 1 -} - -fn read_head(path: &str, max: usize) -> io::Result> { - let mut f = fs::File::open(path)?; - read_head_from_reader(&mut f, max) -} - -fn read_head_from_reader(mut reader: impl Read, max: usize) -> io::Result> { - let mut buf = vec![0u8; max]; - let n = reader.read(&mut buf)?; - buf.truncate(n); - Ok(buf) -} - -fn print_result( - out: &mut W, - filename: &str, - brief: bool, - desc: &str, - mime_type: &str, - use_mime: bool, -) -> io::Result<()> { - if use_mime && !mime_type.is_empty() { - if brief { - writeln!(out, "{}", mime_type) - } else { - writeln!(out, "{}: {}", filename, mime_type) - } - } else if brief { - writeln!(out, "{}", desc) - } else { - writeln!(out, "{}: {}", filename, desc) - } -} - -fn identify(data: &[u8]) -> (String, String) { - if data.is_empty() { - return ("empty".to_string(), "inode/x-empty".to_string()); - } - - // Try infer crate for magic byte detection - if let Some(kind) = infer::get(data) { - let desc = match kind.mime_type() { - "image/png" => "PNG image data", - "image/jpeg" => "JPEG image data", - "image/gif" => "GIF image data", - "image/webp" => "WebP image data", - "image/bmp" => "BMP image data", - "image/tiff" => "TIFF image data", - "image/svg+xml" => "SVG image data", - "image/x-icon" => "ICO image data", - "application/pdf" => "PDF document", - "application/zip" => "Zip archive data", - "application/gzip" => "gzip compressed data", - "application/x-bzip2" => "bzip2 compressed data", - "application/x-xz" => "XZ compressed data", - "application/x-tar" => "POSIX tar archive", - "application/x-rar-compressed" => "RAR archive data", - "application/x-7z-compressed" => "7-zip archive data", - "application/x-executable" | "application/x-elf" => "ELF executable", - "application/wasm" => "WebAssembly (wasm) binary module", - "application/x-mach-binary" => "Mach-O binary", - "audio/mpeg" => "MPEG audio", - "audio/ogg" => "Ogg audio", - "audio/flac" => "FLAC audio", - "audio/wav" | "audio/x-wav" => "RIFF WAVE audio", - "video/mp4" => "MPEG-4 video", - "video/webm" => "WebM video", - "video/x-matroska" => "Matroska video", - "application/x-sqlite3" => "SQLite 3.x database", - "font/woff" => "Web Open Font Format", - "font/woff2" => "Web Open Font Format 2", - _ => kind.mime_type(), - }; - return (desc.to_string(), kind.mime_type().to_string()); - } - - // Check for shebang scripts - if data.len() >= 2 && data[0] == b'#' && data[1] == b'!' { - let first_line = data - .iter() - .take(128) - .take_while(|&&b| b != b'\n') - .copied() - .collect::>(); - if let Ok(line) = std::str::from_utf8(&first_line) { - let interp = line.trim_start_matches("#!"); - let interp = interp.trim(); - // Extract interpreter name - let name = interp.split_whitespace().next().unwrap_or(interp); - let basename = match name.rfind('/') { - Some(pos) => &name[pos + 1..], - None => name, - }; - // Handle "env " pattern - if basename == "env" { - let prog = interp.split_whitespace().nth(1).unwrap_or("script"); - return ( - format!("{} script, ASCII text executable", prog), - "text/x-script".to_string(), - ); - } - return ( - format!("{} script, ASCII text executable", basename), - "text/x-script".to_string(), - ); - } - } - - // Check for known text patterns - if is_json(data) { - return ("JSON text data".to_string(), "application/json".to_string()); - } - if is_xml(data) { - return ("XML document".to_string(), "text/xml".to_string()); - } - if is_html(data) { - return ("HTML document".to_string(), "text/html".to_string()); - } - - // Text vs binary heuristic - if is_text(data) { - return ("ASCII text".to_string(), "text/plain".to_string()); - } - - ("data".to_string(), "application/octet-stream".to_string()) -} - -fn is_text(data: &[u8]) -> bool { - // Check first 8KB for non-text bytes - let check_len = data.len().min(8192); - let mut null_count = 0; - for &b in &data[..check_len] { - if b == 0 { - null_count += 1; - if null_count > 0 { - return false; - } - } - // Allow common control chars: tab, newline, carriage return, form feed, backspace, escape - if b < 0x08 || (b > 0x0D && b < 0x20 && b != 0x1B) { - if b != 0x00 { - // already counted nulls - return false; - } - } - } - true -} - -fn is_json(data: &[u8]) -> bool { - // Quick check: starts with { or [, ignoring leading whitespace - let trimmed = skip_whitespace(data); - !trimmed.is_empty() && (trimmed[0] == b'{' || trimmed[0] == b'[') -} - -fn is_xml(data: &[u8]) -> bool { - let trimmed = skip_whitespace(data); - trimmed.starts_with(b" bool { - let trimmed = skip_whitespace(data); - let lower: Vec = trimmed - .iter() - .take(64) - .map(|b| b.to_ascii_lowercase()) - .collect(); - lower.starts_with(b" &[u8] { - let mut i = 0; - while i < data.len() - && (data[i] == b' ' || data[i] == b'\t' || data[i] == b'\n' || data[i] == b'\r') - { - i += 1; - } - &data[i..] -} diff --git a/registry/native/crates/libs/find/Cargo.toml b/registry/native/crates/libs/find/Cargo.toml deleted file mode 100644 index 1790edb5e8..0000000000 --- a/registry/native/crates/libs/find/Cargo.toml +++ /dev/null @@ -1,9 +0,0 @@ -[package] -name = "secureexec-find" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "find implementation for secure-exec standalone binaries" - -[dependencies] -regex = "1" diff --git a/registry/native/crates/libs/find/src/lib.rs b/registry/native/crates/libs/find/src/lib.rs deleted file mode 100644 index 059e80fa53..0000000000 --- a/registry/native/crates/libs/find/src/lib.rs +++ /dev/null @@ -1,593 +0,0 @@ -//! POSIX find implementation for WASM. -//! -//! Custom implementation using std::fs (WASI) for directory traversal -//! and glob-to-regex conversion for pattern matching. fd-find crate -//! cannot be used directly as it's a binary crate with platform-specific -//! dependencies incompatible with wasm32-wasip1. - -use std::collections::HashSet; -use std::ffi::OsString; -use std::fs; -use std::io::{self, Write}; -use std::path::{Path, PathBuf}; - -use regex::Regex; - -/// Entry point for find command. -pub fn main(args: Vec) -> i32 { - let str_args: Vec = args - .iter() - .skip(1) // skip argv[0] - .map(|a| a.to_string_lossy().to_string()) - .collect(); - - match run_find(&str_args) { - Ok(found) => { - if found { - 0 - } else { - 0 - } // find returns 0 even if no matches - } - Err(msg) => { - eprintln!("find: {}", msg); - 1 - } - } -} - -/// Parsed find expression node. -#[derive(Debug)] -enum Expr { - /// -name PATTERN (glob match on filename only) - Name(Regex), - /// -iname PATTERN (case-insensitive glob match on filename) - IName(Regex), - /// -path PATTERN (glob match on full path) - PathMatch(Regex), - /// -ipath PATTERN (case-insensitive glob match on full path) - IPathMatch(Regex), - /// -type TYPE (d, f, l) - Type(FileType), - /// -empty (file size 0 or empty directory) - Empty, - /// -maxdepth N (handled specially during traversal) - MaxDepth(usize), - /// -mindepth N (handled specially during traversal) - MinDepth(usize), - /// -print (explicit print action) - Print, - /// -not EXPR / ! EXPR - Not(Box), - /// EXPR -a EXPR / EXPR -and EXPR / implicit AND - And(Box, Box), - /// EXPR -o EXPR / EXPR -or EXPR - Or(Box, Box), -} - -#[derive(Debug, Clone, Copy, PartialEq)] -enum FileType { - Directory, - File, - Symlink, -} - -struct FindOptions { - paths: Vec, - expr: Option, - max_depth: Option, - min_depth: usize, -} - -fn run_find(args: &[String]) -> Result { - let opts = parse_args(args)?; - let mut found_any = false; - let stdout = io::stdout(); - let mut out = stdout.lock(); - - for path in &opts.paths { - let mut visited_dirs = HashSet::new(); - walk( - &PathBuf::from(path), - path, - 0, - &opts, - &mut found_any, - &mut out, - &mut visited_dirs, - )?; - } - - out.flush() - .map_err(|e| format!("failed to write output: {e}"))?; - - Ok(found_any) -} - -/// Recursive directory walk. -fn walk( - full_path: &Path, - display_path: &str, - depth: usize, - opts: &FindOptions, - found_any: &mut bool, - out: &mut W, - visited_dirs: &mut HashSet, -) -> Result<(), String> { - // Check max depth before processing - if let Some(max) = opts.max_depth { - if depth > max { - return Ok(()); - } - } - - // Evaluate expression against this entry (if at or below min_depth) - if depth >= opts.min_depth { - let matches = match &opts.expr { - Some(expr) => eval_expr(expr, full_path, display_path), - None => true, // no expression means match everything - }; - - if matches { - *found_any = true; - // Default action is print - writeln!(out, "{}", display_path) - .map_err(|e| format!("failed to write output: {e}"))?; - } - } - - // Recurse into directories - if full_path.is_dir() { - // Check max depth for children - if let Some(max) = opts.max_depth { - if depth >= max { - return Ok(()); - } - } - - let canonical_path = - fs::canonicalize(full_path).map_err(|e| format!("{}: {}", display_path, e))?; - if !visited_dirs.insert(canonical_path.clone()) { - return Ok(()); - } - - let entries = fs::read_dir(full_path).map_err(|e| format!("{}: {}", display_path, e))?; - - let mut sorted_entries: Vec<_> = entries - .collect::, _>>() - .map_err(|e| format!("{}: {}", display_path, e))?; - sorted_entries.sort_by_key(|e| e.file_name()); - - for entry in sorted_entries { - let child_name = entry.file_name().to_string_lossy().to_string(); - let child_display = if display_path == "/" { - format!("/{}", child_name) - } else { - format!("{}/{}", display_path, child_name) - }; - let child_full = entry.path(); - - walk( - &child_full, - &child_display, - depth + 1, - opts, - found_any, - out, - visited_dirs, - )?; - } - visited_dirs.remove(&canonical_path); - } - - Ok(()) -} - -/// Evaluate an expression against a path. -fn eval_expr(expr: &Expr, full_path: &Path, display_path: &str) -> bool { - match expr { - Expr::Name(re) => { - let name = full_path - .file_name() - .map(|n| n.to_string_lossy().to_string()) - .unwrap_or_default(); - re.is_match(&name) - } - Expr::IName(re) => { - let name = full_path - .file_name() - .map(|n| n.to_string_lossy().to_string()) - .unwrap_or_default(); - re.is_match(&name) - } - Expr::PathMatch(re) => re.is_match(display_path), - Expr::IPathMatch(re) => re.is_match(display_path), - Expr::Type(ft) => match ft { - FileType::Directory => full_path.is_dir(), - FileType::File => full_path.is_file(), - FileType::Symlink => full_path - .symlink_metadata() - .map(|m| m.file_type().is_symlink()) - .unwrap_or(false), - }, - Expr::Empty => { - if full_path.is_dir() { - fs::read_dir(full_path) - .map(|mut d| d.next().is_none()) - .unwrap_or(false) - } else if full_path.is_file() { - full_path.metadata().map(|m| m.len() == 0).unwrap_or(false) - } else { - false - } - } - Expr::MaxDepth(_) | Expr::MinDepth(_) => true, // handled in walk() - Expr::Print => true, // print is an action, always "matches" - Expr::Not(inner) => !eval_expr(inner, full_path, display_path), - Expr::And(left, right) => { - eval_expr(left, full_path, display_path) && eval_expr(right, full_path, display_path) - } - Expr::Or(left, right) => { - eval_expr(left, full_path, display_path) || eval_expr(right, full_path, display_path) - } - } -} - -/// Convert a shell glob pattern to a regex pattern. -/// Supports *, ?, [charset], and ** (for path matching). -fn glob_to_regex(glob: &str, case_insensitive: bool) -> Result { - let mut re = String::from("^"); - let chars: Vec = glob.chars().collect(); - let mut i = 0; - - while i < chars.len() { - match chars[i] { - '*' => { - if i + 1 < chars.len() && chars[i + 1] == '*' { - // ** matches everything including / - re.push_str(".*"); - i += 1; - } else { - // * matches everything except / - re.push_str("[^/]*"); - } - } - '?' => re.push_str("[^/]"), - '[' => { - re.push('['); - i += 1; - if i < chars.len() && chars[i] == '!' { - re.push('^'); - i += 1; - } else if i < chars.len() && chars[i] == '^' { - re.push('^'); - i += 1; - } - while i < chars.len() && chars[i] != ']' { - if chars[i] == '\\' && i + 1 < chars.len() { - re.push('\\'); - i += 1; - re.push(chars[i]); - } else { - re.push(chars[i]); - } - i += 1; - } - re.push(']'); - } - '.' | '+' | '(' | ')' | '{' | '}' | '|' | '^' | '$' | '\\' => { - re.push('\\'); - re.push(chars[i]); - } - _ => re.push(chars[i]), - } - i += 1; - } - - re.push('$'); - - let pattern = if case_insensitive { - format!("(?i){}", re) - } else { - re - }; - - Regex::new(&pattern).map_err(|e| format!("invalid pattern: {}", e)) -} - -/// Parse find command arguments. -fn parse_args(args: &[String]) -> Result { - let mut paths: Vec = Vec::new(); - let mut exprs: Vec = Vec::new(); - let mut max_depth: Option = None; - let mut min_depth: usize = 0; - let mut i = 0; - - // First, collect paths (arguments before any expression) - while i < args.len() { - let arg = &args[i]; - if arg.starts_with('-') || arg == "!" || arg == "(" { - break; - } - paths.push(arg.clone()); - i += 1; - } - - if paths.is_empty() { - paths.push(".".to_string()); - } - - // Parse expressions - while i < args.len() { - let arg = &args[i]; - - match arg.as_str() { - "-name" => { - i += 1; - if i >= args.len() { - return Err("-name requires an argument".to_string()); - } - let re = glob_to_regex(&args[i], false)?; - exprs.push(Expr::Name(re)); - } - "-iname" => { - i += 1; - if i >= args.len() { - return Err("-iname requires an argument".to_string()); - } - let re = glob_to_regex(&args[i], true)?; - exprs.push(Expr::IName(re)); - } - "-path" | "-wholename" => { - i += 1; - if i >= args.len() { - return Err(format!("{} requires an argument", arg)); - } - let re = glob_to_regex(&args[i], false)?; - exprs.push(Expr::PathMatch(re)); - } - "-ipath" | "-iwholename" => { - i += 1; - if i >= args.len() { - return Err(format!("{} requires an argument", arg)); - } - let re = glob_to_regex(&args[i], true)?; - exprs.push(Expr::IPathMatch(re)); - } - "-type" => { - i += 1; - if i >= args.len() { - return Err("-type requires an argument".to_string()); - } - let ft = match args[i].as_str() { - "d" => FileType::Directory, - "f" => FileType::File, - "l" => FileType::Symlink, - other => return Err(format!("unknown file type: {}", other)), - }; - exprs.push(Expr::Type(ft)); - } - "-empty" => { - exprs.push(Expr::Empty); - } - "-maxdepth" => { - i += 1; - if i >= args.len() { - return Err("-maxdepth requires an argument".to_string()); - } - let n: usize = args[i] - .parse() - .map_err(|_| format!("-maxdepth: {}: not a number", args[i]))?; - max_depth = Some(n); - } - "-mindepth" => { - i += 1; - if i >= args.len() { - return Err("-mindepth requires an argument".to_string()); - } - let n: usize = args[i] - .parse() - .map_err(|_| format!("-mindepth: {}: not a number", args[i]))?; - min_depth = n; - } - "-print" => { - exprs.push(Expr::Print); - } - "!" | "-not" => { - i += 1; - if i >= args.len() { - return Err("! requires an expression".to_string()); - } - // Parse the next single expression - let (next_expr, next_i) = parse_single_expr(&args, i)?; - if let Some((md, mi)) = extract_depth(&next_expr) { - if let Some(d) = md { - max_depth = Some(d); - } - min_depth = mi.unwrap_or(min_depth); - } - exprs.push(Expr::Not(Box::new(next_expr))); - i = next_i; - continue; // skip the i += 1 at end - } - "-a" | "-and" => { - // Implicit AND between consecutive expressions, -a is explicit - // Just skip it; AND is applied when combining exprs - } - "-o" | "-or" => { - // Combine everything so far as left side of OR - if exprs.is_empty() { - return Err("-or requires a preceding expression".to_string()); - } - let left = combine_and(exprs); - - // Parse remaining as right side - i += 1; - let mut right_exprs = Vec::new(); - while i < args.len() { - let (e, ni) = parse_single_expr(args, i)?; - if let Some((md, mi)) = extract_depth(&e) { - if let Some(d) = md { - max_depth = Some(d); - } - min_depth = mi.unwrap_or(min_depth); - } - right_exprs.push(e); - i = ni; - } - let right = combine_and(right_exprs); - exprs = vec![Expr::Or(Box::new(left), Box::new(right))]; - break; - } - "(" => { - // Grouped expression - find matching ) - i += 1; - let mut depth = 1; - let start = i; - while i < args.len() && depth > 0 { - if args[i] == "(" { - depth += 1; - } - if args[i] == ")" { - depth -= 1; - } - if depth > 0 { - i += 1; - } - } - if depth != 0 { - return Err("unmatched '('".to_string()); - } - let sub_args: Vec = args[start..i].to_vec(); - let sub_opts = parse_args_exprs(&sub_args)?; - if let Some(e) = sub_opts { - exprs.push(e); - } - } - other if other.starts_with('-') => { - return Err(format!("unknown option: {}", other)); - } - _ => { - // Treat as path if no expressions yet, otherwise error - return Err(format!("unexpected argument: {}", arg)); - } - } - i += 1; - } - - let expr = if exprs.is_empty() { - None - } else { - Some(combine_and(exprs)) - }; - - Ok(FindOptions { - paths, - expr, - max_depth, - min_depth, - }) -} - -/// Parse a single expression at position i, returning the expression and the next position. -fn parse_single_expr(args: &[String], i: usize) -> Result<(Expr, usize), String> { - if i >= args.len() { - return Err("unexpected end of expression".to_string()); - } - - let arg = &args[i]; - match arg.as_str() { - "-name" => { - if i + 1 >= args.len() { - return Err("-name requires an argument".to_string()); - } - let re = glob_to_regex(&args[i + 1], false)?; - Ok((Expr::Name(re), i + 2)) - } - "-iname" => { - if i + 1 >= args.len() { - return Err("-iname requires an argument".to_string()); - } - let re = glob_to_regex(&args[i + 1], true)?; - Ok((Expr::IName(re), i + 2)) - } - "-path" | "-wholename" => { - if i + 1 >= args.len() { - return Err(format!("{} requires an argument", arg)); - } - let re = glob_to_regex(&args[i + 1], false)?; - Ok((Expr::PathMatch(re), i + 2)) - } - "-type" => { - if i + 1 >= args.len() { - return Err("-type requires an argument".to_string()); - } - let ft = match args[i + 1].as_str() { - "d" => FileType::Directory, - "f" => FileType::File, - "l" => FileType::Symlink, - other => return Err(format!("unknown file type: {}", other)), - }; - Ok((Expr::Type(ft), i + 2)) - } - "-empty" => Ok((Expr::Empty, i + 1)), - "-maxdepth" => { - if i + 1 >= args.len() { - return Err("-maxdepth requires an argument".to_string()); - } - let n: usize = args[i + 1] - .parse() - .map_err(|_| format!("-maxdepth: {}: not a number", args[i + 1]))?; - Ok((Expr::MaxDepth(n), i + 2)) - } - "-mindepth" => { - if i + 1 >= args.len() { - return Err("-mindepth requires an argument".to_string()); - } - let n: usize = args[i + 1] - .parse() - .map_err(|_| format!("-mindepth: {}: not a number", args[i + 1]))?; - Ok((Expr::MinDepth(n), i + 2)) - } - "-print" => Ok((Expr::Print, i + 1)), - _ => Err(format!("unknown expression: {}", arg)), - } -} - -/// Parse expression-only args (for grouped expressions). -fn parse_args_exprs(args: &[String]) -> Result, String> { - let mut exprs = Vec::new(); - let mut i = 0; - - while i < args.len() { - let (expr, next_i) = parse_single_expr(args, i)?; - exprs.push(expr); - i = next_i; - } - - if exprs.is_empty() { - Ok(None) - } else { - Ok(Some(combine_and(exprs))) - } -} - -/// Extract depth settings from an expression. -fn extract_depth(expr: &Expr) -> Option<(Option, Option)> { - match expr { - Expr::MaxDepth(n) => Some((Some(*n), None)), - Expr::MinDepth(n) => Some((None, Some(*n))), - _ => None, - } -} - -/// Combine a list of expressions with AND. -fn combine_and(mut exprs: Vec) -> Expr { - if exprs.len() == 1 { - return exprs.remove(0); - } - let first = exprs.remove(0); - exprs - .into_iter() - .fold(first, |acc, e| Expr::And(Box::new(acc), Box::new(e))) -} diff --git a/registry/native/crates/libs/git/Cargo.toml b/registry/native/crates/libs/git/Cargo.toml deleted file mode 100644 index 59d150be52..0000000000 --- a/registry/native/crates/libs/git/Cargo.toml +++ /dev/null @@ -1,15 +0,0 @@ -[package] -name = "secureexec-git" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "Minimal git implementation for secure-exec VM" - -[lib] -name = "secureexec_git" -path = "src/lib.rs" - -[dependencies] -sha1 = "0.10.6" -flate2 = "1.1" -wasi-http = { package = "secureexec-wasi-http", path = "../wasi-http" } diff --git a/registry/native/crates/libs/git/README.md b/registry/native/crates/libs/git/README.md deleted file mode 100644 index 26fcb37043..0000000000 --- a/registry/native/crates/libs/git/README.md +++ /dev/null @@ -1,36 +0,0 @@ -# secure-exec VM git compatibility - -The `git` command in secure-exec is a clean-room, Apache-2.0 reimplementation of a -small git subset for WasmVM guests. - -## Supported commands - -- `git init` -- `git add` -- `git commit -m ...` -- `git rev-parse ` -- `git branch` -- `git checkout` -- `git clone ` -- `git clone http://...` -- `git clone https://...` - -Remote clone support is limited to unauthenticated smart-HTTP fetches. The -implementation can read from `http://` and `https://` remotes, but it does not -support credentials, push, or SSH transport. - -## Unsupported commands and transports - -The guest command exits with a typed `GitSubcommandUnsupported` error for: - -- `git push` -- `git fetch` -- `git pull` -- `git remote` -- SSH-style clone URLs such as `git@host:owner/repo.git` and `ssh://...` -- `git://...` clone URLs -- Authenticated HTTP(S) remotes such as `https://user@host/repo.git` -- Submodules, hooks, GPG signing, and other advanced porcelain workflows - -If you need behavior outside this subset, keep using host git or extend the -clean-room implementation deliberately. diff --git a/registry/native/crates/libs/git/src/lib.rs b/registry/native/crates/libs/git/src/lib.rs deleted file mode 100644 index bd32b5520f..0000000000 --- a/registry/native/crates/libs/git/src/lib.rs +++ /dev/null @@ -1,2127 +0,0 @@ -//! Minimal clean-room git implementation (Apache-2.0). -//! -//! Supports: init, add, commit, branch, checkout (with DWIM), local clone, -//! and smart-HTTP clone over http:// and https://. - -use flate2::bufread::ZlibDecoder as BufZlibDecoder; -use flate2::read::ZlibDecoder; -use flate2::write::ZlibEncoder; -use flate2::Compression; -use sha1::{Digest, Sha1}; -use std::collections::{BTreeMap, HashMap, HashSet}; -use std::ffi::{OsStr, OsString}; -use std::fmt; -use std::fs; -use std::io::{self, Cursor, Read, Write}; -use std::path::{Component, Path, PathBuf}; -use wasi_http::{HttpClient, Method, Request}; - -// ─── Hex utilities ────────────────────────────────────────────────────────── - -fn hex(bytes: &[u8]) -> String { - bytes.iter().map(|b| format!("{:02x}", b)).collect() -} - -fn unhex(s: &str) -> io::Result<[u8; 20]> { - if s.len() != 40 { - return Err(err(&format!("invalid hash length: {}", s.len()))); - } - let bytes: Vec = (0..s.len()) - .step_by(2) - .map(|i| { - u8::from_str_radix(&s[i..i + 2], 16).map_err(|e| err(&format!("invalid hex: {}", e))) - }) - .collect::>>()?; - let mut hash = [0u8; 20]; - hash.copy_from_slice(&bytes); - Ok(hash) -} - -fn err(msg: &str) -> io::Error { - io::Error::new(io::ErrorKind::Other, msg.to_string()) -} - -fn print_stdout_line(args: fmt::Arguments<'_>) -> io::Result<()> { - let mut stdout = io::stdout().lock(); - stdout.write_fmt(args)?; - stdout.write_all(b"\n")?; - stdout.flush() -} - -const SUPPORT_DOC_PATH: &str = "registry/native/crates/libs/git/README.md"; -const MAX_GIT_OBJECT_BYTES: usize = 128 * 1024 * 1024; -const MAX_GIT_OBJECT_HEADER_BYTES: usize = 128; -const MAX_INDEX_ENTRIES: usize = 1_000_000; -const MAX_INDEX_BYTES: usize = 256 * 1024 * 1024; -const MAX_ADVERTISED_REFS: usize = 100_000; -const MAX_HTTP_ERROR_BODY_BYTES: usize = 8192; -const MAX_PKT_LINES: usize = 100_000; -const MAX_PKT_LINE_PAYLOAD_BYTES: usize = 16 * 1024 * 1024; -const MAX_REF_ADVERTISEMENT_BYTES: usize = 16 * 1024 * 1024; -const MAX_PACK_INFLATED_BYTES: usize = 256 * 1024 * 1024; -const MAX_PACK_BYTES: usize = 256 * 1024 * 1024; -const MAX_PACK_OBJECTS: usize = 1_000_000; -const MAX_PACK_RESOLVED_BYTES: usize = 256 * 1024 * 1024; - -fn unsupported(subcommand: &str, detail: &str) -> io::Error { - err(&format!( - "GitSubcommandUnsupported: git {} {} See {} for supported transports and commands.", - subcommand, detail, SUPPORT_DOC_PATH - )) -} - -/// WASI-safe mkdir -p: create_dir_all has issues with WASI permission checks -/// on already-existing directories, so we create one level at a time. -/// We also treat PermissionDenied as "already exists" because some WASI runtimes -/// return EACCES instead of EEXIST for existing directories. -fn mkdirs(path: &Path) -> io::Result<()> { - let mut ancestors: Vec<&Path> = path.ancestors().collect(); - ancestors.reverse(); // root first - for dir in ancestors { - if dir == Path::new("/") || dir == Path::new("") { - continue; - } - match fs::create_dir(dir) { - Ok(()) => {} - Err(e) if e.kind() == io::ErrorKind::AlreadyExists => {} - // WASI kernels may return EACCES for existing directories - Err(e) if e.kind() == io::ErrorKind::PermissionDenied => { - // Verify it actually exists; propagate if it truly doesn't - if !dir.exists() { - return Err(e); - } - } - Err(e) => return Err(e), - } - } - Ok(()) -} - -fn dir_is_empty(path: &Path) -> io::Result { - if !path.exists() { - return Ok(true); - } - if !path.is_dir() { - return Ok(false); - } - Ok(fs::read_dir(path)?.next().is_none()) -} - -fn read_to_end_limited(reader: R, limit: usize, context: &str) -> io::Result> { - let limit_plus_one = limit - .checked_add(1) - .ok_or_else(|| err(&format!("{} size limit is too large", context)))?; - let mut out = Vec::new(); - reader.take(limit_plus_one as u64).read_to_end(&mut out)?; - if out.len() > limit { - return Err(err(&format!("{} exceeds size limit", context))); - } - Ok(out) -} - -fn read_file_limited(path: &Path, limit: usize, context: &str) -> io::Result> { - let metadata = fs::metadata(path).map_err(|e| { - err(&format!( - "cannot stat {} '{}': {}", - context, - path.display(), - e - )) - })?; - if metadata.len() > limit as u64 { - return Err(err(&format!( - "{} '{}' exceeds size limit", - context, - path.display() - ))); - } - fs::read(path).map_err(|e| { - err(&format!( - "cannot read {} '{}': {}", - context, - path.display(), - e - )) - }) -} - -fn try_reserve_exact(vec: &mut Vec, additional: usize, context: &str) -> io::Result<()> { - vec.try_reserve_exact(additional) - .map_err(|_| err(&format!("{} allocation failed", context))) -} - -fn append_pack_bytes(pack: &mut Vec, bytes: &[u8]) -> io::Result<()> { - let new_len = pack - .len() - .checked_add(bytes.len()) - .ok_or_else(|| err("packfile size overflow"))?; - if new_len > MAX_PACK_BYTES { - return Err(err("packfile exceeds size limit")); - } - try_reserve_exact(pack, bytes.len(), "packfile")?; - pack.extend_from_slice(bytes); - Ok(()) -} - -fn add_pack_inflated_bytes(total: &mut usize, bytes: usize) -> io::Result<()> { - *total = total - .checked_add(bytes) - .ok_or_else(|| err("inflated pack size overflow"))?; - if *total > MAX_PACK_INFLATED_BYTES { - return Err(err("inflated pack data exceeds size limit")); - } - Ok(()) -} - -fn add_pack_resolved_bytes(total: &mut usize, bytes: usize) -> io::Result<()> { - *total = total - .checked_add(bytes) - .ok_or_else(|| err("resolved pack size overflow"))?; - if *total > MAX_PACK_RESOLVED_BYTES { - return Err(err("resolved pack data exceeds size limit")); - } - Ok(()) -} - -fn add_pkt_payload_bytes(total: &mut usize, bytes: usize) -> io::Result<()> { - *total = total - .checked_add(bytes) - .ok_or_else(|| err("pkt-line payload size overflow"))?; - if *total > MAX_PKT_LINE_PAYLOAD_BYTES { - return Err(err("pkt-line payload exceeds size limit")); - } - Ok(()) -} - -fn add_advertised_ref_count(total: usize) -> io::Result<()> { - if total > MAX_ADVERTISED_REFS { - return Err(err("remote advertised too many refs")); - } - Ok(()) -} - -fn response_body_preview(body: &[u8]) -> String { - let limit = body.len().min(MAX_HTTP_ERROR_BODY_BYTES); - let mut preview = String::from_utf8_lossy(&body[..limit]).trim().to_string(); - if body.len() > limit { - preview.push_str("..."); - } - preview -} - -fn normalize_repo_path(path: &str) -> io::Result { - if path.is_empty() || path.as_bytes().contains(&0) { - return Err(err("invalid empty or nul-containing repository path")); - } - - let mut parts = Vec::new(); - for component in Path::new(path).components() { - match component { - Component::Normal(part) if part == OsStr::new(".git") => { - return Err(err("repository path must not enter .git")); - } - Component::Normal(part) => { - let part = part - .to_str() - .ok_or_else(|| err("repository path must be utf-8"))?; - if part.is_empty() { - return Err(err("repository path contains an empty component")); - } - parts.push(part); - } - Component::CurDir - | Component::ParentDir - | Component::RootDir - | Component::Prefix(_) => { - return Err(err("repository path must be relative and normalized")); - } - } - } - - if parts.is_empty() { - return Err(err("repository path must name a file")); - } - - Ok(parts.join("/")) -} - -fn worktree_path(workdir: &Path, repo_path: &str) -> io::Result { - Ok(workdir.join(normalize_repo_path(repo_path)?)) -} - -fn validate_ref_tail(name: &str) -> io::Result<&str> { - if name.is_empty() - || name.as_bytes().contains(&0) - || name.starts_with('/') - || name.ends_with('/') - || name.contains('\\') - { - return Err(err("invalid git ref name")); - } - - for part in name.split('/') { - if part.is_empty() || part == "." || part == ".." { - return Err(err("invalid git ref name")); - } - } - - Ok(name) -} - -fn validate_branch_name(name: &str) -> io::Result<&str> { - validate_ref_tail(name) -} - -fn validate_refname(refname: &str) -> io::Result<&str> { - if refname == "HEAD" { - return Ok(refname); - } - - for prefix in ["refs/heads/", "refs/remotes/", "refs/tags/"] { - if let Some(tail) = refname.strip_prefix(prefix) { - validate_ref_tail(tail)?; - return Ok(refname); - } - } - - Err(err("unsupported git ref name")) -} - -fn hash_bytes(obj_type: &str, data: &[u8]) -> [u8; 20] { - let header = format!("{} {}\0", obj_type, data.len()); - let mut hasher = Sha1::new(); - hasher.update(header.as_bytes()); - hasher.update(data); - hasher.finalize().into() -} - -fn is_zero_oid(hash: &[u8; 20]) -> bool { - hash.iter().all(|b| *b == 0) -} - -fn is_remote_source(source: &str) -> bool { - source.starts_with("http://") || source.starts_with("https://") -} - -fn is_ssh_clone_source(source: &str) -> bool { - source.starts_with("git@") - || source.starts_with("ssh://") - || source.starts_with("git://") - || source.starts_with("ssh+git://") -} - -fn has_http_auth(source: &str) -> bool { - if !is_remote_source(source) { - return false; - } - - let Some(scheme_sep) = source.find("://") else { - return false; - }; - let authority = &source[scheme_sep + 3..] - .split_once('/') - .map(|(authority, _)| authority) - .unwrap_or(&source[scheme_sep + 3..]); - - authority.contains('@') -} - -fn infer_clone_destination(source: &str) -> io::Result { - let basename = if is_remote_source(source) || is_ssh_clone_source(source) { - let trimmed = source.trim_end_matches('/'); - let leaf = if let Some((_, scp_path)) = trimmed.rsplit_once(':') { - if is_ssh_clone_source(source) && !scp_path.contains('/') { - scp_path - } else { - trimmed - .rsplit('/') - .next() - .filter(|segment| !segment.is_empty()) - .ok_or_else(|| err("could not determine destination path from source"))? - } - } else { - trimmed - .rsplit('/') - .next() - .filter(|segment| !segment.is_empty()) - .ok_or_else(|| err("could not determine destination path from source"))? - }; - leaf.to_string() - } else { - Path::new(source) - .file_name() - .map(|name| name.to_string_lossy().to_string()) - .filter(|name| !name.is_empty()) - .ok_or_else(|| err("could not determine destination path from source"))? - }; - - Ok(basename - .strip_suffix(".git") - .unwrap_or(&basename) - .to_string()) -} - -fn prepare_clone_destination( - dest: &Path, - source_label: &str, - default_branch: &str, -) -> io::Result { - validate_branch_name(default_branch)?; - mkdirs(dest)?; - - let dst_git = dest.join(".git"); - mkdirs(&dst_git.join("objects"))?; - mkdirs(&dst_git.join("refs/heads"))?; - mkdirs(&dst_git.join("refs/tags"))?; - mkdirs(&dst_git.join("refs/remotes/origin"))?; - - fs::write( - dst_git.join("HEAD"), - format!("ref: refs/heads/{}\n", default_branch), - )?; - fs::write( - dst_git.join("config"), - format!( - "[core]\n\trepositoryformatversion = 0\n\tfilemode = true\n\tbare = false\n\ - [remote \"origin\"]\n\turl = {}\n\tfetch = +refs/heads/*:refs/remotes/origin/*\n", - source_label - ), - )?; - - Ok(dst_git) -} - -#[derive(Debug)] -enum PktLine { - Data(Vec), - Flush, -} - -fn parse_pkt_lines(data: &[u8]) -> io::Result> { - let mut pos = 0usize; - let mut lines = Vec::new(); - let mut payload_bytes = 0usize; - - while pos + 4 <= data.len() { - if lines.len() >= MAX_PKT_LINES { - return Err(err("too many pkt-lines")); - } - let len_str = - std::str::from_utf8(&data[pos..pos + 4]).map_err(|_| err("invalid pkt-line"))?; - let len = usize::from_str_radix(len_str, 16).map_err(|_| err("invalid pkt-line length"))?; - pos += 4; - - if len == 0 { - lines.push(PktLine::Flush); - continue; - } - if len < 4 || pos + len - 4 > data.len() { - return Err(err("truncated pkt-line")); - } - - let payload_len = len - 4; - add_pkt_payload_bytes(&mut payload_bytes, payload_len)?; - lines.push(PktLine::Data(data[pos..pos + payload_len].to_vec())); - pos += len - 4; - } - - if pos != data.len() { - return Err(err("trailing bytes after pkt-lines")); - } - - Ok(lines) -} - -#[derive(Debug, Default)] -struct RemoteAdvertisement { - head_hash: Option<[u8; 20]>, - head_target: Option, - branches: BTreeMap, - tags: BTreeMap, -} - -fn parse_advertised_ref(line: &[u8], adv: &mut RemoteAdvertisement) -> io::Result<()> { - let (ref_part, capability_part) = if let Some(nul) = line.iter().position(|b| *b == 0) { - (&line[..nul], Some(&line[nul + 1..])) - } else { - (line, None) - }; - - let ref_text = std::str::from_utf8(ref_part) - .map_err(|_| err("invalid advertised ref"))? - .trim_end_matches('\n'); - if ref_text.is_empty() { - return Ok(()); - } - - let (hash_hex, refname) = ref_text - .split_once(' ') - .ok_or_else(|| err("malformed advertised ref"))?; - let hash = unhex(hash_hex)?; - - if refname == "HEAD" { - adv.head_hash = Some(hash); - } else if refname.starts_with("refs/heads/") { - validate_refname(refname)?; - adv.branches.insert(refname.to_string(), hash); - add_advertised_ref_count(adv.branches.len() + adv.tags.len())?; - } else if refname.starts_with("refs/tags/") { - validate_refname(refname)?; - adv.tags.insert(refname.to_string(), hash); - add_advertised_ref_count(adv.branches.len() + adv.tags.len())?; - } - - if let Some(capabilities) = capability_part { - let caps = std::str::from_utf8(capabilities).map_err(|_| err("invalid capability list"))?; - for cap in caps.split_whitespace() { - if let Some(target) = cap.strip_prefix("symref=HEAD:") { - validate_refname(target)?; - adv.head_target = Some(target.to_string()); - } - } - } - - Ok(()) -} - -fn fetch_remote_advertisement(url: &str) -> io::Result { - let info_refs_url = format!( - "{}/info/refs?service=git-upload-pack", - url.trim_end_matches('/') - ); - let req = Request::new(Method::Get, &info_refs_url) - .map_err(|e| err(&format!("bad info/refs URL: {}", e)))? - .header("Accept", "application/x-git-upload-pack-advertisement") - .header("Git-Protocol", "version=0"); - - let resp = HttpClient::new() - .send(&req) - .map_err(|e| err(&format!("fetch info/refs failed: {}", e)))?; - if resp.status != 200 { - let body = response_body_preview(&resp.body); - return Err(err(&format!( - "remote advertised refs request failed (HTTP {}): {}", - resp.status, body - ))); - } - if resp.body.len() > MAX_REF_ADVERTISEMENT_BYTES { - return Err(err("remote advertised refs response exceeds size limit")); - } - - let lines = parse_pkt_lines(&resp.body)?; - let mut adv = RemoteAdvertisement::default(); - let mut saw_service = false; - let mut in_refs = false; - - for line in lines { - match line { - PktLine::Flush => { - if saw_service { - in_refs = true; - } - } - PktLine::Data(data) => { - if !saw_service { - let service = std::str::from_utf8(&data) - .map_err(|_| err("invalid git service header"))?; - if service.trim_end_matches('\n') != "# service=git-upload-pack" { - return Err(err("unexpected git service advertisement")); - } - saw_service = true; - continue; - } - if !in_refs { - continue; - } - parse_advertised_ref(&data, &mut adv)?; - } - } - } - - if !saw_service { - return Err(err("missing git-upload-pack service header")); - } - - Ok(adv) -} - -fn branch_name_from_ref(refname: &str) -> io::Result { - refname - .strip_prefix("refs/heads/") - .map(validate_branch_name) - .transpose()? - .map(|name| name.to_string()) - .ok_or_else(|| err("expected refs/heads/* ref")) -} - -fn default_branch_ref(adv: &RemoteAdvertisement) -> Option<(String, [u8; 20])> { - if let Some(target) = adv.head_target.as_ref() { - if let Some(hash) = adv.branches.get(target) { - return Some((target.clone(), *hash)); - } - } - - if let Some(head_hash) = adv.head_hash { - if let Some((name, hash)) = adv.branches.iter().find(|(_, hash)| **hash == head_hash) { - return Some((name.clone(), *hash)); - } - } - - adv.branches - .iter() - .next() - .map(|(name, hash)| (name.clone(), *hash)) -} - -fn pkt_line_bytes(payload: &[u8]) -> Vec { - let mut out = format!("{:04x}", payload.len() + 4).into_bytes(); - out.extend_from_slice(payload); - out -} - -fn fetch_remote_pack(url: &str, wants: &[String]) -> io::Result> { - if wants.is_empty() { - return Ok(Vec::new()); - } - - let upload_pack_url = format!("{}/git-upload-pack", url.trim_end_matches('/')); - let mut body = Vec::new(); - for (i, want) in wants.iter().enumerate() { - let line = if i == 0 { - format!( - "want {} side-band-64k ofs-delta no-progress include-tag agent=agentos/1.0\n", - want - ) - } else { - format!("want {}\n", want) - }; - body.extend_from_slice(&pkt_line_bytes(line.as_bytes())); - } - body.extend_from_slice(b"0000"); - body.extend_from_slice(&pkt_line_bytes(b"done\n")); - - let req = Request::new(Method::Post, &upload_pack_url) - .map_err(|e| err(&format!("bad upload-pack URL: {}", e)))? - .header("Content-Type", "application/x-git-upload-pack-request") - .header("Accept", "application/x-git-upload-pack-result") - .header("Git-Protocol", "version=0") - .body(body); - - let resp = HttpClient::new() - .send(&req) - .map_err(|e| err(&format!("git-upload-pack failed: {}", e)))?; - if resp.status != 200 { - let body = response_body_preview(&resp.body); - return Err(err(&format!( - "git-upload-pack returned HTTP {}: {}", - resp.status, body - ))); - } - - if resp.body.len() > MAX_PACK_BYTES { - return Err(err("packfile exceeds size limit")); - } - if resp.body.starts_with(b"PACK") { - return Ok(resp.body); - } - - let mut pack = Vec::new(); - for line in parse_pkt_lines(&resp.body)? { - match line { - PktLine::Flush => {} - PktLine::Data(payload) => { - if payload == b"NAK\n" || payload.starts_with(b"ACK ") { - continue; - } - if payload.starts_with(b"ERR ") { - let msg = String::from_utf8_lossy(&payload[4..]); - return Err(err(&format!("remote upload-pack error: {}", msg.trim()))); - } - if payload.starts_with(b"PACK") { - append_pack_bytes(&mut pack, &payload)?; - continue; - } - if payload.is_empty() { - continue; - } - match payload[0] { - 1 => append_pack_bytes(&mut pack, &payload[1..])?, - 2 => {} - 3 => { - let msg = String::from_utf8_lossy(&payload[1..]); - return Err(err(&format!("remote upload-pack error: {}", msg.trim()))); - } - _ => return Err(err("unexpected upload-pack response payload")), - } - } - } - } - - if pack.is_empty() { - return Err(err("git-upload-pack response did not include a packfile")); - } - Ok(pack) -} - -#[derive(Clone, Debug)] -enum PackedObjectKind { - Full { obj_type: String, data: Vec }, - OfsDelta { base_offset: usize, delta: Vec }, - RefDelta { base_hash: [u8; 20], delta: Vec }, -} - -#[derive(Clone, Debug)] -struct PackedObject { - offset: usize, - kind: PackedObjectKind, -} - -#[derive(Clone, Debug)] -struct ResolvedObject { - obj_type: String, - data: Vec, - hash: [u8; 20], -} - -fn parse_pack_object_header(pack: &[u8], offset: &mut usize) -> io::Result<(u8, usize)> { - if *offset >= pack.len() { - return Err(err("truncated pack object header")); - } - - let mut byte = pack[*offset]; - *offset += 1; - - let obj_type = (byte >> 4) & 0x7; - let mut size = (byte & 0x0f) as usize; - let mut shift = 4usize; - - while byte & 0x80 != 0 { - if *offset >= pack.len() { - return Err(err("truncated pack object size")); - } - byte = pack[*offset]; - *offset += 1; - if shift >= usize::BITS as usize { - return Err(err("pack object size is too large")); - } - size |= ((byte & 0x7f) as usize) - .checked_shl(shift as u32) - .ok_or_else(|| err("pack object size is too large"))?; - shift += 7; - } - - Ok((obj_type, size)) -} - -fn parse_ofs_delta_base( - pack: &[u8], - offset: &mut usize, - object_offset: usize, -) -> io::Result { - if *offset >= pack.len() { - return Err(err("truncated ofs-delta base")); - } - - let mut byte = pack[*offset]; - *offset += 1; - let mut distance = (byte & 0x7f) as usize; - - while byte & 0x80 != 0 { - if *offset >= pack.len() { - return Err(err("truncated ofs-delta base")); - } - byte = pack[*offset]; - *offset += 1; - distance = distance - .checked_add(1) - .and_then(|value| value.checked_shl(7)) - .map(|value| value | ((byte & 0x7f) as usize)) - .ok_or_else(|| err("ofs-delta base distance is too large"))?; - } - - object_offset - .checked_sub(distance) - .ok_or_else(|| err("invalid ofs-delta base distance")) -} - -fn inflate_pack_stream(data: &[u8], expected_size: usize) -> io::Result<(Vec, usize)> { - if expected_size > MAX_GIT_OBJECT_BYTES { - return Err(err("pack object exceeds size limit")); - } - let cursor = Cursor::new(data); - let mut decoder = BufZlibDecoder::new(cursor); - let out = read_to_end_limited(&mut decoder, expected_size, "pack object")?; - if out.len() != expected_size { - return Err(err("pack object size mismatch")); - } - let consumed = decoder.get_ref().position() as usize; - Ok((out, consumed)) -} - -fn parse_packfile(pack: &[u8]) -> io::Result> { - if pack.len() < 12 + 20 { - return Err(err("packfile too small")); - } - if &pack[..4] != b"PACK" { - return Err(err("invalid packfile signature")); - } - - let version = u32::from_be_bytes(pack[4..8].try_into().unwrap()); - if version != 2 && version != 3 { - return Err(err(&format!("unsupported packfile version {}", version))); - } - let object_count = u32::from_be_bytes(pack[8..12].try_into().unwrap()) as usize; - let pack_end = pack.len() - 20; - let mut offset = 12usize; - let max_count_by_bytes = pack_end.saturating_sub(offset); - if object_count > MAX_PACK_OBJECTS || object_count > max_count_by_bytes { - return Err(err("packfile object count is too large")); - } - let mut objects = Vec::new(); - try_reserve_exact(&mut objects, object_count, "pack object table")?; - let mut inflated_bytes = 0usize; - - for _ in 0..object_count { - if offset >= pack_end { - return Err(err("truncated packfile")); - } - - let object_offset = offset; - let (obj_type, object_size) = parse_pack_object_header(pack, &mut offset)?; - - let kind = match obj_type { - 1 | 2 | 3 | 4 => { - let obj_type = match obj_type { - 1 => "commit", - 2 => "tree", - 3 => "blob", - 4 => "tag", - _ => unreachable!(), - }; - let (data, consumed) = inflate_pack_stream(&pack[offset..pack_end], object_size)?; - offset += consumed; - add_pack_inflated_bytes(&mut inflated_bytes, data.len())?; - PackedObjectKind::Full { - obj_type: obj_type.to_string(), - data, - } - } - 6 => { - let base_offset = parse_ofs_delta_base(pack, &mut offset, object_offset)?; - let (delta, consumed) = inflate_pack_stream(&pack[offset..pack_end], object_size)?; - offset += consumed; - add_pack_inflated_bytes(&mut inflated_bytes, delta.len())?; - PackedObjectKind::OfsDelta { base_offset, delta } - } - 7 => { - if offset + 20 > pack_end { - return Err(err("truncated ref-delta base")); - } - let mut base_hash = [0u8; 20]; - base_hash.copy_from_slice(&pack[offset..offset + 20]); - offset += 20; - let (delta, consumed) = inflate_pack_stream(&pack[offset..pack_end], object_size)?; - offset += consumed; - add_pack_inflated_bytes(&mut inflated_bytes, delta.len())?; - PackedObjectKind::RefDelta { base_hash, delta } - } - _ => return Err(err(&format!("unsupported pack object type {}", obj_type))), - }; - - objects.push(PackedObject { - offset: object_offset, - kind, - }); - } - - Ok(objects) -} - -fn read_delta_varint(data: &[u8], pos: &mut usize) -> io::Result { - let mut value = 0usize; - let mut shift = 0usize; - - loop { - if *pos >= data.len() { - return Err(err("truncated delta header")); - } - let byte = data[*pos]; - *pos += 1; - - if shift >= usize::BITS as usize { - return Err(err("delta varint is too large")); - } - value |= ((byte & 0x7f) as usize) - .checked_shl(shift as u32) - .ok_or_else(|| err("delta varint is too large"))?; - if byte & 0x80 == 0 { - return Ok(value); - } - shift += 7; - } -} - -fn ensure_delta_output_room( - current_len: usize, - additional_len: usize, - result_size: usize, -) -> io::Result<()> { - let next_len = current_len - .checked_add(additional_len) - .ok_or_else(|| err("delta result size overflow"))?; - if next_len > result_size { - return Err(err("delta result exceeds declared size")); - } - Ok(()) -} - -fn apply_delta(base: &[u8], delta: &[u8]) -> io::Result> { - let mut pos = 0usize; - let base_size = read_delta_varint(delta, &mut pos)?; - if base_size != base.len() { - return Err(err("delta base size mismatch")); - } - let result_size = read_delta_varint(delta, &mut pos)?; - if result_size > MAX_GIT_OBJECT_BYTES { - return Err(err("delta result exceeds size limit")); - } - let mut out = Vec::new(); - try_reserve_exact(&mut out, result_size, "delta result")?; - - while pos < delta.len() { - let opcode = delta[pos]; - pos += 1; - - if opcode & 0x80 != 0 { - let mut copy_offset = 0usize; - let mut copy_size = 0usize; - - if opcode & 0x01 != 0 { - copy_offset |= delta - .get(pos) - .copied() - .ok_or_else(|| err("truncated delta copy"))? - as usize; - pos += 1; - } - if opcode & 0x02 != 0 { - copy_offset |= (delta - .get(pos) - .copied() - .ok_or_else(|| err("truncated delta copy"))? - as usize) - << 8; - pos += 1; - } - if opcode & 0x04 != 0 { - copy_offset |= (delta - .get(pos) - .copied() - .ok_or_else(|| err("truncated delta copy"))? - as usize) - << 16; - pos += 1; - } - if opcode & 0x08 != 0 { - copy_offset |= (delta - .get(pos) - .copied() - .ok_or_else(|| err("truncated delta copy"))? - as usize) - << 24; - pos += 1; - } - - if opcode & 0x10 != 0 { - copy_size |= delta - .get(pos) - .copied() - .ok_or_else(|| err("truncated delta copy"))? - as usize; - pos += 1; - } - if opcode & 0x20 != 0 { - copy_size |= (delta - .get(pos) - .copied() - .ok_or_else(|| err("truncated delta copy"))? - as usize) - << 8; - pos += 1; - } - if opcode & 0x40 != 0 { - copy_size |= (delta - .get(pos) - .copied() - .ok_or_else(|| err("truncated delta copy"))? - as usize) - << 16; - pos += 1; - } - if copy_size == 0 { - copy_size = 0x10000; - } - - let end = copy_offset - .checked_add(copy_size) - .ok_or_else(|| err("delta copy overflow"))?; - if end > base.len() { - return Err(err("delta copy exceeds base object")); - } - ensure_delta_output_room(out.len(), copy_size, result_size)?; - out.extend_from_slice(&base[copy_offset..end]); - } else if opcode != 0 { - let insert_len = opcode as usize; - let end = pos - .checked_add(insert_len) - .ok_or_else(|| err("delta insert overflow"))?; - if end > delta.len() { - return Err(err("truncated delta insert")); - } - ensure_delta_output_room(out.len(), insert_len, result_size)?; - out.extend_from_slice(&delta[pos..end]); - pos = end; - } else { - return Err(err("invalid delta opcode")); - } - } - - if out.len() != result_size { - return Err(err("delta result size mismatch")); - } - - Ok(out) -} - -fn maybe_read_local_object(git_dir: &Path, hash: &[u8; 20]) -> io::Result> { - let h = hex(hash); - let path = git_dir.join("objects").join(&h[..2]).join(&h[2..]); - if !path.exists() { - return Ok(None); - } - let (obj_type, data) = read_object(git_dir, hash)?; - Ok(Some(ResolvedObject { - obj_type, - data, - hash: *hash, - })) -} - -fn find_entry_by_hash( - target: &[u8; 20], - git_dir: &Path, - objects: &[PackedObject], - offset_to_index: &HashMap, - memo: &mut [Option], - visiting: &mut [bool], - resolved_bytes: &mut usize, -) -> io::Result> { - for idx in 0..objects.len() { - if visiting[idx] { - continue; - } - let resolved = resolve_packed_object( - idx, - git_dir, - objects, - offset_to_index, - memo, - visiting, - resolved_bytes, - )?; - if resolved.hash == *target { - return Ok(Some(idx)); - } - } - - Ok(None) -} - -fn resolve_packed_object( - idx: usize, - git_dir: &Path, - objects: &[PackedObject], - offset_to_index: &HashMap, - memo: &mut [Option], - visiting: &mut [bool], - resolved_bytes: &mut usize, -) -> io::Result { - if let Some(resolved) = memo[idx].as_ref() { - return Ok(resolved.clone()); - } - if visiting[idx] { - return Err(err("cyclic pack delta dependency")); - } - visiting[idx] = true; - - let resolved = match &objects[idx].kind { - PackedObjectKind::Full { obj_type, data } => ResolvedObject { - obj_type: obj_type.clone(), - data: data.clone(), - hash: hash_bytes(obj_type, data), - }, - PackedObjectKind::OfsDelta { base_offset, delta } => { - let base_idx = *offset_to_index - .get(base_offset) - .ok_or_else(|| err("missing ofs-delta base object"))?; - let base = resolve_packed_object( - base_idx, - git_dir, - objects, - offset_to_index, - memo, - visiting, - resolved_bytes, - )?; - let data = apply_delta(&base.data, delta)?; - let hash = hash_bytes(&base.obj_type, &data); - ResolvedObject { - obj_type: base.obj_type, - data, - hash, - } - } - PackedObjectKind::RefDelta { base_hash, delta } => { - let base = if let Some(local) = maybe_read_local_object(git_dir, base_hash)? { - local - } else if let Some(base_idx) = find_entry_by_hash( - base_hash, - git_dir, - objects, - offset_to_index, - memo, - visiting, - resolved_bytes, - )? { - resolve_packed_object( - base_idx, - git_dir, - objects, - offset_to_index, - memo, - visiting, - resolved_bytes, - )? - } else { - return Err(err("missing ref-delta base object")); - }; - - let data = apply_delta(&base.data, delta)?; - let hash = hash_bytes(&base.obj_type, &data); - ResolvedObject { - obj_type: base.obj_type, - data, - hash, - } - } - }; - - visiting[idx] = false; - add_pack_resolved_bytes(resolved_bytes, resolved.data.len())?; - memo[idx] = Some(resolved.clone()); - Ok(resolved) -} - -fn store_pack_objects(git_dir: &Path, pack: &[u8]) -> io::Result<()> { - if pack.is_empty() { - return Ok(()); - } - - let objects = parse_packfile(pack)?; - let offset_to_index: HashMap = objects - .iter() - .enumerate() - .map(|(idx, obj)| (obj.offset, idx)) - .collect(); - let mut memo: Vec> = vec![None; objects.len()]; - let mut visiting = vec![false; objects.len()]; - let mut resolved_bytes = 0usize; - - for idx in 0..objects.len() { - let resolved = resolve_packed_object( - idx, - git_dir, - &objects, - &offset_to_index, - &mut memo, - &mut visiting, - &mut resolved_bytes, - )?; - let stored = hash_object(git_dir, &resolved.obj_type, &resolved.data)?; - if stored != resolved.hash { - return Err(err("pack object hash mismatch")); - } - } - Ok(()) -} - -fn cmd_clone_remote(source: &str, dest: &Path) -> io::Result<()> { - if dest.exists() && !dir_is_empty(dest)? { - return Err(err(&format!( - "destination path '{}' already exists and is not an empty directory", - dest.display() - ))); - } - - let advertisement = fetch_remote_advertisement(source)?; - let mut wants: Vec = Vec::new(); - let mut seen = HashSet::new(); - for hash in advertisement.branches.values() { - if !is_zero_oid(hash) { - let hash_hex = hex(hash); - if seen.insert(hash_hex.clone()) { - wants.push(hash_hex); - } - } - } - if wants.is_empty() { - if let Some(head_hash) = advertisement.head_hash { - if !is_zero_oid(&head_hash) { - wants.push(hex(&head_hash)); - } - } - } - - let default_ref = if let Some((refname, hash)) = default_branch_ref(&advertisement) { - Some((refname.clone(), branch_name_from_ref(&refname)?, hash)) - } else { - None - }; - let default_branch = default_ref - .as_ref() - .map(|(_, branch, _)| branch.clone()) - .or_else(|| { - advertisement - .head_target - .as_ref() - .and_then(|target| branch_name_from_ref(target).ok()) - }) - .unwrap_or_else(|| "main".to_string()); - - let pack = fetch_remote_pack(source, &wants)?; - - let dst_git = prepare_clone_destination(dest, source, &default_branch)?; - store_pack_objects(&dst_git, &pack)?; - - for (refname, hash) in &advertisement.branches { - if is_zero_oid(hash) { - continue; - } - let branch = branch_name_from_ref(refname)?; - update_ref(&dst_git, &format!("refs/remotes/origin/{}", branch), hash)?; - } - - for (refname, hash) in &advertisement.tags { - if is_zero_oid(hash) { - continue; - } - update_ref(&dst_git, refname, hash)?; - } - - let default_hash = default_ref - .as_ref() - .map(|(_, _, hash)| *hash) - .or(advertisement.head_hash) - .filter(|hash| !is_zero_oid(hash)); - - if let Some(hash) = default_hash { - update_ref(&dst_git, &format!("refs/heads/{}", default_branch), &hash)?; - cmd_checkout(dest, &default_branch, false)?; - } - - Ok(()) -} - -// ─── Object store ─────────────────────────────────────────────────────────── - -fn hash_object(git_dir: &Path, obj_type: &str, data: &[u8]) -> io::Result<[u8; 20]> { - if data.len() > MAX_GIT_OBJECT_BYTES { - return Err(err("git object exceeds size limit")); - } - let header = format!("{} {}\0", obj_type, data.len()); - let mut hasher = Sha1::new(); - hasher.update(header.as_bytes()); - hasher.update(data); - let hash: [u8; 20] = hasher.finalize().into(); - - let h = hex(&hash); - let dir = git_dir.join("objects").join(&h[..2]); - let path = dir.join(&h[2..]); - if !path.exists() { - mkdirs(&dir)?; - let mut enc = ZlibEncoder::new(Vec::new(), Compression::default()); - enc.write_all(header.as_bytes())?; - enc.write_all(data)?; - fs::write(&path, enc.finish()?)?; - } - Ok(hash) -} - -fn read_object(git_dir: &Path, hash: &[u8; 20]) -> io::Result<(String, Vec)> { - let h = hex(hash); - let path = git_dir.join("objects").join(&h[..2]).join(&h[2..]); - let read_limit = MAX_GIT_OBJECT_BYTES - .checked_add(MAX_GIT_OBJECT_HEADER_BYTES) - .ok_or_else(|| err("git object size limit is too large"))?; - let compressed = read_file_limited(&path, read_limit, "git object")?; - let mut dec = ZlibDecoder::new(&compressed[..]); - let buf = read_to_end_limited(&mut dec, read_limit, "git object")?; - - let nul = buf - .iter() - .position(|&b| b == 0) - .ok_or_else(|| err("no nul in object"))?; - let header = - std::str::from_utf8(&buf[..nul]).map_err(|_| err("invalid object header encoding"))?; - let (typ, size) = header - .split_once(' ') - .ok_or_else(|| err("malformed object header"))?; - let size: usize = size.parse().map_err(|_| err("invalid object size"))?; - if size > MAX_GIT_OBJECT_BYTES { - return Err(err("git object exceeds size limit")); - } - if buf.len() - nul - 1 != size { - return Err(err("git object size mismatch")); - } - Ok((typ.to_string(), buf[nul + 1..].to_vec())) -} - -// ─── Index ────────────────────────────────────────────────────────────────── - -#[derive(Clone, Debug)] -struct IndexEntry { - mode: u32, - sha1: [u8; 20], - name: String, -} - -fn read_index(git_dir: &Path) -> io::Result> { - let path = git_dir.join("index"); - if !path.exists() { - return Ok(Vec::new()); - } - let data = read_file_limited(&path, MAX_INDEX_BYTES, "git index")?; - if data.len() < 12 || &data[0..4] != b"DIRC" { - return Err(err("invalid index file")); - } - let version = u32::from_be_bytes(data[4..8].try_into().unwrap()); - if version != 2 { - return Err(err(&format!("unsupported index version {}", version))); - } - let count = u32::from_be_bytes(data[8..12].try_into().unwrap()) as usize; - let max_count_by_bytes = data.len().saturating_sub(12) / 62; - if count > MAX_INDEX_ENTRIES || count > max_count_by_bytes { - return Err(err("index entry count is too large")); - } - - let mut entries = Vec::new(); - try_reserve_exact(&mut entries, count, "index entry table")?; - let mut pos = 12; - - for _ in 0..count { - if pos + 62 > data.len() { - return Err(err("truncated index")); - } - // Stat fields at pos+0..pos+24 (ctime, mtime, dev, ino) - skip - let mode = u32::from_be_bytes(data[pos + 24..pos + 28].try_into().unwrap()); - // Skip uid(4), gid(4), size(4) at pos+28..pos+40 - let mut sha1 = [0u8; 20]; - sha1.copy_from_slice(&data[pos + 40..pos + 60]); - // Flags at pos+60..pos+62 - // Find name (NUL-terminated after fixed 62 bytes) - let name_start = pos + 62; - let nul_offset = data[name_start..] - .iter() - .position(|&b| b == 0) - .ok_or_else(|| err("unterminated index entry name"))?; - let name = String::from_utf8(data[name_start..name_start + nul_offset].to_vec()) - .map_err(|_| err("invalid entry name"))?; - let name = normalize_repo_path(&name)?; - - entries.push(IndexEntry { mode, sha1, name }); - - // Advance past padding: entry padded to 8-byte boundary - let entry_len = 62 + nul_offset; - pos += (entry_len + 8) & !7; - } - - Ok(entries) -} - -fn write_index(git_dir: &Path, entries: &[IndexEntry]) -> io::Result<()> { - let entry_count = u32::try_from(entries.len()).map_err(|_| err("too many index entries"))?; - let mut buf = Vec::new(); - buf.extend_from_slice(b"DIRC"); - buf.extend_from_slice(&2u32.to_be_bytes()); - buf.extend_from_slice(&entry_count.to_be_bytes()); - - for entry in entries { - let name = normalize_repo_path(&entry.name)?; - if name.len() > 0xFFF { - return Err(err("index entry name is too long")); - } - let entry_start = buf.len(); - // ctime(8) + mtime(8) + dev(4) + ino(4) = 24 bytes of zeros - buf.extend_from_slice(&[0u8; 24]); - buf.extend_from_slice(&entry.mode.to_be_bytes()); - // uid(4) + gid(4) + size(4) = 12 bytes of zeros - buf.extend_from_slice(&[0u8; 12]); - buf.extend_from_slice(&entry.sha1); - // Flags: name length in lower 12 bits - buf.extend_from_slice(&(name.len() as u16).to_be_bytes()); - buf.extend_from_slice(name.as_bytes()); - // Pad to 8-byte boundary (1-8 NUL bytes) - let entry_len = buf.len() - entry_start; - let padded = (entry_len + 8) & !7; - buf.resize(entry_start + padded, 0); - } - - // SHA-1 checksum of entire index - let checksum: [u8; 20] = Sha1::digest(&buf).into(); - buf.extend_from_slice(&checksum); - - fs::write(git_dir.join("index"), &buf) -} - -// ─── Refs ─────────────────────────────────────────────────────────────────── - -fn resolve_ref(git_dir: &Path, refname: &str) -> io::Result> { - let refname = validate_refname(refname)?; - let ref_path = git_dir.join(refname); - if !ref_path.exists() { - return Ok(None); - } - let content = fs::read_to_string(&ref_path)?; - let content = content.trim(); - if let Some(target) = content.strip_prefix("ref: ") { - resolve_ref(git_dir, target) - } else { - Ok(Some(unhex(content)?)) - } -} - -fn resolve_head(git_dir: &Path) -> io::Result> { - resolve_ref(git_dir, "HEAD") -} - -fn head_branch(git_dir: &Path) -> io::Result> { - let head = fs::read_to_string(git_dir.join("HEAD"))?; - let head = head.trim(); - Ok(head.strip_prefix("ref: refs/heads/").map(|s| s.to_string())) -} - -fn update_ref(git_dir: &Path, refname: &str, hash: &[u8; 20]) -> io::Result<()> { - let refname = validate_refname(refname)?; - let ref_path = git_dir.join(refname); - if let Some(parent) = ref_path.parent() { - mkdirs(parent)?; - } - fs::write(&ref_path, format!("{}\n", hex(hash))) -} - -// ─── Tree operations ──────────────────────────────────────────────────────── - -fn build_tree(git_dir: &Path, entries: &[IndexEntry]) -> io::Result<[u8; 20]> { - build_tree_at(git_dir, entries, "") -} - -fn build_tree_at(git_dir: &Path, entries: &[IndexEntry], prefix: &str) -> io::Result<[u8; 20]> { - struct TreeEntry { - mode: String, - name: String, - hash: [u8; 20], - } - - let mut tree_entries: Vec = Vec::new(); - let mut subdirs: BTreeMap> = BTreeMap::new(); - - for entry in entries { - let relative = if prefix.is_empty() { - &entry.name[..] - } else if let Some(rest) = entry.name.strip_prefix(prefix) { - rest - } else { - continue; - }; - - if let Some(slash) = relative.find('/') { - let dir = &relative[..slash]; - subdirs - .entry(dir.to_string()) - .or_default() - .push(entry.clone()); - } else { - tree_entries.push(TreeEntry { - mode: format!("{:o}", entry.mode), - name: relative.to_string(), - hash: entry.sha1, - }); - } - } - - for (dir, sub_entries) in &subdirs { - let sub_prefix = if prefix.is_empty() { - format!("{}/", dir) - } else { - format!("{}{}/", prefix, dir) - }; - let subtree_hash = build_tree_at(git_dir, sub_entries, &sub_prefix)?; - tree_entries.push(TreeEntry { - mode: "40000".to_string(), - name: dir.clone(), - hash: subtree_hash, - }); - } - - // Sort: directories get trailing / for comparison - tree_entries.sort_by(|a, b| { - let ak = if a.mode == "40000" { - format!("{}/", a.name) - } else { - a.name.clone() - }; - let bk = if b.mode == "40000" { - format!("{}/", b.name) - } else { - b.name.clone() - }; - ak.cmp(&bk) - }); - - let mut data = Vec::new(); - for te in &tree_entries { - data.extend_from_slice(te.mode.as_bytes()); - data.push(b' '); - data.extend_from_slice(te.name.as_bytes()); - data.push(0); - data.extend_from_slice(&te.hash); - } - - hash_object(git_dir, "tree", &data) -} - -/// Read a tree recursively into flat (path, mode, hash) entries. -fn read_tree_entries(git_dir: &Path, hash: &[u8; 20], prefix: &str) -> io::Result> { - let (typ, data) = read_object(git_dir, hash)?; - if typ != "tree" { - return Err(err("expected tree object")); - } - - let mut entries = Vec::new(); - let mut pos = 0; - - while pos < data.len() { - let space = data[pos..] - .iter() - .position(|&b| b == b' ') - .ok_or_else(|| err("bad tree entry"))?; - let mode_str = - std::str::from_utf8(&data[pos..pos + space]).map_err(|_| err("bad tree mode"))?; - let mode = u32::from_str_radix(mode_str, 8).map_err(|_| err("bad tree mode number"))?; - pos += space + 1; - - let nul = data[pos..] - .iter() - .position(|&b| b == 0) - .ok_or_else(|| err("bad tree entry name"))?; - let name = std::str::from_utf8(&data[pos..pos + nul]).map_err(|_| err("bad name"))?; - if name.contains('/') { - return Err(err("tree entry name must not contain '/'")); - } - if normalize_repo_path(name)? != name { - return Err(err("tree entry name must be normalized")); - } - pos += nul + 1; - - if pos + 20 > data.len() { - return Err(err("truncated tree hash")); - } - let mut hash_buf = [0u8; 20]; - hash_buf.copy_from_slice(&data[pos..pos + 20]); - pos += 20; - - let full_name = if prefix.is_empty() { - name.to_string() - } else { - format!("{}{}", prefix, name) - }; - - if mode == 0o40000 { - let sub = read_tree_entries(git_dir, &hash_buf, &format!("{}/", full_name))?; - entries.extend(sub); - } else { - let full_name = normalize_repo_path(&full_name)?; - entries.push(IndexEntry { - mode, - sha1: hash_buf, - name: full_name, - }); - } - } - - Ok(entries) -} - -/// Extract the tree hash from a commit object. -fn commit_tree(git_dir: &Path, commit_hash: &[u8; 20]) -> io::Result<[u8; 20]> { - let (typ, data) = read_object(git_dir, commit_hash)?; - if typ != "commit" { - return Err(err("not a commit object")); - } - let text = String::from_utf8_lossy(&data); - let tree_hex = text - .lines() - .find_map(|l| l.strip_prefix("tree ")) - .ok_or_else(|| err("no tree line in commit"))?; - unhex(tree_hex) -} - -// ─── Commands ─────────────────────────────────────────────────────────────── - -fn cmd_init(path: &Path, quiet: bool) -> io::Result<()> { - let git_dir = path.join(".git"); - // Create directories one at a time to avoid create_dir_all issues on WASI - for dir in &[ - path.to_path_buf(), - git_dir.clone(), - git_dir.join("objects"), - git_dir.join("refs"), - git_dir.join("refs/heads"), - git_dir.join("refs/tags"), - ] { - match fs::create_dir(dir) { - Ok(()) => {} - Err(e) if e.kind() == io::ErrorKind::AlreadyExists => {} - Err(e) => return Err(err(&format!("mkdir {}: {}", dir.display(), e))), - } - } - fs::write(git_dir.join("HEAD"), "ref: refs/heads/main\n") - .map_err(|e| err(&format!("write HEAD: {}", e)))?; - fs::write( - git_dir.join("config"), - "[core]\n\trepositoryformatversion = 0\n\tfilemode = true\n\tbare = false\n", - ) - .map_err(|e| err(&format!("write config: {}", e)))?; - if !quiet { - print_stdout_line(format_args!( - "Initialized empty Git repository in {}/.git/", - path.display() - ))?; - } - Ok(()) -} - -fn collect_add_paths(workdir: &Path, rel_path: &str, out: &mut Vec) -> io::Result<()> { - if rel_path == ".git" || rel_path.starts_with(".git/") || rel_path.ends_with("/.git") { - return Ok(()); - } - - if rel_path == "." { - for entry in fs::read_dir(workdir)? { - let entry = entry?; - let name = entry - .file_name() - .to_str() - .ok_or_else(|| err("repository path must be utf-8"))? - .to_owned(); - if name == ".git" { - continue; - } - collect_add_paths(workdir, &name, out)?; - } - return Ok(()); - } - - let repo_path = normalize_repo_path(rel_path)?; - let file_path = workdir.join(&repo_path); - if file_path.is_dir() { - for entry in fs::read_dir(&file_path)? { - let entry = entry?; - let name = entry - .file_name() - .to_str() - .ok_or_else(|| err("repository path must be utf-8"))? - .to_owned(); - if name == ".git" { - continue; - } - collect_add_paths(workdir, &format!("{repo_path}/{name}"), out)?; - } - return Ok(()); - } - - if file_path.exists() { - out.push(repo_path); - } - Ok(()) -} - -fn cmd_add(workdir: &Path, paths: &[String]) -> io::Result<()> { - let git_dir = workdir.join(".git"); - let mut entries = read_index(&git_dir).map_err(|e| { - err(&format!( - "cannot read index at {}: {}", - git_dir.display(), - e - )) - })?; - - for rel_path in paths { - let mut repo_paths = Vec::new(); - collect_add_paths(workdir, rel_path, &mut repo_paths)?; - if repo_paths.is_empty() { - return Err(err(&format!( - "pathspec '{}' did not match any files", - rel_path - ))); - } - for repo_path in repo_paths { - let file_path = workdir.join(&repo_path); - if !file_path.exists() { - return Err(err(&format!( - "pathspec '{}' did not match any files (looked at {})", - rel_path, - file_path.display() - ))); - } - let content = read_file_limited(&file_path, MAX_GIT_OBJECT_BYTES, "file")?; - let hash = hash_object(&git_dir, "blob", &content)?; - - entries.retain(|e| e.name != repo_path); - entries.push(IndexEntry { - mode: 0o100644, - sha1: hash, - name: repo_path, - }); - } - } - - entries.sort_by(|a, b| a.name.cmp(&b.name)); - write_index(&git_dir, &entries) -} - -fn cmd_commit(workdir: &Path, message: &str) -> io::Result<()> { - let git_dir = workdir.join(".git"); - let entries = read_index(&git_dir)?; - if entries.is_empty() { - return Err(err("nothing to commit")); - } - - let tree_hash = build_tree(&git_dir, &entries)?; - let parent = resolve_head(&git_dir)?; - - let name = std::env::var("GIT_AUTHOR_NAME") - .or_else(|_| std::env::var("GIT_COMMITTER_NAME")) - .unwrap_or_else(|_| "secure-exec".to_string()); - let email = std::env::var("GIT_AUTHOR_EMAIL") - .or_else(|_| std::env::var("GIT_COMMITTER_EMAIL")) - .unwrap_or_else(|_| "agent@os".to_string()); - let timestamp = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_secs()) - .unwrap_or(1700000000); - - let mut commit = String::new(); - commit.push_str(&format!("tree {}\n", hex(&tree_hash))); - if let Some(p) = parent { - commit.push_str(&format!("parent {}\n", hex(&p))); - } - commit.push_str(&format!( - "author {} <{}> {} +0000\n", - name, email, timestamp - )); - commit.push_str(&format!( - "committer {} <{}> {} +0000\n", - name, email, timestamp - )); - commit.push_str(&format!("\n{}\n", message)); - - let commit_hash = hash_object(&git_dir, "commit", commit.as_bytes())?; - - // Update the ref that HEAD points to - let head_content = fs::read_to_string(git_dir.join("HEAD"))?; - let head_content = head_content.trim(); - if let Some(refname) = head_content.strip_prefix("ref: ") { - update_ref(&git_dir, refname, &commit_hash)?; - } else { - fs::write(git_dir.join("HEAD"), format!("{}\n", hex(&commit_hash)))?; - } - - Ok(()) -} - -fn cmd_rev_parse(workdir: &Path, args: &[String]) -> io::Result<()> { - let git_dir = workdir.join(".git"); - if args.len() != 1 { - return Err(err("usage: git rev-parse ")); - } - - let hash = resolve_ref(&git_dir, &args[0])?.ok_or_else(|| { - err(&format!( - "unknown revision or path not in the working tree: {}", - args[0] - )) - })?; - print_stdout_line(format_args!("{}", hex(&hash))) -} - -fn cmd_branch(workdir: &Path) -> io::Result<()> { - let git_dir = workdir.join(".git"); - let heads_dir = git_dir.join("refs/heads"); - let current = head_branch(&git_dir)?; - - let mut branches: Vec = Vec::new(); - if heads_dir.exists() { - for entry in fs::read_dir(&heads_dir)? { - let entry = entry?; - let name = entry.file_name().to_string_lossy().to_string(); - if name == "." || name == ".." { - continue; - } - if entry.file_type()?.is_file() { - branches.push(name); - } - } - } - branches.sort(); - - for branch in &branches { - if Some(branch.as_str()) == current.as_deref() { - print_stdout_line(format_args!("* {}", branch))?; - } else { - print_stdout_line(format_args!(" {}", branch))?; - } - } - - Ok(()) -} - -fn cmd_checkout(workdir: &Path, target: &str, create_branch: bool) -> io::Result<()> { - let git_dir = workdir.join(".git"); - - if create_branch { - validate_branch_name(target)?; - let head = resolve_head(&git_dir)?.ok_or_else(|| err("HEAD not found for new branch"))?; - update_ref(&git_dir, &format!("refs/heads/{}", target), &head)?; - fs::write( - git_dir.join("HEAD"), - format!("ref: refs/heads/{}\n", target), - )?; - return Ok(()); - } - - // Resolve target: local branch first, then DWIM remote tracking - validate_branch_name(target)?; - let branch_ref = format!("refs/heads/{}", target); - let commit_hash = if let Some(h) = resolve_ref(&git_dir, &branch_ref)? { - fs::write(git_dir.join("HEAD"), format!("ref: {}\n", branch_ref))?; - h - } else { - let remote_ref = format!("refs/remotes/origin/{}", target); - if let Some(h) = resolve_ref(&git_dir, &remote_ref)? { - update_ref(&git_dir, &branch_ref, &h)?; - fs::write(git_dir.join("HEAD"), format!("ref: {}\n", branch_ref))?; - h - } else { - return Err(err(&format!( - "pathspec '{}' did not match any branch", - target - ))); - } - }; - - // Read target tree - let tree_hash = commit_tree(&git_dir, &commit_hash)?; - let new_entries = read_tree_entries(&git_dir, &tree_hash, "")?; - - // Clean up files from current index that aren't in target - let old_entries = read_index(&git_dir)?; - let new_names: HashSet<&str> = new_entries.iter().map(|e| e.name.as_str()).collect(); - for old in &old_entries { - if !new_names.contains(old.name.as_str()) { - let p = worktree_path(workdir, &old.name)?; - if p.exists() { - fs::remove_file(&p).map_err(|e| err(&format!("remove {}: {}", p.display(), e)))?; - } - } - } - - // Write files from target tree - for entry in &new_entries { - let p = worktree_path(workdir, &entry.name)?; - if let Some(parent) = p.parent() { - mkdirs(parent)?; - } - let (_, blob) = read_object(&git_dir, &entry.sha1)?; - fs::write(&p, &blob)?; - } - - // Update index - write_index(&git_dir, &new_entries)?; - - Ok(()) -} - -fn cmd_clone_local(source: &Path, dest: &Path) -> io::Result<()> { - let src_git = source.join(".git"); - if !src_git.is_dir() { - return Err(err(&format!( - "'{}' does not appear to be a git repository", - source.display() - ))); - } - - if dest.exists() && !dir_is_empty(dest)? { - return Err(err(&format!( - "destination path '{}' already exists and is not an empty directory", - dest.display() - ))); - } - - mkdirs(dest)?; - - let dst_git = dest.join(".git"); - - // Init destination - mkdirs(&dst_git.join("objects"))?; - mkdirs(&dst_git.join("refs/heads"))?; - mkdirs(&dst_git.join("refs/tags"))?; - mkdirs(&dst_git.join("refs/remotes/origin"))?; - - // Copy all objects - copy_dir_recursive(&src_git.join("objects"), &dst_git.join("objects"))?; - - // Create remote tracking branches from source heads - let src_heads = src_git.join("refs/heads"); - if src_heads.exists() { - copy_dir_recursive(&src_heads, &dst_git.join("refs/remotes/origin"))?; - } - - // Determine default branch from source HEAD - let src_head = fs::read_to_string(src_git.join("HEAD"))?; - let src_head = src_head.trim(); - let default_branch = src_head - .strip_prefix("ref: refs/heads/") - .unwrap_or("main") - .to_string(); - validate_branch_name(&default_branch)?; - - // Create local branch for default - let remote_ref = format!("refs/remotes/origin/{}", default_branch); - let mut has_default_branch = false; - if let Some(hash) = resolve_ref(&dst_git, &remote_ref)? { - update_ref(&dst_git, &format!("refs/heads/{}", default_branch), &hash)?; - has_default_branch = true; - } - - // Set HEAD and write config - fs::write( - dst_git.join("HEAD"), - format!("ref: refs/heads/{}\n", default_branch), - )?; - fs::write( - dst_git.join("config"), - format!( - "[core]\n\trepositoryformatversion = 0\n\tfilemode = true\n\tbare = false\n\ - [remote \"origin\"]\n\turl = {}\n\tfetch = +refs/heads/*:refs/remotes/origin/*\n", - source.display() - ), - )?; - - // Checkout working tree - if has_default_branch { - cmd_checkout(dest, &default_branch, false)?; - } - - Ok(()) -} - -fn copy_dir_recursive(src: &Path, dst: &Path) -> io::Result<()> { - mkdirs(dst)?; - if !src.exists() { - return Ok(()); - } - for entry in fs::read_dir(src)? { - let entry = entry?; - let name = entry.file_name().to_string_lossy().to_string(); - if name == "." || name == ".." { - continue; - } - let dst_path = dst.join(entry.file_name()); - if entry.file_type()?.is_dir() { - copy_dir_recursive(&entry.path(), &dst_path)?; - } else { - let content = read_file_limited( - &entry.path(), - MAX_GIT_OBJECT_BYTES + MAX_GIT_OBJECT_HEADER_BYTES, - "git repository file", - )?; - fs::write(&dst_path, content)?; - } - } - Ok(()) -} - -// ─── Entry point ──────────────────────────────────────────────────────────── - -pub fn main(args: Vec) -> i32 { - let str_args: Vec = args - .iter() - .map(|a| a.to_string_lossy().to_string()) - .collect(); - match run(&str_args) { - Ok(()) => 0, - Err(e) => { - eprintln!("fatal: {}", e); - 128 - } - } -} - -fn run(args: &[String]) -> io::Result<()> { - let mut i = 1; // skip argv[0] - let mut workdir = std::env::current_dir().unwrap_or_else(|_| PathBuf::from("/")); - - // Parse global options - while i < args.len() { - if args[i] == "-C" { - i += 1; - if i >= args.len() { - return Err(err("-C requires a directory argument")); - } - let p = PathBuf::from(&args[i]); - workdir = if p.is_absolute() { p } else { workdir.join(p) }; - i += 1; - } else if args[i] == "-c" { - i += 1; - if i >= args.len() { - return Err(err("-c requires a config assignment")); - } - i += 1; - } else { - break; - } - } - - if i >= args.len() { - eprintln!("usage: git []"); - return Err(err("no subcommand")); - } - - let subcmd = &args[i]; - let sub_args = &args[i + 1..]; - - match subcmd.as_str() { - "init" => { - let mut path_arg = None; - let mut quiet = false; - for arg in sub_args { - if arg == "-q" || arg == "--quiet" { - quiet = true; - continue; - } - if arg.starts_with('-') { - return Err(unsupported("init", &format!("does not support `{}`.", arg))); - } - if path_arg.replace(arg).is_some() { - return Err(err("usage: git init []")); - } - } - let path = if let Some(path_arg) = path_arg { - let p = PathBuf::from(path_arg); - if p.is_absolute() { - p - } else { - workdir.join(p) - } - } else { - workdir - }; - cmd_init(&path, quiet) - } - "add" => { - if sub_args.is_empty() { - return Err(err("nothing specified, nothing added")); - } - let paths: Vec = sub_args.iter().map(|s| s.to_string()).collect(); - cmd_add(&workdir, &paths) - } - "commit" => { - let mut message = None; - let mut j = 0; - while j < sub_args.len() { - if sub_args[j] == "-m" && j + 1 < sub_args.len() { - message = Some(sub_args[j + 1].clone()); - j += 2; - } else { - j += 1; - } - } - let msg = message.ok_or_else(|| err("no commit message (-m)"))?; - cmd_commit(&workdir, &msg) - } - "branch" => cmd_branch(&workdir), - "rev-parse" => cmd_rev_parse(&workdir, sub_args), - "checkout" => { - let mut create = false; - let mut target = None; - for arg in sub_args { - if arg == "-b" { - create = true; - } else if !arg.starts_with('-') { - target = Some(arg.clone()); - } - } - let t = target.ok_or_else(|| err("no branch name specified"))?; - cmd_checkout(&workdir, &t, create) - } - "clone" => { - if sub_args.is_empty() || sub_args.len() > 2 { - return Err(err("usage: git clone []")); - } - let src_arg = &sub_args[0]; - if is_ssh_clone_source(src_arg) { - return Err(unsupported( - "clone", - &format!("does not support SSH or git:// remotes (`{}`).", src_arg), - )); - } - if has_http_auth(src_arg) { - return Err(unsupported( - "clone", - &format!( - "does not support authenticated HTTP(S) remotes (`{}`).", - src_arg - ), - )); - } - let dst_arg = if sub_args.len() == 2 { - sub_args[1].clone() - } else { - infer_clone_destination(src_arg)? - }; - let dst = PathBuf::from(&dst_arg); - let dst = if dst.is_absolute() { - dst - } else { - workdir.join(dst) - }; - print_stdout_line(format_args!("Cloning into '{}'...", dst.display()))?; - if is_remote_source(src_arg) { - cmd_clone_remote(src_arg, &dst) - } else { - let src = PathBuf::from(src_arg); - let src = if src.is_absolute() { - src - } else { - workdir.join(src) - }; - cmd_clone_local(&src, &dst) - } - } - other => Err(unsupported( - other, - "is not implemented in the secure-exec VM git command.", - )), - } -} diff --git a/registry/native/crates/libs/grep/Cargo.toml b/registry/native/crates/libs/grep/Cargo.toml deleted file mode 100644 index 3cf619a9c4..0000000000 --- a/registry/native/crates/libs/grep/Cargo.toml +++ /dev/null @@ -1,9 +0,0 @@ -[package] -name = "secureexec-grep" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "grep/egrep/fgrep/rg implementations for secure-exec standalone binaries" - -[dependencies] -regex = "1" diff --git a/registry/native/crates/libs/grep/src/lib.rs b/registry/native/crates/libs/grep/src/lib.rs deleted file mode 100644 index c42db0e8ac..0000000000 --- a/registry/native/crates/libs/grep/src/lib.rs +++ /dev/null @@ -1,623 +0,0 @@ -//! grep implementation using the regex crate (ripgrep's pure Rust regex engine). -//! -//! Supports grep, egrep (grep -E), and fgrep (grep -F) modes. -//! Dispatches on argv[0] basename for standalone binary usage. - -mod rg_cmd; - -use std::ffi::OsString; -use std::io::{self, BufRead, Read, Write}; -use std::path::Path; - -use regex::Regex; - -const MAX_INPUT_LINE_BYTES: usize = 16 * 1024 * 1024; -const MAX_PATTERN_BYTES: usize = 16 * 1024 * 1024; -const MAX_PATTERNS: usize = 100_000; - -/// Unified grep entry point. Dispatches on argv[0]: -/// - "egrep" -> Extended mode -/// - "fgrep" -> Fixed mode -/// - default -> Basic mode -pub fn main(args: Vec) -> i32 { - let mode = match args.first().and_then(|a| Path::new(a).file_name()) { - Some(name) if name == "egrep" => GrepMode::Extended, - Some(name) if name == "fgrep" => GrepMode::Fixed, - _ => GrepMode::Basic, - }; - run_grep(args, mode) -} - -/// Entry point for grep command (Basic mode). -pub fn grep(args: Vec) -> i32 { - run_grep(args, GrepMode::Basic) -} - -/// Entry point for egrep command (Extended regex). -pub fn egrep(args: Vec) -> i32 { - run_grep(args, GrepMode::Extended) -} - -/// Entry point for fgrep command (Fixed strings). -pub fn fgrep(args: Vec) -> i32 { - run_grep(args, GrepMode::Fixed) -} - -/// Entry point for rg command. -pub fn rg(args: Vec) -> i32 { - rg_cmd::rg(args) -} - -/// grep mode determines how patterns are interpreted. -#[derive(Clone, Copy, PartialEq)] -enum GrepMode { - /// Basic regular expressions (default grep) - Basic, - /// Extended regular expressions (egrep / grep -E) - Extended, - /// Fixed strings (fgrep / grep -F) - Fixed, -} - -struct GrepOptions { - mode: GrepMode, - ignore_case: bool, - invert_match: bool, - count_only: bool, - files_with_matches: bool, - files_without_matches: bool, - line_numbers: bool, - word_regexp: bool, - line_regexp: bool, - max_count: Option, - quiet: bool, - patterns: Vec, - pattern_bytes: usize, - files: Vec, -} - -impl GrepOptions { - fn new(mode: GrepMode) -> Self { - Self { - mode, - ignore_case: false, - invert_match: false, - count_only: false, - files_with_matches: false, - files_without_matches: false, - line_numbers: false, - word_regexp: false, - line_regexp: false, - max_count: None, - quiet: false, - patterns: Vec::new(), - pattern_bytes: 0, - files: Vec::new(), - } - } -} - -fn run_grep(args: Vec, default_mode: GrepMode) -> i32 { - let str_args: Vec = args - .iter() - .skip(1) // skip argv[0] - .map(|a| a.to_string_lossy().to_string()) - .collect(); - - let opts = match parse_args(&str_args, default_mode) { - Ok(opts) => opts, - Err(msg) => { - eprintln!("grep: {}", msg); - return 2; - } - }; - - if opts.patterns.is_empty() { - eprintln!("grep: no pattern specified"); - return 2; - } - - let regex = match build_regex(&opts) { - Ok(r) => r, - Err(msg) => { - eprintln!("grep: {}", msg); - return 2; - } - }; - - let multiple_files = opts.files.len() > 1; - let mut any_match = false; - let mut had_error = false; - - if opts.files.is_empty() { - // Read from stdin - let stdin = io::stdin(); - let reader = stdin.lock(); - match search_reader(reader, None, ®ex, &opts, multiple_files) { - Ok(found) => any_match |= found, - Err(e) => { - eprintln!("grep: {}", e); - had_error = true; - } - } - } else { - for file in &opts.files { - if file == "-" { - let stdin = io::stdin(); - let reader = stdin.lock(); - let label = if multiple_files { - Some("(standard input)") - } else { - None - }; - match search_reader(reader, label, ®ex, &opts, multiple_files) { - Ok(found) => any_match |= found, - Err(e) => { - eprintln!("grep: {}: {}", file, e); - had_error = true; - } - } - } else { - match std::fs::File::open(file) { - Ok(f) => { - let reader = io::BufReader::new(f); - let label = if multiple_files { - Some(file.as_str()) - } else { - None - }; - match search_reader(reader, label, ®ex, &opts, multiple_files) { - Ok(found) => any_match |= found, - Err(e) => { - eprintln!("grep: {}: {}", file, e); - had_error = true; - } - } - } - Err(e) => { - eprintln!("grep: {}: {}", file, e); - had_error = true; - } - } - } - } - } - - if had_error { - 2 - } else if any_match { - 0 - } else { - 1 - } -} - -fn parse_args(args: &[String], default_mode: GrepMode) -> Result { - let mut opts = GrepOptions::new(default_mode); - let mut i = 0; - let mut pattern_from_args = false; - - while i < args.len() { - let arg = &args[i]; - - if arg == "--" { - i += 1; - // Remaining args are files (or first is pattern if none yet) - break; - } - - if arg.starts_with('-') && arg.len() > 1 && !arg.starts_with("--") { - let chars: Vec = arg[1..].chars().collect(); - let mut j = 0; - while j < chars.len() { - match chars[j] { - 'E' => opts.mode = GrepMode::Extended, - 'F' => opts.mode = GrepMode::Fixed, - 'G' => opts.mode = GrepMode::Basic, - 'i' | 'y' => opts.ignore_case = true, - 'v' => opts.invert_match = true, - 'c' => opts.count_only = true, - 'l' => opts.files_with_matches = true, - 'L' => opts.files_without_matches = true, - 'n' => opts.line_numbers = true, - 'w' => opts.word_regexp = true, - 'x' => opts.line_regexp = true, - 'q' | 's' => opts.quiet = true, - 'h' => {} // suppress filename (handled by multiple_files logic) - 'H' => {} // force filename - 'e' => { - // -e PATTERN (rest of this flag group or next arg) - let rest: String = chars[j + 1..].iter().collect(); - if !rest.is_empty() { - push_pattern(&mut opts, rest)?; - pattern_from_args = true; - j = chars.len(); // consumed rest - continue; - } else { - i += 1; - if i >= args.len() { - return Err("option requires an argument -- 'e'".to_string()); - } - push_pattern(&mut opts, args[i].clone())?; - pattern_from_args = true; - j = chars.len(); - continue; - } - } - 'f' => { - i += 1; - if i >= args.len() { - return Err("option requires an argument -- 'f'".to_string()); - } - read_patterns_from_file(&mut opts, &args[i])?; - pattern_from_args = true; - j = chars.len(); - continue; - } - 'm' => { - i += 1; - if i >= args.len() { - return Err("option requires an argument -- 'm'".to_string()); - } - opts.max_count = Some( - args[i] - .parse() - .map_err(|_| format!("invalid max count '{}'", args[i]))?, - ); - j = chars.len(); - continue; - } - _ => { - return Err(format!("invalid option -- '{}'", chars[j])); - } - } - j += 1; - } - } else if arg.starts_with("--") { - match arg.as_str() { - "--extended-regexp" => opts.mode = GrepMode::Extended, - "--fixed-strings" => opts.mode = GrepMode::Fixed, - "--basic-regexp" => opts.mode = GrepMode::Basic, - "--ignore-case" => opts.ignore_case = true, - "--invert-match" => opts.invert_match = true, - "--count" => opts.count_only = true, - "--files-with-matches" => opts.files_with_matches = true, - "--files-without-match" => opts.files_without_matches = true, - "--line-number" => opts.line_numbers = true, - "--word-regexp" => opts.word_regexp = true, - "--line-regexp" => opts.line_regexp = true, - "--quiet" | "--silent" => opts.quiet = true, - _ if arg.starts_with("--regexp=") => { - push_pattern(&mut opts, arg[9..].to_string())?; - pattern_from_args = true; - } - _ if arg.starts_with("--max-count=") => { - opts.max_count = Some( - arg[12..] - .parse() - .map_err(|_| format!("invalid max count '{}'", &arg[12..]))?, - ); - } - _ => { - return Err(format!("unrecognized option '{}'", arg)); - } - } - } else { - // Positional argument: first is pattern (if no -e), rest are files - if !pattern_from_args && opts.patterns.is_empty() { - push_pattern(&mut opts, arg.clone())?; - pattern_from_args = true; - } else { - opts.files.push(arg.clone()); - } - } - i += 1; - } - - // Remaining args after -- - while i < args.len() { - if !pattern_from_args && opts.patterns.is_empty() { - push_pattern(&mut opts, args[i].clone())?; - pattern_from_args = true; - } else { - opts.files.push(args[i].clone()); - } - i += 1; - } - - Ok(opts) -} - -fn push_pattern(opts: &mut GrepOptions, pattern: String) -> Result<(), String> { - if opts.patterns.len() >= MAX_PATTERNS { - return Err("too many patterns".to_string()); - } - let next_bytes = opts - .pattern_bytes - .checked_add(pattern.len()) - .ok_or_else(|| "pattern data too large".to_string())?; - if next_bytes > MAX_PATTERN_BYTES { - return Err("pattern data exceeds size limit".to_string()); - } - opts.pattern_bytes = next_bytes; - opts.patterns.push(pattern); - Ok(()) -} - -fn read_patterns_from_file(opts: &mut GrepOptions, path: &str) -> Result<(), String> { - let metadata = std::fs::metadata(path).map_err(|e| format!("{}: {}", path, e))?; - if metadata.len() > MAX_PATTERN_BYTES as u64 { - return Err(format!("{}: pattern file exceeds size limit", path)); - } - let file = std::fs::File::open(path).map_err(|e| format!("{}: {}", path, e))?; - let limit = MAX_PATTERN_BYTES - .checked_add(1) - .ok_or_else(|| "pattern file size limit is too large".to_string())?; - let mut content = String::new(); - file.take(limit as u64) - .read_to_string(&mut content) - .map_err(|e| format!("{}: {}", path, e))?; - if content.len() > MAX_PATTERN_BYTES { - return Err(format!("{}: pattern file exceeds size limit", path)); - } - for line in content.lines() { - if !line.is_empty() { - push_pattern(opts, line.to_string())?; - } - } - Ok(()) -} - -/// Build a compiled regex from the grep options. -fn build_regex(opts: &GrepOptions) -> Result { - let pattern = if opts.patterns.len() == 1 { - build_single_pattern(&opts.patterns[0], opts) - } else { - // Multiple patterns: combine with alternation - let parts: Vec = opts - .patterns - .iter() - .map(|p| format!("(?:{})", build_single_pattern(p, opts))) - .collect(); - parts.join("|") - }; - - let mut builder = regex::RegexBuilder::new(&pattern); - builder.case_insensitive(opts.ignore_case); - - builder - .build() - .map_err(|e| format!("invalid pattern: {}", e)) -} - -/// Convert a single pattern string to a regex pattern based on mode. -fn build_single_pattern(pattern: &str, opts: &GrepOptions) -> String { - let base = match opts.mode { - GrepMode::Fixed => regex::escape(pattern), - GrepMode::Basic => convert_bre_to_ere(pattern), - GrepMode::Extended => pattern.to_string(), - }; - - let wrapped = if opts.word_regexp { - format!(r"\b(?:{})\b", base) - } else if opts.line_regexp { - format!("^(?:{})$", base) - } else { - base - }; - - wrapped -} - -/// Convert POSIX Basic Regular Expression to Extended (Rust regex syntax). -/// In BRE: \(, \), \{, \}, \+, \?, \| are special; unescaped versions are literal. -/// In ERE (and Rust regex): (, ), {, }, +, ?, | are special without backslash. -fn convert_bre_to_ere(bre: &str) -> String { - let mut result = String::with_capacity(bre.len()); - let chars: Vec = bre.chars().collect(); - let mut i = 0; - - while i < chars.len() { - if chars[i] == '\\' && i + 1 < chars.len() { - match chars[i + 1] { - '(' => { - result.push('('); - i += 2; - } - ')' => { - result.push(')'); - i += 2; - } - '{' => { - result.push('{'); - i += 2; - } - '}' => { - result.push('}'); - i += 2; - } - '+' => { - result.push('+'); - i += 2; - } - '?' => { - result.push('?'); - i += 2; - } - '|' => { - result.push('|'); - i += 2; - } - '1'..='9' => { - // Backreference - not supported in Rust regex, pass through - result.push('\\'); - result.push(chars[i + 1]); - i += 2; - } - _ => { - result.push('\\'); - result.push(chars[i + 1]); - i += 2; - } - } - } else { - match chars[i] { - // In BRE, unescaped (, ), {, }, +, ? are literal - '(' => { - result.push_str("\\("); - i += 1; - } - ')' => { - result.push_str("\\)"); - i += 1; - } - '{' => { - result.push_str("\\{"); - i += 1; - } - '}' => { - result.push_str("\\}"); - i += 1; - } - _ => { - result.push(chars[i]); - i += 1; - } - } - } - } - - result -} - -/// Search a reader for matching lines. Returns true if any match was found. -fn search_reader( - reader: R, - filename: Option<&str>, - regex: &Regex, - opts: &GrepOptions, - show_filename: bool, -) -> io::Result { - let mut buf_reader = io::BufReader::new(reader); - let mut match_count: usize = 0; - let mut line_num: usize = 0; - let stdout = io::stdout(); - let mut out = stdout.lock(); - let mut line_buf = Vec::new(); - - while let Some(line) = read_line_bounded(&mut buf_reader, &mut line_buf)? { - line_num += 1; - - let is_match = regex.is_match(&line); - let is_match = if opts.invert_match { - !is_match - } else { - is_match - }; - - if is_match { - match_count += 1; - - if opts.quiet { - return Ok(true); - } - - if opts.files_with_matches { - if let Some(name) = filename { - writeln!(out, "{}", name)?; - } else { - writeln!(out, "(standard input)")?; - } - out.flush()?; - return Ok(true); - } - - if !opts.count_only && !opts.files_without_matches { - let prefix = match (show_filename, filename, opts.line_numbers) { - (true, Some(name), true) => format!("{}:{}:", name, line_num), - (true, Some(name), false) => format!("{}:", name), - (_, _, true) => format!("{}:", line_num), - _ => String::new(), - }; - writeln!(out, "{}{}", prefix, line)?; - } - - if let Some(max) = opts.max_count { - if match_count >= max { - break; - } - } - } - } - - if opts.count_only && !opts.quiet { - if show_filename { - if let Some(name) = filename { - writeln!(out, "{}:{}", name, match_count)?; - } else { - writeln!(out, "{}", match_count)?; - } - } else { - writeln!(out, "{}", match_count)?; - } - } - - if opts.files_without_matches && match_count == 0 { - if let Some(name) = filename { - writeln!(out, "{}", name)?; - } else { - writeln!(out, "(standard input)")?; - } - } - - out.flush()?; - - Ok(match_count > 0) -} - -fn read_line_bounded( - reader: &mut R, - line_buf: &mut Vec, -) -> io::Result> { - line_buf.clear(); - - loop { - let available = reader.fill_buf()?; - if available.is_empty() { - if line_buf.is_empty() { - return Ok(None); - } - break; - } - - let newline = available.iter().position(|&b| b == b'\n'); - let take = newline.map_or(available.len(), |pos| pos + 1); - let next_len = line_buf - .len() - .checked_add(take) - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "input line too long"))?; - if next_len > MAX_INPUT_LINE_BYTES { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "input line exceeds size limit", - )); - } - - line_buf.extend_from_slice(&available[..take]); - reader.consume(take); - if newline.is_some() { - break; - } - } - - if line_buf.ends_with(b"\n") { - line_buf.pop(); - if line_buf.ends_with(b"\r") { - line_buf.pop(); - } - } - - String::from_utf8(line_buf.clone()) - .map(Some) - .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e)) -} diff --git a/registry/native/crates/libs/grep/src/rg_cmd.rs b/registry/native/crates/libs/grep/src/rg_cmd.rs deleted file mode 100644 index 9449f3c678..0000000000 --- a/registry/native/crates/libs/grep/src/rg_cmd.rs +++ /dev/null @@ -1,1142 +0,0 @@ -//! rg (ripgrep) implementation using the regex crate (ripgrep's pure Rust regex engine). -//! -//! Provides ripgrep-compatible search. Uses the same regex engine as ripgrep. -//! POSIX grep/egrep/fgrep remain in lib.rs for BRE/ERE/fixed string compatibility. - -use std::collections::{HashSet, VecDeque}; -use std::ffi::OsString; -use std::io::{self, BufRead, Read, Write}; -use std::path::{Path, PathBuf}; - -use regex::{Regex, RegexBuilder}; - -const MAX_CONTEXT_LINES: usize = 100_000; -const MAX_CONTEXT_BYTES: usize = 16 * 1024 * 1024; -const MAX_FILE_RESULTS: usize = 1_000_000; -const MAX_INPUT_LINE_BYTES: usize = 16 * 1024 * 1024; -const MAX_PATTERN_BYTES: usize = 16 * 1024 * 1024; -const MAX_PATTERNS: usize = 100_000; - -/// Entry point for rg command. -pub fn rg(args: Vec) -> i32 { - let str_args: Vec = args - .iter() - .skip(1) - .map(|a| a.to_string_lossy().to_string()) - .collect(); - - match run(&str_args) { - Ok(code) => code, - Err(msg) => { - eprintln!("rg: {}", msg); - 2 - } - } -} - -struct Options { - patterns: Vec, - paths: Vec, - files_mode: bool, - ignore_case: bool, - smart_case: bool, - invert_match: bool, - count_only: bool, - files_with_matches: bool, - files_without_matches: bool, - line_numbers: Option, - word_regexp: bool, - line_regexp: bool, - fixed_strings: bool, - max_count: Option, - quiet: bool, - only_matching: bool, - after_context: usize, - before_context: usize, - show_filename: Option, - hidden: bool, - max_depth: Option, - sort_modified: bool, - glob_patterns: Vec, - pattern_bytes: usize, - type_include: Vec, - type_exclude: Vec, -} - -impl Options { - fn new() -> Self { - Self { - patterns: Vec::new(), - paths: Vec::new(), - files_mode: false, - ignore_case: false, - smart_case: true, - invert_match: false, - count_only: false, - files_with_matches: false, - files_without_matches: false, - line_numbers: None, - word_regexp: false, - line_regexp: false, - fixed_strings: false, - max_count: None, - quiet: false, - only_matching: false, - after_context: 0, - before_context: 0, - show_filename: None, - hidden: false, - max_depth: None, - sort_modified: false, - glob_patterns: Vec::new(), - pattern_bytes: 0, - type_include: Vec::new(), - type_exclude: Vec::new(), - } - } - - fn show_line_numbers(&self) -> bool { - self.line_numbers.unwrap_or(true) - } - - fn resolve_show_filename(&self, multi: bool) -> bool { - self.show_filename.unwrap_or(multi) - } - - fn has_context(&self) -> bool { - self.before_context > 0 || self.after_context > 0 - } -} - -fn run(args: &[String]) -> Result { - if args.len() == 1 && (args[0] == "--version" || args[0] == "-V") { - let stdout = io::stdout(); - let mut out = stdout.lock(); - writeln!(out, "ripgrep 14.1.0 (secure-exec)").map_err(|e| e.to_string())?; - out.flush().map_err(|e| e.to_string())?; - return Ok(0); - } - - let opts = parse_args(args)?; - - if opts.files_mode { - let paths = if opts.paths.is_empty() { - vec![".".to_string()] - } else { - opts.paths.clone() - }; - - let files = collect_files_from_paths(&paths, &opts).map_err(|e| e.to_string())?; - let stdout = io::stdout(); - let mut out = stdout.lock(); - for path in files { - writeln!(out, "{}", path.to_string_lossy()).map_err(|e| e.to_string())?; - } - out.flush().map_err(|e| e.to_string())?; - return Ok(0); - } - - if opts.patterns.is_empty() { - return Err("no pattern provided".to_string()); - } - - let regex = build_regex(&opts)?; - - if opts.paths.is_empty() { - // No paths: read from stdin - let stdin = io::stdin(); - let stdout = io::stdout(); - let mut out = stdout.lock(); - let result = search_stream(stdin.lock(), ®ex, &opts, None, false, &mut out) - .map_err(|e| e.to_string())?; - if opts.quiet { - return Ok(if result.matches > 0 { 0 } else { 1 }); - } - print_file_result(None, &result, &opts, &mut out).map_err(|e| e.to_string())?; - out.flush().map_err(|e| e.to_string())?; - return Ok(if result.matches > 0 { 0 } else { 1 }); - } - - let files = collect_files_from_paths(&opts.paths, &opts).map_err(|e| e.to_string())?; - let multi = files.len() > 1; - let show_fn = opts.resolve_show_filename(multi); - let mut any_match = false; - let mut had_error = false; - let stdout = io::stdout(); - let mut out = stdout.lock(); - - for path in &files { - match std::fs::File::open(path) { - Ok(f) => { - let reader = io::BufReader::new(f); - let fname = if show_fn { - Some(path.to_string_lossy().to_string()) - } else { - None - }; - let result = - match search_stream(reader, ®ex, &opts, fname.as_deref(), show_fn, &mut out) - { - Ok(result) => result, - Err(e) => { - eprintln!("rg: {}: {}", path.display(), e); - had_error = true; - continue; - } - }; - if result.matches > 0 { - any_match = true; - } - if opts.quiet && any_match { - return Ok(0); - } - if !opts.quiet { - print_file_result(fname.as_deref(), &result, &opts, &mut out) - .map_err(|e| e.to_string())?; - } - } - Err(e) => { - eprintln!("rg: {}: {}", path.display(), e); - had_error = true; - } - } - } - out.flush().map_err(|e| e.to_string())?; - - if had_error { - Ok(2) - } else if any_match { - Ok(0) - } else { - Ok(1) - } -} - -// --- Argument parsing --- - -fn parse_args(args: &[String]) -> Result { - let mut opts = Options::new(); - let mut i = 0; - let mut explicit_pattern = false; - - while i < args.len() { - let arg = &args[i]; - - if arg == "--" { - i += 1; - break; - } - - // Long options - if arg.starts_with("--") { - match arg.as_str() { - "--ignore-case" => opts.ignore_case = true, - "--case-sensitive" => { - opts.ignore_case = false; - opts.smart_case = false; - } - "--smart-case" => opts.smart_case = true, - "--invert-match" => opts.invert_match = true, - "--count" => opts.count_only = true, - "--files" => opts.files_mode = true, - "--files-with-matches" => opts.files_with_matches = true, - "--files-without-match" => opts.files_without_matches = true, - "--line-number" => opts.line_numbers = Some(true), - "--no-line-number" => opts.line_numbers = Some(false), - "--word-regexp" => opts.word_regexp = true, - "--line-regexp" => opts.line_regexp = true, - "--fixed-strings" => opts.fixed_strings = true, - "--quiet" | "--silent" => opts.quiet = true, - "--only-matching" => opts.only_matching = true, - "--hidden" | "--no-ignore" => opts.hidden = true, - "--follow" | "--no-ignore-vcs" | "--no-ignore-parent" => {} - "--with-filename" => opts.show_filename = Some(true), - "--no-filename" => opts.show_filename = Some(false), - "--no-heading" | "--heading" => {} // no-op (we always use inline format) - "--color=auto" | "--color=always" | "--color=never" => {} // no-op in WASI - "--no-color" => {} - _ if arg.starts_with("--color=") => {} - _ if arg.starts_with("--max-depth=") => { - opts.max_depth = Some( - arg[12..] - .parse() - .map_err(|_| format!("invalid number: '{}'", &arg[12..]))?, - ); - } - _ if arg.starts_with("--sort=") => match &arg[7..] { - "modified" => opts.sort_modified = true, - value => return Err(format!("unsupported sort: '{}'", value)), - }, - _ if arg.starts_with("--threads=") => {} - _ if arg.starts_with("--regexp=") => { - push_pattern(&mut opts, arg[9..].to_string())?; - explicit_pattern = true; - } - _ if arg.starts_with("--max-count=") => { - opts.max_count = Some( - arg[12..] - .parse() - .map_err(|_| format!("invalid number: '{}'", &arg[12..]))?, - ); - } - _ if arg.starts_with("--after-context=") => { - opts.after_context = parse_context_count(&arg[16..])?; - } - _ if arg.starts_with("--before-context=") => { - opts.before_context = parse_context_count(&arg[17..])?; - } - _ if arg.starts_with("--context=") => { - let n = parse_context_count(&arg[10..])?; - opts.before_context = n; - opts.after_context = n; - } - _ if arg.starts_with("--glob=") => { - opts.glob_patterns.push(arg[7..].to_string()); - } - _ if arg.starts_with("--type=") => { - opts.type_include.push(arg[7..].to_string()); - } - _ if arg.starts_with("--type-not=") => { - opts.type_exclude.push(arg[11..].to_string()); - } - "--regexp" | "--max-count" | "--after-context" | "--before-context" - | "--context" | "--glob" | "--type" | "--type-not" | "--file" | "--color" - | "--max-depth" | "--threads" => { - i += 1; - if i >= args.len() { - return Err(format!("{} requires an argument", arg)); - } - match arg.as_str() { - "--regexp" => { - push_pattern(&mut opts, args[i].clone())?; - explicit_pattern = true; - } - "--max-count" => { - opts.max_count = Some( - args[i] - .parse() - .map_err(|_| format!("invalid number: '{}'", args[i]))?, - ); - } - "--after-context" => { - opts.after_context = parse_context_count(&args[i])?; - } - "--before-context" => { - opts.before_context = parse_context_count(&args[i])?; - } - "--context" => { - let n = parse_context_count(&args[i])?; - opts.before_context = n; - opts.after_context = n; - } - "--glob" => opts.glob_patterns.push(args[i].clone()), - "--type" => opts.type_include.push(args[i].clone()), - "--type-not" => opts.type_exclude.push(args[i].clone()), - "--file" => { - read_patterns_from_file(&mut opts, &args[i])?; - explicit_pattern = true; - } - "--color" => {} // no-op - "--max-depth" => { - opts.max_depth = Some( - args[i] - .parse() - .map_err(|_| format!("invalid number: '{}'", args[i]))?, - ); - } - "--threads" => {} // no-op - _ => unreachable!(), - } - } - _ => return Err(format!("unrecognized option '{}'", arg)), - } - i += 1; - continue; - } - - // Short options - if arg.starts_with('-') && arg.len() > 1 { - let chars: Vec = arg[1..].chars().collect(); - let mut j = 0; - while j < chars.len() { - match chars[j] { - 'i' => opts.ignore_case = true, - 's' => { - opts.ignore_case = false; - opts.smart_case = false; - } - 'S' => opts.smart_case = true, - 'v' => opts.invert_match = true, - 'c' => opts.count_only = true, - 'l' => opts.files_with_matches = true, - 'n' => opts.line_numbers = Some(true), - 'N' => opts.line_numbers = Some(false), - 'w' => opts.word_regexp = true, - 'x' => opts.line_regexp = true, - 'F' => opts.fixed_strings = true, - 'q' => opts.quiet = true, - 'o' => opts.only_matching = true, - 'H' => opts.show_filename = Some(true), - '.' => opts.hidden = true, - 'e' => { - let rest: String = chars[j + 1..].iter().collect(); - if !rest.is_empty() { - push_pattern(&mut opts, rest)?; - } else { - i += 1; - if i >= args.len() { - return Err("option requires an argument -- 'e'".to_string()); - } - push_pattern(&mut opts, args[i].clone())?; - } - explicit_pattern = true; - j = chars.len(); - continue; - } - 'f' => { - i += 1; - if i >= args.len() { - return Err("option requires an argument -- 'f'".to_string()); - } - read_patterns_from_file(&mut opts, &args[i])?; - explicit_pattern = true; - j = chars.len(); - continue; - } - 'm' => { - i += 1; - if i >= args.len() { - return Err("option requires an argument -- 'm'".to_string()); - } - opts.max_count = Some( - args[i] - .parse() - .map_err(|_| format!("invalid number: '{}'", args[i]))?, - ); - j = chars.len(); - continue; - } - 'j' => { - i += 1; - if i >= args.len() { - return Err("option requires an argument -- 'j'".to_string()); - } - j = chars.len(); - continue; - } - 'A' => { - i += 1; - if i >= args.len() { - return Err("option requires an argument -- 'A'".to_string()); - } - opts.after_context = parse_context_count(&args[i])?; - j = chars.len(); - continue; - } - 'B' => { - i += 1; - if i >= args.len() { - return Err("option requires an argument -- 'B'".to_string()); - } - opts.before_context = parse_context_count(&args[i])?; - j = chars.len(); - continue; - } - 'C' => { - i += 1; - if i >= args.len() { - return Err("option requires an argument -- 'C'".to_string()); - } - let n = parse_context_count(&args[i])?; - opts.before_context = n; - opts.after_context = n; - j = chars.len(); - continue; - } - 'g' => { - i += 1; - if i >= args.len() { - return Err("option requires an argument -- 'g'".to_string()); - } - opts.glob_patterns.push(args[i].clone()); - j = chars.len(); - continue; - } - 't' => { - i += 1; - if i >= args.len() { - return Err("option requires an argument -- 't'".to_string()); - } - opts.type_include.push(args[i].clone()); - j = chars.len(); - continue; - } - 'T' => { - i += 1; - if i >= args.len() { - return Err("option requires an argument -- 'T'".to_string()); - } - opts.type_exclude.push(args[i].clone()); - j = chars.len(); - continue; - } - _ => return Err(format!("invalid option -- '{}'", chars[j])), - } - j += 1; - } - i += 1; - continue; - } - - // Positional argument - if opts.files_mode { - opts.paths.push(arg.clone()); - } else if !explicit_pattern && opts.patterns.is_empty() { - push_pattern(&mut opts, arg.clone())?; - explicit_pattern = true; - } else { - opts.paths.push(arg.clone()); - } - i += 1; - } - - // Remaining args after -- - while i < args.len() { - if opts.files_mode { - opts.paths.push(args[i].clone()); - } else if !explicit_pattern && opts.patterns.is_empty() { - push_pattern(&mut opts, args[i].clone())?; - explicit_pattern = true; - } else { - opts.paths.push(args[i].clone()); - } - i += 1; - } - - Ok(opts) -} - -fn parse_context_count(value: &str) -> Result { - let count: usize = value - .parse() - .map_err(|_| format!("invalid number: '{}'", value))?; - if count > MAX_CONTEXT_LINES { - return Err(format!("context count '{}' exceeds size limit", value)); - } - Ok(count) -} - -fn push_pattern(opts: &mut Options, pattern: String) -> Result<(), String> { - if opts.patterns.len() >= MAX_PATTERNS { - return Err("too many patterns".to_string()); - } - let next_bytes = opts - .pattern_bytes - .checked_add(pattern.len()) - .ok_or_else(|| "pattern data too large".to_string())?; - if next_bytes > MAX_PATTERN_BYTES { - return Err("pattern data exceeds size limit".to_string()); - } - opts.pattern_bytes = next_bytes; - opts.patterns.push(pattern); - Ok(()) -} - -fn read_patterns_from_file(opts: &mut Options, path: &str) -> Result<(), String> { - let metadata = std::fs::metadata(path).map_err(|e| format!("{}: {}", path, e))?; - if metadata.len() > MAX_PATTERN_BYTES as u64 { - return Err(format!("{}: pattern file exceeds size limit", path)); - } - let file = std::fs::File::open(path).map_err(|e| format!("{}: {}", path, e))?; - let limit = MAX_PATTERN_BYTES - .checked_add(1) - .ok_or_else(|| "pattern file size limit is too large".to_string())?; - let mut content = String::new(); - file.take(limit as u64) - .read_to_string(&mut content) - .map_err(|e| format!("{}: {}", path, e))?; - if content.len() > MAX_PATTERN_BYTES { - return Err(format!("{}: pattern file exceeds size limit", path)); - } - for line in content.lines() { - if !line.is_empty() { - push_pattern(opts, line.to_string())?; - } - } - Ok(()) -} - -// --- Pattern building --- - -fn build_regex(opts: &Options) -> Result { - let combined = if opts.patterns.len() == 1 { - prepare_pattern(&opts.patterns[0], opts) - } else { - let parts: Vec = opts - .patterns - .iter() - .map(|p| format!("(?:{})", prepare_pattern(p, opts))) - .collect(); - parts.join("|") - }; - - let case_insensitive = if opts.ignore_case { - true - } else if opts.smart_case { - // Smart case: insensitive unless pattern has uppercase - !combined.chars().any(|c| c.is_uppercase()) - } else { - false - }; - - RegexBuilder::new(&combined) - .case_insensitive(case_insensitive) - .build() - .map_err(|e| format!("regex error: {}", e)) -} - -fn prepare_pattern(pattern: &str, opts: &Options) -> String { - let base = if opts.fixed_strings { - regex::escape(pattern) - } else { - pattern.to_string() - }; - - if opts.word_regexp { - format!(r"\b(?:{})\b", base) - } else if opts.line_regexp { - format!("^(?:{})$", base) - } else { - base - } -} - -// --- File collection --- - -fn collect_files_from_paths(paths: &[String], opts: &Options) -> io::Result> { - let mut files = Vec::new(); - for path_str in paths { - let path = Path::new(path_str); - let metadata = std::fs::symlink_metadata(path)?; - let file_type = metadata.file_type(); - if file_type.is_dir() { - let mut active_dirs = HashSet::new(); - walk_dir(path, path, opts, &mut files, 0, &mut active_dirs)?; - } else if file_type.is_file() { - if should_include(path, path, false, opts) { - push_collected_file(&mut files, path.to_path_buf())?; - } - } else { - continue; - } - } - if opts.sort_modified { - files.sort_by_key(|path| { - std::fs::metadata(path) - .and_then(|meta| meta.modified()) - .ok() - }); - } else { - files.sort(); - } - Ok(files) -} - -fn push_collected_file(out: &mut Vec, path: PathBuf) -> io::Result<()> { - if out.len() >= MAX_FILE_RESULTS { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "file result count exceeds size limit", - )); - } - out.push(path); - Ok(()) -} - -fn walk_dir( - root: &Path, - dir: &Path, - opts: &Options, - out: &mut Vec, - depth: usize, - active_dirs: &mut HashSet, -) -> io::Result<()> { - let canonical = std::fs::canonicalize(dir)?; - if !active_dirs.insert(canonical.clone()) { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!("recursive directory cycle at {}", dir.display()), - )); - } - - let result = (|| { - for entry in std::fs::read_dir(dir)? { - let entry = entry?; - let path = entry.path(); - let name = entry.file_name(); - let name_str = name.to_string_lossy(); - let file_type = entry.file_type()?; - let is_dir = file_type.is_dir(); - - // Skip hidden files/dirs unless --hidden - if !opts.hidden && name_str.starts_with('.') { - continue; - } - - if !should_include(root, &path, is_dir, opts) { - continue; - } - - if is_dir { - if opts.max_depth.map(|max| depth < max).unwrap_or(true) { - walk_dir(root, &path, opts, out, depth + 1, active_dirs)?; - } - } else if file_type.is_file() { - push_collected_file(out, path)?; - } - } - Ok(()) - })(); - - active_dirs.remove(&canonical); - result -} - -fn should_include(root: &Path, path: &Path, is_dir: bool, opts: &Options) -> bool { - let ext = path - .extension() - .map(|e| e.to_string_lossy().to_string()) - .unwrap_or_default(); - let relative_path = path - .strip_prefix(root) - .unwrap_or(path) - .to_string_lossy() - .replace('\\', "/"); - let file_name = path - .file_name() - .map(|n| n.to_string_lossy().to_string()) - .unwrap_or_default(); - - // Type include filters - if !opts.type_include.is_empty() { - let included = opts.type_include.iter().any(|t| { - type_extensions(t) - .map(|exts| exts.iter().any(|e| ext == *e)) - .unwrap_or(false) - }); - if !included { - return false; - } - } - - // Type exclude filters - if !opts.type_exclude.is_empty() { - let excluded = opts.type_exclude.iter().any(|t| { - type_extensions(t) - .map(|exts| exts.iter().any(|e| ext == *e)) - .unwrap_or(false) - }); - if excluded { - return false; - } - } - - // Glob filters - if !opts.glob_patterns.is_empty() { - for pattern in &opts.glob_patterns { - let (negated, pat) = if let Some(rest) = pattern.strip_prefix('!') { - (true, rest) - } else { - (false, pattern.as_str()) - }; - let matches = glob_matches(pat, &relative_path, &file_name, is_dir); - if negated && matches { - return false; - } - if !negated && !matches { - return false; - } - } - } - - true -} - -fn type_extensions(type_name: &str) -> Option<&'static [&'static str]> { - match type_name { - "rust" | "rs" => Some(&["rs"]), - "py" | "python" => Some(&["py", "pyi"]), - "js" | "javascript" => Some(&["js", "jsx", "mjs"]), - "ts" | "typescript" => Some(&["ts", "tsx", "mts"]), - "c" => Some(&["c", "h"]), - "cpp" | "c++" => Some(&["cpp", "cxx", "cc", "hpp", "hxx", "h"]), - "java" => Some(&["java"]), - "go" => Some(&["go"]), - "html" => Some(&["html", "htm"]), - "css" => Some(&["css"]), - "json" => Some(&["json"]), - "yaml" | "yml" => Some(&["yml", "yaml"]), - "toml" => Some(&["toml"]), - "md" | "markdown" => Some(&["md", "markdown"]), - "txt" | "text" => Some(&["txt"]), - "sh" | "shell" | "bash" => Some(&["sh", "bash"]), - "xml" => Some(&["xml"]), - "sql" => Some(&["sql"]), - "lua" => Some(&["lua"]), - "ruby" | "rb" => Some(&["rb"]), - "php" => Some(&["php"]), - "swift" => Some(&["swift"]), - "kotlin" | "kt" => Some(&["kt", "kts"]), - _ => None, - } -} - -fn glob_matches(pattern: &str, relative_path: &str, file_name: &str, is_dir: bool) -> bool { - let normalized = pattern.trim_end_matches('/'); - if pattern.ends_with('/') { - return is_dir - && relative_path - .split('/') - .any(|segment| segment == normalized || segment == file_name); - } - - let target = if pattern.contains('/') { - relative_path - } else { - file_name - }; - - let mut regex_pattern = String::from("^"); - let chars: Vec = normalized.chars().collect(); - let mut i = 0; - while i < chars.len() { - match chars[i] { - '*' => { - if i + 1 < chars.len() && chars[i + 1] == '*' { - regex_pattern.push_str(".*"); - i += 2; - } else { - regex_pattern.push_str("[^/]*"); - i += 1; - } - } - '?' => { - regex_pattern.push_str("[^/]"); - i += 1; - } - '{' => { - if let Some(end) = chars[i + 1..].iter().position(|c| *c == '}') { - let group: String = chars[i + 1..i + 1 + end].iter().collect(); - regex_pattern.push('('); - regex_pattern.push_str( - &group - .split(',') - .map(regex::escape) - .collect::>() - .join("|"), - ); - regex_pattern.push(')'); - i += end + 2; - } else { - regex_pattern.push_str("\\{"); - i += 1; - } - } - '.' | '+' | '(' | ')' | '|' | '^' | '$' | '[' | ']' | '\\' => { - regex_pattern.push('\\'); - regex_pattern.push(chars[i]); - i += 1; - } - other => { - regex_pattern.push(other); - i += 1; - } - } - } - regex_pattern.push('$'); - - Regex::new(®ex_pattern) - .map(|regex| regex.is_match(target)) - .unwrap_or(false) -} - -// --- Search --- - -struct FileResult { - matches: usize, - is_binary: bool, -} - -enum ResultLine { - Match(usize, String), - Context(usize, String), - Separator, -} - -fn search_stream( - mut reader: R, - regex: &Regex, - opts: &Options, - filename: Option<&str>, - show_filename: bool, - out: &mut W, -) -> io::Result { - let mut result = FileResult { - matches: 0, - is_binary: false, - }; - - let collect_lines = - !opts.quiet && !opts.files_with_matches && !opts.files_without_matches && !opts.count_only; - - let mut before_buf: VecDeque<(usize, String)> = VecDeque::new(); - let mut before_buf_bytes: usize = 0; - let mut after_remaining: usize = 0; - let mut last_printed: usize = 0; - let mut line_buf = Vec::new(); - let mut lineno: usize = 0; - - while let Some(line) = read_line_bounded(&mut reader, &mut line_buf)? { - lineno = lineno - .checked_add(1) - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "line number overflow"))?; - - // Binary detection: null bytes in line data - if line.as_bytes().contains(&0) { - result.is_binary = true; - break; - } - - let is_match = regex.is_match(&line) != opts.invert_match; - - if is_match { - result.matches += 1; - - if opts.quiet || opts.files_with_matches { - break; - } - - if collect_lines { - // Separator for non-contiguous match groups - if opts.has_context() && last_printed > 0 { - let first_before = before_buf.front().map(|(n, _)| *n).unwrap_or(lineno); - if first_before > last_printed + 1 { - print_result_line( - out, - filename, - opts, - show_filename, - &ResultLine::Separator, - )?; - } - } - - // Flush before-context buffer - for (bno, btext) in before_buf.drain(..) { - if bno > last_printed { - print_result_line( - out, - filename, - opts, - show_filename, - &ResultLine::Context(bno, btext), - )?; - last_printed = bno; - } - } - before_buf_bytes = 0; - - // Emit match - if opts.only_matching && !opts.invert_match { - for mat in regex.find_iter(&line) { - print_result_line( - out, - filename, - opts, - show_filename, - &ResultLine::Match(lineno, mat.as_str().to_string()), - )?; - } - } else { - print_result_line( - out, - filename, - opts, - show_filename, - &ResultLine::Match(lineno, line), - )?; - } - last_printed = lineno; - after_remaining = opts.after_context; - } - - if let Some(max) = opts.max_count { - if result.matches >= max { - break; - } - } - } else if collect_lines { - if after_remaining > 0 { - print_result_line( - out, - filename, - opts, - show_filename, - &ResultLine::Context(lineno, line), - )?; - last_printed = lineno; - after_remaining -= 1; - } else if opts.before_context > 0 { - before_buf_bytes = before_buf_bytes.checked_add(line.len()).ok_or_else(|| { - io::Error::new(io::ErrorKind::InvalidData, "context buffer too large") - })?; - if before_buf_bytes > MAX_CONTEXT_BYTES { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "context buffer exceeds size limit", - )); - } - before_buf.push_back((lineno, line)); - if before_buf.len() > opts.before_context { - if let Some((_, removed)) = before_buf.pop_front() { - before_buf_bytes = before_buf_bytes.saturating_sub(removed.len()); - } - } - } - } - } - - Ok(result) -} - -// --- Output --- - -fn print_file_result( - filename: Option<&str>, - result: &FileResult, - opts: &Options, - out: &mut W, -) -> io::Result<()> { - if result.is_binary { - if result.matches > 0 { - if let Some(name) = filename { - writeln!(out, "Binary file {} matches.", name)?; - } - } - return Ok(()); - } - - if opts.files_with_matches { - if result.matches > 0 { - let name = filename.unwrap_or("(standard input)"); - writeln!(out, "{}", name)?; - } - return Ok(()); - } - - if opts.files_without_matches { - if result.matches == 0 { - let name = filename.unwrap_or("(standard input)"); - writeln!(out, "{}", name)?; - } - return Ok(()); - } - - if opts.count_only { - if let Some(name) = filename { - writeln!(out, "{}:{}", name, result.matches)?; - } else { - writeln!(out, "{}", result.matches)?; - } - return Ok(()); - } - - Ok(()) -} - -fn print_result_line( - out: &mut W, - filename: Option<&str>, - opts: &Options, - show_filename: bool, - line: &ResultLine, -) -> io::Result<()> { - match line { - ResultLine::Match(lineno, text) => { - let mut prefix = String::new(); - if show_filename { - if let Some(name) = filename { - prefix.push_str(name); - prefix.push(':'); - } - } - if opts.show_line_numbers() { - prefix.push_str(&lineno.to_string()); - prefix.push(':'); - } - writeln!(out, "{}{}", prefix, text) - } - ResultLine::Context(lineno, text) => { - let mut prefix = String::new(); - if show_filename { - if let Some(name) = filename { - prefix.push_str(name); - prefix.push('-'); - } - } - if opts.show_line_numbers() { - prefix.push_str(&lineno.to_string()); - prefix.push('-'); - } - writeln!(out, "{}{}", prefix, text) - } - ResultLine::Separator => writeln!(out, "--"), - } -} - -fn read_line_bounded( - reader: &mut R, - line_buf: &mut Vec, -) -> io::Result> { - line_buf.clear(); - - loop { - let available = reader.fill_buf()?; - if available.is_empty() { - if line_buf.is_empty() { - return Ok(None); - } - break; - } - - let newline = available.iter().position(|&b| b == b'\n'); - let take = newline.map_or(available.len(), |pos| pos + 1); - let next_len = line_buf - .len() - .checked_add(take) - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "input line too long"))?; - if next_len > MAX_INPUT_LINE_BYTES { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "input line exceeds size limit", - )); - } - - line_buf.extend_from_slice(&available[..take]); - reader.consume(take); - if newline.is_some() { - break; - } - } - - if line_buf.ends_with(b"\n") { - line_buf.pop(); - if line_buf.ends_with(b"\r") { - line_buf.pop(); - } - } - - String::from_utf8(line_buf.clone()) - .map(Some) - .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e)) -} diff --git a/registry/native/crates/libs/gzip/Cargo.toml b/registry/native/crates/libs/gzip/Cargo.toml deleted file mode 100644 index eed27c5c29..0000000000 --- a/registry/native/crates/libs/gzip/Cargo.toml +++ /dev/null @@ -1,9 +0,0 @@ -[package] -name = "secureexec-gzip" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "gzip/gunzip/zcat implementations for secure-exec standalone binaries" - -[dependencies] -flate2 = { version = "1.0", default-features = false, features = ["rust_backend"] } diff --git a/registry/native/crates/libs/jq/Cargo.toml b/registry/native/crates/libs/jq/Cargo.toml deleted file mode 100644 index a258274ae3..0000000000 --- a/registry/native/crates/libs/jq/Cargo.toml +++ /dev/null @@ -1,12 +0,0 @@ -[package] -name = "secureexec-jq" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "jq implementation for secure-exec standalone binaries" - -[dependencies] -jaq-core = "2.2" -jaq-std = "2.1" -jaq-json = { version = "1.1", features = ["serde_json"] } -serde_json = "1" diff --git a/registry/native/crates/libs/jq/src/lib.rs b/registry/native/crates/libs/jq/src/lib.rs deleted file mode 100644 index 9fe36c8542..0000000000 --- a/registry/native/crates/libs/jq/src/lib.rs +++ /dev/null @@ -1,293 +0,0 @@ -//! jq implementation using the jaq crate (pure Rust, jq-compatible). -//! -//! Wraps jaq-core/jaq-std/jaq-json to provide a standard jq CLI interface. - -use std::ffi::OsString; -use std::io::{self, Read, Write}; - -use jaq_core::load::{Arena, File, Loader}; -use jaq_core::{Compiler, Ctx, RcIter}; -use jaq_json::Val; - -const MAX_INPUT_BYTES: usize = 16 * 1024 * 1024; -const MAX_INPUT_VALUES: usize = 100_000; - -/// Entry point for jq command. -pub fn main(args: Vec) -> i32 { - let str_args: Vec = args - .iter() - .skip(1) // skip argv[0] - .map(|a| a.to_string_lossy().to_string()) - .collect(); - - match run_jq(&str_args) { - Ok(code) => code, - Err(msg) => { - eprintln!("jq: {}", msg); - 2 - } - } -} - -struct JqOptions { - filter: String, - raw_output: bool, - raw_input: bool, - slurp: bool, - compact: bool, - null_input: bool, - exit_status: bool, - join_output: bool, - args: Vec<(String, String)>, - jsonargs: Vec<(String, Val)>, -} - -fn parse_args(args: &[String]) -> Result { - let mut opts = JqOptions { - filter: String::new(), - raw_output: false, - raw_input: false, - slurp: false, - compact: false, - null_input: false, - exit_status: false, - join_output: false, - args: Vec::new(), - jsonargs: Vec::new(), - }; - - let mut filter_set = false; - let mut i = 0; - - while i < args.len() { - let arg = &args[i]; - - if arg == "--" { - break; - } - - if arg.starts_with('-') && arg.len() > 1 && !arg.starts_with("--") { - // Handle combined short flags like -rn, -crS, etc. - let flags = &arg[1..]; - for c in flags.chars() { - match c { - 'r' => opts.raw_output = true, - 'R' => opts.raw_input = true, - 's' => opts.slurp = true, - 'c' => opts.compact = true, - 'n' => opts.null_input = true, - 'e' => opts.exit_status = true, - 'j' => opts.join_output = true, - 'S' => {} // sort keys - jaq sorts by default - _ => return Err(format!("unknown option: -{}", c)), - } - } - } else if arg == "--raw-output" || arg == "--raw-output0" { - opts.raw_output = true; - } else if arg == "--raw-input" { - opts.raw_input = true; - } else if arg == "--slurp" { - opts.slurp = true; - } else if arg == "--compact-output" { - opts.compact = true; - } else if arg == "--null-input" { - opts.null_input = true; - } else if arg == "--exit-status" { - opts.exit_status = true; - } else if arg == "--join-output" { - opts.join_output = true; - } else if arg == "--arg" { - if i + 2 >= args.len() { - return Err("--arg requires name and value".to_string()); - } - let name = args[i + 1].clone(); - let value = args[i + 2].clone(); - opts.args.push((name, value)); - i += 2; - } else if arg == "--argjson" { - if i + 2 >= args.len() { - return Err("--argjson requires name and value".to_string()); - } - let name = args[i + 1].clone(); - let json_str = &args[i + 2]; - let value: serde_json::Value = serde_json::from_str(json_str) - .map_err(|e| format!("invalid JSON for --argjson: {}", e))?; - opts.jsonargs.push((name, Val::from(value))); - i += 2; - } else if !filter_set { - opts.filter = arg.clone(); - filter_set = true; - } else { - return Err(format!("unexpected argument: {}", arg)); - } - - i += 1; - } - - if !filter_set { - return Err("no filter provided".to_string()); - } - - Ok(opts) -} - -fn read_inputs(opts: &JqOptions) -> Result, String> { - if opts.null_input { - return Ok(vec![Val::from(serde_json::Value::Null)]); - } - - let mut stdin_data = String::new(); - io::stdin() - .take((MAX_INPUT_BYTES + 1) as u64) - .read_to_string(&mut stdin_data) - .map_err(|e| format!("failed to read stdin: {}", e))?; - if stdin_data.len() > MAX_INPUT_BYTES { - return Err("stdin exceeds size limit".to_string()); - } - - if opts.raw_input { - if opts.slurp { - let mut arr = Vec::new(); - for line in stdin_data.lines() { - push_input_value(&mut arr, serde_json::Value::String(line.to_string()))?; - } - Ok(vec![Val::from(serde_json::Value::Array(arr))]) - } else { - let mut lines = Vec::new(); - for line in stdin_data.lines() { - push_input_value( - &mut lines, - Val::from(serde_json::Value::String(line.to_string())), - )?; - } - Ok(lines) - } - } else { - let trimmed = stdin_data.trim(); - if trimmed.is_empty() { - return Ok(vec![Val::from(serde_json::Value::Null)]); - } - - let mut values = Vec::new(); - let decoder = serde_json::Deserializer::from_str(trimmed).into_iter::(); - for result in decoder { - let value = result.map_err(|e| format!("invalid JSON input: {}", e))?; - push_input_value(&mut values, value)?; - } - - if opts.slurp { - Ok(vec![Val::from(serde_json::Value::Array(values))]) - } else { - Ok(values.into_iter().map(Val::from).collect()) - } - } -} - -fn push_input_value(values: &mut Vec, value: T) -> Result<(), String> { - if values.len() >= MAX_INPUT_VALUES { - return Err("too many input values".to_string()); - } - values.push(value); - Ok(()) -} - -/// Format a jaq Val as a string for output. -fn format_output(val: &Val, opts: &JqOptions) -> Result { - let compact_str = format!("{}", val); - - // For raw output, unquote strings - if opts.raw_output { - if compact_str.starts_with('"') && compact_str.ends_with('"') && compact_str.len() >= 2 { - if let Ok(unescaped) = serde_json::from_str::(&compact_str) { - return Ok(unescaped); - } - } - } - - if opts.compact { - Ok(compact_str) - } else { - // Pretty print via serde_json - if let Ok(v) = serde_json::from_str::(&compact_str) { - serde_json::to_string_pretty(&v).map_err(|e| format!("output format error: {}", e)) - } else { - Ok(compact_str) - } - } -} - -fn run_jq(args: &[String]) -> Result { - let opts = parse_args(args)?; - let inputs = read_inputs(&opts)?; - - // Set up variable bindings for --arg and --argjson - let mut var_vals: Vec = Vec::new(); - - for (_name, value) in &opts.args { - var_vals.push(Val::from(serde_json::Value::String(value.clone()))); - } - for (_name, value) in &opts.jsonargs { - var_vals.push(value.clone()); - } - - // Load and compile filter - let loader = Loader::new(jaq_std::defs().chain(jaq_json::defs())); - let arena = Arena::default(); - let program = File { - code: opts.filter.as_str(), - path: (), - }; - let modules = loader - .load(&arena, program) - .map_err(|errs| format!("parse error: {:?}", errs))?; - - let filter = Compiler::default() - .with_funs(jaq_std::funs().chain(jaq_json::funs())) - .compile(modules) - .map_err(|errs| format!("compile error: {:?}", errs))?; - - let empty_inputs = RcIter::new(core::iter::empty()); - let stdout = io::stdout(); - let mut out = stdout.lock(); - let mut had_false_or_null = false; - - for input in inputs { - let ctx = Ctx::new(var_vals.iter().cloned(), &empty_inputs); - let results = filter.run((ctx, input)); - - for result in results { - match result { - Ok(val) => { - let s = format_output(&val, &opts)?; - - // Track for --exit-status - let compact = format!("{}", val); - if compact == "null" || compact == "false" { - had_false_or_null = true; - } - - if opts.join_output { - write!(out, "{}", s) - .map_err(|e| format!("failed to write stdout: {}", e))?; - } else { - writeln!(out, "{}", s) - .map_err(|e| format!("failed to write stdout: {}", e))?; - } - } - Err(e) => { - eprintln!("jq: error: {}", e); - return Ok(5); - } - } - } - } - - if opts.exit_status && had_false_or_null { - return Ok(1); - } - - out.flush() - .map_err(|e| format!("failed to flush stdout: {}", e))?; - - Ok(0) -} diff --git a/registry/native/crates/libs/rev/Cargo.toml b/registry/native/crates/libs/rev/Cargo.toml deleted file mode 100644 index dda6c9d74d..0000000000 --- a/registry/native/crates/libs/rev/Cargo.toml +++ /dev/null @@ -1,8 +0,0 @@ -[package] -name = "secureexec-rev" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "rev implementation for secure-exec standalone binaries" - -[dependencies] diff --git a/registry/native/crates/libs/shims/src/lib.rs b/registry/native/crates/libs/shims/src/lib.rs deleted file mode 100644 index 9e2358b3e8..0000000000 --- a/registry/native/crates/libs/shims/src/lib.rs +++ /dev/null @@ -1,13 +0,0 @@ -//! Shim implementations for commands that require subprocess support. -//! -//! Commands like `env` and `timeout` need to spawn child processes -//! via wasi-ext syscalls (through std::process::Command patches) -//! rather than using uutils versions. - -pub mod env; -pub mod nice; -pub mod nohup; -pub mod stdbuf; -pub mod timeout; -pub mod which; -pub mod xargs; diff --git a/registry/native/crates/libs/strings-cmd/Cargo.toml b/registry/native/crates/libs/strings-cmd/Cargo.toml deleted file mode 100644 index 30e9834b62..0000000000 --- a/registry/native/crates/libs/strings-cmd/Cargo.toml +++ /dev/null @@ -1,8 +0,0 @@ -[package] -name = "secureexec-strings-cmd" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "strings command implementation for secure-exec standalone binaries" - -[dependencies] diff --git a/registry/native/crates/libs/tar/Cargo.toml b/registry/native/crates/libs/tar/Cargo.toml deleted file mode 100644 index 6e42ed60f2..0000000000 --- a/registry/native/crates/libs/tar/Cargo.toml +++ /dev/null @@ -1,10 +0,0 @@ -[package] -name = "secureexec-tar" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "tar implementation for secure-exec standalone binaries" - -[dependencies] -flate2 = { version = "1.0", default-features = false, features = ["rust_backend"] } -tar = "0.4" diff --git a/registry/native/crates/libs/tar/src/lib.rs b/registry/native/crates/libs/tar/src/lib.rs deleted file mode 100644 index d20f009a37..0000000000 --- a/registry/native/crates/libs/tar/src/lib.rs +++ /dev/null @@ -1,748 +0,0 @@ -//! tar implementation — create, extract, and list tape archives. -//! -//! Supports -c create, -x extract, -t list. -//! Options: -f archive, -z gzip, -v verbose, -C directory, --strip-components=N. - -use std::collections::HashSet; -use std::ffi::OsString; -use std::fs::{self, File, OpenOptions}; -use std::io::{self, Read, Write}; -use std::path::{Component, Path, PathBuf}; - -use flate2::read::GzDecoder; -use flate2::write::GzEncoder; -use flate2::Compression; - -const MAX_ARCHIVE_ENTRIES: usize = 100_000; -const MAX_CREATE_DEPTH: usize = 256; -const MAX_DIRECTORY_ENTRIES: usize = 100_000; - -#[derive(PartialEq)] -enum Mode { - None, - Create, - Extract, - List, -} - -/// Unified tar entry point. -pub fn main(args: Vec) -> i32 { - let str_args: Vec = args - .iter() - .skip(1) - .map(|a| a.to_string_lossy().to_string()) - .collect(); - - if str_args.is_empty() { - eprintln!("tar: must specify one of -c, -x, -t"); - return 1; - } - - let mut mode = Mode::None; - let mut archive_file: Option = None; - let mut gzip = false; - let mut verbose = false; - let mut directory: Option = None; - let mut strip_components: usize = 0; - let mut paths: Vec = Vec::new(); - - let mut i = 0; - let mut first_arg = true; - - while i < str_args.len() { - let arg = &str_args[i]; - - if arg.starts_with("--strip-components=") { - if let Ok(n) = arg["--strip-components=".len()..].parse() { - strip_components = n; - } - first_arg = false; - } else if arg == "--strip-components" { - i += 1; - if i < str_args.len() { - strip_components = str_args[i].parse().unwrap_or(0); - } - first_arg = false; - } else if arg == "-C" || arg == "--directory" { - i += 1; - if i < str_args.len() { - directory = Some(str_args[i].clone()); - } - first_arg = false; - } else if arg == "--help" { - print_usage(); - return 0; - } else if arg.starts_with('-') || first_arg { - // tar's first argument can omit the leading dash (e.g., `tar czf`) - let flags = if arg.starts_with('-') { - &arg[1..] - } else { - &arg[..] - }; - let mut chars = flags.chars().peekable(); - while let Some(ch) = chars.next() { - match ch { - 'c' => mode = Mode::Create, - 'x' => mode = Mode::Extract, - 't' => mode = Mode::List, - 'z' => gzip = true, - 'v' => verbose = true, - 'f' => { - let rest: String = chars.collect(); - if !rest.is_empty() { - archive_file = Some(rest); - } else { - i += 1; - if i < str_args.len() { - archive_file = Some(str_args[i].clone()); - } - } - break; - } - 'C' => { - let rest: String = chars.collect(); - if !rest.is_empty() { - directory = Some(rest); - } else { - i += 1; - if i < str_args.len() { - directory = Some(str_args[i].clone()); - } - } - break; - } - _ => { - eprintln!("tar: unknown option: {}", ch); - return 1; - } - } - } - first_arg = false; - } else { - paths.push(arg.clone()); - first_arg = false; - } - - i += 1; - } - - // Auto-detect gzip from filename - if !gzip { - if let Some(ref f) = archive_file { - if f.ends_with(".tar.gz") || f.ends_with(".tgz") { - gzip = true; - } - } - } - - let result = match mode { - Mode::Create => do_create( - archive_file.as_deref(), - gzip, - verbose, - directory.as_deref(), - &paths, - ), - Mode::Extract => do_extract( - archive_file.as_deref(), - gzip, - verbose, - directory.as_deref(), - strip_components, - ), - Mode::List => do_list(archive_file.as_deref(), gzip, verbose), - Mode::None => { - eprintln!("tar: must specify one of -c, -x, -t"); - return 1; - } - }; - - match result { - Ok(()) => 0, - Err(e) => { - eprintln!("tar: {}", e); - 1 - } - } -} - -fn do_create( - archive_file: Option<&str>, - gzip: bool, - verbose: bool, - directory: Option<&str>, - paths: &[String], -) -> io::Result<()> { - if paths.is_empty() { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "cowardly refusing to create an empty archive", - )); - } - - match archive_file { - Some("-") | None => create_to_writer(io::stdout(), gzip, verbose, directory, paths), - Some(path) => { - let mut archive = BoundedVecWriter::new(512 * 1024 * 1024); - create_to_writer(&mut archive, gzip, verbose, directory, paths)?; - fs::write(path, archive.into_inner()) - } - } -} - -struct BoundedVecWriter { - bytes: Vec, - capacity: usize, -} - -impl BoundedVecWriter { - fn new(capacity: usize) -> Self { - Self { - bytes: Vec::new(), - capacity, - } - } - - fn into_inner(self) -> Vec { - self.bytes - } -} - -impl Write for BoundedVecWriter { - fn write(&mut self, buf: &[u8]) -> io::Result { - let Some(new_len) = self.bytes.len().checked_add(buf.len()) else { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "archive is too large to buffer", - )); - }; - if new_len > self.capacity { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "archive is too large to buffer", - )); - } - self.bytes.extend_from_slice(buf); - Ok(buf.len()) - } - - fn flush(&mut self) -> io::Result<()> { - Ok(()) - } -} - -fn create_to_writer( - writer: W, - gzip: bool, - verbose: bool, - directory: Option<&str>, - paths: &[String], -) -> io::Result<()> { - if gzip { - let encoder = GzEncoder::new(writer, Compression::default()); - let mut builder = tar::Builder::new(encoder); - append_paths(&mut builder, directory, paths, verbose)?; - let encoder = builder.into_inner()?; - let mut writer = encoder.finish()?; - writer.flush() - } else { - let mut builder = tar::Builder::new(writer); - append_paths(&mut builder, directory, paths, verbose)?; - let mut writer = builder.into_inner()?; - writer.flush() - } -} - -fn append_paths( - builder: &mut tar::Builder, - directory: Option<&str>, - paths: &[String], - verbose: bool, -) -> io::Result<()> { - let mut entry_count = 0; - for path in paths { - append_path( - builder, - resolve_disk_path(directory, Path::new(path)), - Path::new(path), - verbose, - 0, - &mut entry_count, - )?; - } - Ok(()) -} - -fn append_path( - builder: &mut tar::Builder, - disk_path: PathBuf, - archive_path: &Path, - verbose: bool, - depth: usize, - entry_count: &mut usize, -) -> io::Result<()> { - if depth > MAX_CREATE_DEPTH { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - format!( - "maximum directory depth exceeded at {}", - disk_path.display() - ), - )); - } - increment_entry_count(entry_count)?; - - let meta = fs::symlink_metadata(&disk_path)?; - - if meta.is_dir() { - append_dir( - builder, - &disk_path, - archive_path, - verbose, - depth, - entry_count, - )?; - } else if meta.is_file() { - if verbose { - eprintln!("{}", archive_path.display()); - } - let mut header = tar::Header::new_gnu(); - header.set_entry_type(tar::EntryType::Regular); - header.set_size(meta.len()); - header.set_mode(0o644); - header.set_cksum(); - let mut file = File::open(&disk_path)?; - builder.append_data(&mut header, archive_path, &mut file)?; - } else if meta.file_type().is_symlink() { - if verbose { - eprintln!("{}", archive_path.display()); - } - let target = fs::read_link(&disk_path)?; - let mut header = tar::Header::new_gnu(); - header.set_entry_type(tar::EntryType::Symlink); - header.set_size(0); - header.set_mode(0o777); - header.set_cksum(); - builder.append_link(&mut header, archive_path, &target)?; - } - - Ok(()) -} - -fn append_dir( - builder: &mut tar::Builder, - disk_dir: &Path, - archive_dir: &Path, - verbose: bool, - depth: usize, - entry_count: &mut usize, -) -> io::Result<()> { - if verbose { - eprintln!("{}/", archive_dir.display()); - } - - let mut header = tar::Header::new_gnu(); - header.set_entry_type(tar::EntryType::Directory); - header.set_size(0); - header.set_mode(0o755); - header.set_cksum(); - builder.append_data(&mut header, archive_dir, io::empty())?; - - let mut dir_entries = 0; - for entry_result in fs::read_dir(disk_dir)? { - let entry = entry_result?; - dir_entries += 1; - if dir_entries > MAX_DIRECTORY_ENTRIES { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - format!("too many entries in {}", disk_dir.display()), - )); - } - let archive_child = archive_dir.join(entry.file_name()); - append_path( - builder, - entry.path(), - &archive_child, - verbose, - depth + 1, - entry_count, - )?; - } - - Ok(()) -} - -fn do_extract( - archive_file: Option<&str>, - gzip: bool, - verbose: bool, - directory: Option<&str>, - strip_components: usize, -) -> io::Result<()> { - let reader = open_read(archive_file, gzip)?; - let mut archive = tar::Archive::new(reader); - let mut known_dirs = HashSet::new(); - if let Some(base) = directory { - validate_extract_base(Path::new(base))?; - known_dirs.insert(PathBuf::from(base)); - } - - let mut entry_count = 0; - for entry_result in archive.entries()? { - increment_entry_count(&mut entry_count)?; - let mut entry = entry_result?; - let orig_path = entry.path()?.into_owned(); - validate_archive_input_path(&orig_path)?; - - let relative_dest = match strip_path_components(&orig_path, strip_components) { - Some(p) if !p.as_os_str().is_empty() => p, - _ => continue, - }; - validate_relative_output_path(&relative_dest)?; - validate_extract_depth(&relative_dest)?; - let dest = resolve_output_path(directory, &relative_dest); - - if verbose { - eprintln!("{}", orig_path.display()); - } - - match entry.header().entry_type() { - tar::EntryType::Directory => { - ensure_relative_dir_exists(directory, &relative_dest, &mut known_dirs)?; - } - tar::EntryType::Regular | tar::EntryType::GNUSparse => { - if let Some(parent) = dest.parent() { - if !parent.as_os_str().is_empty() { - let relative_parent = - relative_dest.parent().unwrap_or_else(|| Path::new("")); - ensure_relative_dir_exists(directory, relative_parent, &mut known_dirs)?; - } - } - reject_existing_symlink(&dest)?; - let mut output = OpenOptions::new() - .create(true) - .write(true) - .truncate(true) - .open(&dest) - .map_err(|e| { - io::Error::new(e.kind(), format!("open {}: {}", dest.display(), e)) - })?; - io::copy(&mut entry, &mut output).map_err(|e| { - io::Error::new(e.kind(), format!("write {}: {}", dest.display(), e)) - })?; - output.flush().map_err(|e| { - io::Error::new(e.kind(), format!("write {}: {}", dest.display(), e)) - })?; - } - tar::EntryType::Symlink => { - if let Some(target) = entry.link_name()? { - validate_symlink_target(target.as_ref())?; - if let Some(parent) = dest.parent() { - if !parent.as_os_str().is_empty() { - let relative_parent = - relative_dest.parent().unwrap_or_else(|| Path::new("")); - ensure_relative_dir_exists( - directory, - relative_parent, - &mut known_dirs, - )?; - } - } - reject_existing_symlink(&dest)?; - #[allow(deprecated)] - std::fs::soft_link(target.as_ref(), &dest).map_err(|e| { - io::Error::new(e.kind(), format!("symlink {}: {}", dest.display(), e)) - })?; - } - } - _ => { - // Skip hard links, char/block devices, etc. - } - } - } - - Ok(()) -} - -fn resolve_disk_path(directory: Option<&str>, path: &Path) -> PathBuf { - match directory { - Some(base) if !path.is_absolute() => Path::new(base).join(path), - _ => path.to_path_buf(), - } -} - -fn resolve_output_path(directory: Option<&str>, path: &Path) -> PathBuf { - match directory { - Some(base) if path.is_relative() => Path::new(base).join(path), - _ => path.to_path_buf(), - } -} - -fn ensure_relative_dir_exists( - directory: Option<&str>, - relative_path: &Path, - known_dirs: &mut HashSet, -) -> io::Result<()> { - let mut current = match directory { - Some(base) => PathBuf::from(base), - None => PathBuf::new(), - }; - - for component in relative_path.components() { - match component { - Component::CurDir => {} - Component::Normal(part) => { - current.push(part); - if known_dirs.contains(¤t) { - continue; - } - match fs::create_dir(¤t) { - Ok(()) => { - known_dirs.insert(current.clone()); - } - Err(err) if err.kind() == io::ErrorKind::AlreadyExists => { - let metadata = fs::symlink_metadata(¤t).map_err(|metadata_err| { - io::Error::new( - metadata_err.kind(), - format!("metadata {}: {}", current.display(), metadata_err), - ) - })?; - if metadata.file_type().is_symlink() || !metadata.is_dir() { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - format!("refusing to extract through {}", current.display()), - )); - } - known_dirs.insert(current.clone()); - } - Err(err) => { - return Err(io::Error::new( - err.kind(), - format!("create_dir {}: {}", current.display(), err), - )); - } - } - } - Component::Prefix(_) | Component::RootDir | Component::ParentDir => { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - format!("unsupported path component in {}", relative_path.display()), - )); - } - } - } - - Ok(()) -} - -fn do_list(archive_file: Option<&str>, gzip: bool, verbose: bool) -> io::Result<()> { - let reader = open_read(archive_file, gzip)?; - let mut archive = tar::Archive::new(reader); - let stdout = io::stdout(); - let mut out = stdout.lock(); - - let mut entry_count = 0; - for entry_result in archive.entries()? { - increment_entry_count(&mut entry_count)?; - let entry = entry_result?; - let path = entry.path()?; - - if verbose { - let h = entry.header(); - let size = h.size().unwrap_or(0); - let mode = h.mode().unwrap_or(0o644); - let type_ch = match h.entry_type() { - tar::EntryType::Directory => 'd', - tar::EntryType::Symlink => 'l', - _ => '-', - }; - writeln!( - out, - "{}{} {:>8} {}", - type_ch, - format_mode(mode), - size, - path.display() - )?; - } else { - writeln!(out, "{}", path.display())?; - } - } - - out.flush() -} - -fn open_read(archive_file: Option<&str>, gzip: bool) -> io::Result> { - let reader: Box = match archive_file { - Some("-") | None => Box::new(io::stdin()), - Some(path) => Box::new(File::open(path)?), - }; - - if gzip { - Ok(Box::new(GzDecoder::new(reader))) - } else { - Ok(reader) - } -} - -fn strip_path_components(path: &Path, n: usize) -> Option { - let mut remaining = n; - let mut stripped = PathBuf::new(); - - for component in path.components() { - match component { - Component::Prefix(_) | Component::RootDir | Component::CurDir => {} - Component::ParentDir => { - if remaining == 0 { - stripped.push(component.as_os_str()); - } - } - Component::Normal(part) => { - if remaining > 0 { - remaining -= 1; - } else { - stripped.push(part); - } - } - } - } - - if stripped.as_os_str().is_empty() { - None - } else { - Some(stripped) - } -} - -fn increment_entry_count(count: &mut usize) -> io::Result<()> { - *count += 1; - if *count > MAX_ARCHIVE_ENTRIES { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "too many archive entries", - )); - } - Ok(()) -} - -fn validate_relative_output_path(path: &Path) -> io::Result<()> { - for component in path.components() { - match component { - Component::Normal(_) | Component::CurDir => {} - Component::Prefix(_) | Component::RootDir | Component::ParentDir => { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - format!("refusing to extract unsafe path {}", path.display()), - )); - } - } - } - Ok(()) -} - -fn validate_archive_input_path(path: &Path) -> io::Result<()> { - for component in path.components() { - match component { - Component::Normal(_) | Component::CurDir => {} - Component::Prefix(_) | Component::RootDir | Component::ParentDir => { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - format!("refusing to extract unsafe path {}", path.display()), - )); - } - } - } - Ok(()) -} - -fn validate_extract_depth(path: &Path) -> io::Result<()> { - let depth = path - .components() - .filter(|component| matches!(component, Component::Normal(_))) - .count(); - if depth > MAX_CREATE_DEPTH { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - format!("maximum extraction depth exceeded at {}", path.display()), - )); - } - Ok(()) -} - -fn validate_extract_base(path: &Path) -> io::Result<()> { - let metadata = fs::symlink_metadata(path).map_err(|err| { - io::Error::new(err.kind(), format!("metadata {}: {}", path.display(), err)) - })?; - if metadata.file_type().is_symlink() || !metadata.is_dir() { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - format!("refusing to extract through {}", path.display()), - )); - } - Ok(()) -} - -fn validate_symlink_target(target: &Path) -> io::Result<()> { - for component in target.components() { - match component { - Component::Normal(_) | Component::CurDir => {} - Component::Prefix(_) | Component::RootDir | Component::ParentDir => { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - format!( - "refusing to extract unsafe symlink target {}", - target.display() - ), - )); - } - } - } - Ok(()) -} - -fn reject_existing_symlink(path: &Path) -> io::Result<()> { - match fs::symlink_metadata(path) { - Ok(metadata) if metadata.file_type().is_symlink() => Err(io::Error::new( - io::ErrorKind::InvalidInput, - format!("refusing to overwrite symlink {}", path.display()), - )), - Ok(_) => Ok(()), - Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(()), - Err(err) => Err(io::Error::new( - err.kind(), - format!("metadata {}: {}", path.display(), err), - )), - } -} - -fn format_mode(mode: u32) -> String { - let mut s = String::with_capacity(9); - for &(bit, ch) in &[ - (0o400, 'r'), - (0o200, 'w'), - (0o100, 'x'), - (0o040, 'r'), - (0o020, 'w'), - (0o010, 'x'), - (0o004, 'r'), - (0o002, 'w'), - (0o001, 'x'), - ] { - s.push(if mode & bit != 0 { ch } else { '-' }); - } - s -} - -fn print_usage() { - eprintln!("Usage: tar [options] [files...]"); - eprintln!(" -c create archive"); - eprintln!(" -x extract archive"); - eprintln!(" -t list archive contents"); - eprintln!(" -f FILE archive filename (- for stdin/stdout)"); - eprintln!(" -z gzip compress/decompress"); - eprintln!(" -v verbose"); - eprintln!(" -C DIR change directory"); - eprintln!(" --strip-components=N"); - eprintln!(" strip N leading components on extract"); -} diff --git a/registry/native/crates/libs/tree/Cargo.toml b/registry/native/crates/libs/tree/Cargo.toml deleted file mode 100644 index adac0490e9..0000000000 --- a/registry/native/crates/libs/tree/Cargo.toml +++ /dev/null @@ -1,8 +0,0 @@ -[package] -name = "secureexec-tree" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "tree implementation for secure-exec standalone binaries" - -[dependencies] diff --git a/registry/native/crates/libs/tree/src/lib.rs b/registry/native/crates/libs/tree/src/lib.rs deleted file mode 100644 index 71cfb7044c..0000000000 --- a/registry/native/crates/libs/tree/src/lib.rs +++ /dev/null @@ -1,268 +0,0 @@ -//! tree -- list directory contents in a tree-like format -//! -//! Recursive directory walk with box-drawing characters. -//! Supports -a (show hidden), -d (dirs only), -L depth, -I exclude pattern. - -use std::ffi::OsString; -use std::fs; -use std::io::{self, Write}; -use std::path::Path; - -const DEFAULT_MAX_DEPTH: usize = 256; -const MAX_TOTAL_ENTRIES: usize = 100_000; -const MAX_DIRECTORY_ENTRIES: usize = 100_000; - -pub fn main(args: Vec) -> i32 { - let str_args: Vec = args - .iter() - .skip(1) - .map(|a| a.to_string_lossy().to_string()) - .collect(); - - let mut show_hidden = false; - let mut dirs_only = false; - let mut max_depth: Option = None; - let mut exclude_pattern: Option = None; - let mut paths: Vec = Vec::new(); - - let mut i = 0; - while i < str_args.len() { - match str_args[i].as_str() { - "-a" => show_hidden = true, - "-d" => dirs_only = true, - "-L" => { - i += 1; - if i >= str_args.len() { - eprintln!("tree: option '-L' requires an argument"); - return 1; - } - match str_args[i].parse::() { - Ok(d) => max_depth = Some(d), - Err(_) => { - eprintln!("tree: invalid level '{}'", str_args[i]); - return 1; - } - } - } - "-I" => { - i += 1; - if i >= str_args.len() { - eprintln!("tree: option '-I' requires an argument"); - return 1; - } - exclude_pattern = Some(str_args[i].clone()); - } - s if s.starts_with('-') => { - eprintln!("tree: unknown option '{}'", s); - return 1; - } - _ => paths.push(str_args[i].clone()), - } - i += 1; - } - - if paths.is_empty() { - paths.push(".".to_string()); - } - - let stdout = io::stdout(); - let mut out = stdout.lock(); - let mut dir_count: usize = 0; - let mut file_count: usize = 0; - - for (idx, path) in paths.iter().enumerate() { - if let Err(e) = writeln!(out, "{}", path).and_then(|_| { - walk_tree( - Path::new(path), - "", - 1, - max_depth, - show_hidden, - dirs_only, - exclude_pattern.as_deref(), - &mut dir_count, - &mut file_count, - &mut out, - ) - }) { - eprintln!("tree: {}", e); - return 1; - } - if idx + 1 < paths.len() { - if let Err(e) = writeln!(out) { - eprintln!("tree: {}", e); - return 1; - } - } - } - - if let Err(e) = writeln!(out) { - eprintln!("tree: {}", e); - return 1; - } - if dirs_only { - if let Err(e) = writeln!( - out, - "{} director{}", - dir_count, - if dir_count == 1 { "y" } else { "ies" } - ) { - eprintln!("tree: {}", e); - return 1; - } - } else { - if let Err(e) = writeln!( - out, - "{} director{}, {} file{}", - dir_count, - if dir_count == 1 { "y" } else { "ies" }, - file_count, - if file_count == 1 { "" } else { "s" } - ) { - eprintln!("tree: {}", e); - return 1; - } - } - - match out.flush() { - Ok(()) => 0, - Err(e) => { - eprintln!("tree: {}", e); - 1 - } - } -} - -fn matches_exclude(name: &str, pattern: &str) -> bool { - // Simple glob matching: supports * as wildcard - if pattern.contains('*') { - let parts: Vec<&str> = pattern.split('*').collect(); - if parts.len() == 2 { - let (prefix, suffix) = (parts[0], parts[1]); - return name.starts_with(prefix) && name.ends_with(suffix); - } - // Fallback: check if any non-wildcard part matches - parts.iter().all(|p| p.is_empty() || name.contains(p)) - } else { - name == pattern - } -} - -fn walk_tree( - dir: &Path, - prefix: &str, - depth: usize, - max_depth: Option, - show_hidden: bool, - dirs_only: bool, - exclude: Option<&str>, - dir_count: &mut usize, - file_count: &mut usize, - out: &mut W, -) -> io::Result<()> { - if let Some(max) = max_depth { - if depth > max { - return Ok(()); - } - } else if depth > DEFAULT_MAX_DEPTH { - return Ok(()); - } - - let mut entries: Vec = match fs::read_dir(dir) { - Ok(rd) => { - let mut entries = Vec::new(); - for entry_result in rd { - let entry = entry_result?; - if entries.len() >= MAX_DIRECTORY_ENTRIES { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - format!("too many entries in {}", dir.display()), - )); - } - entries.push(entry); - } - entries - } - Err(e) => { - writeln!(out, "{}[error opening dir: {}]", prefix, e)?; - return Ok(()); - } - }; - - // Sort entries alphabetically - entries.sort_by(|a, b| a.file_name().cmp(&b.file_name())); - - // Filter entries - entries.retain(|e| { - let name = e.file_name().to_string_lossy().to_string(); - // Skip hidden unless -a - if !show_hidden && name.starts_with('.') { - return false; - } - // Skip excluded patterns - if let Some(pat) = exclude { - if matches_exclude(&name, pat) { - return false; - } - } - // Skip files if -d - if dirs_only { - if let Ok(ft) = e.file_type() { - if !ft.is_dir() { - return false; - } - } - } - true - }); - - let count = entries.len(); - - for (idx, entry) in entries.iter().enumerate() { - if *dir_count + *file_count >= MAX_TOTAL_ENTRIES { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "too many tree entries", - )); - } - - let is_last = idx + 1 == count; - let connector = if is_last { - "\u{2514}\u{2500}\u{2500} " // └── - } else { - "\u{251c}\u{2500}\u{2500} " // ├── - }; - - let name = entry.file_name().to_string_lossy().to_string(); - let file_type = entry.file_type()?; - let is_dir = file_type.is_dir() && !file_type.is_symlink(); - - write!(out, "{}{}", prefix, connector)?; - writeln!(out, "{}", name)?; - - if is_dir { - *dir_count += 1; - let child_prefix = if is_last { - format!("{} ", prefix) - } else { - format!("{}\u{2502} ", prefix) // │ - }; - walk_tree( - &dir.join(&name), - &child_prefix, - depth + 1, - max_depth, - show_hidden, - dirs_only, - exclude, - dir_count, - file_count, - out, - )?; - } else { - *file_count += 1; - } - } - - Ok(()) -} diff --git a/registry/native/crates/libs/wasi-pty/src/lib.rs b/registry/native/crates/libs/wasi-pty/src/lib.rs deleted file mode 100644 index af94b6ef71..0000000000 --- a/registry/native/crates/libs/wasi-pty/src/lib.rs +++ /dev/null @@ -1,503 +0,0 @@ -//! WASI PTY-based process management via host_process FFI. -//! -//! Provides [`WasiPtyChild`] — an interactive process handle using a pseudo-terminal -//! instead of pipes. This is the WASI equivalent of `ExecCommandSession` from -//! `codex-utils-pty` (which wraps `portable-pty` on native platforms). -//! -//! Key difference from [`wasi_spawn::WasiChild`]: -//! - Uses a PTY master/slave pair instead of separate stdout/stderr pipes -//! - All child output (stdout + stderr) is multiplexed through the PTY master -//! - Supports interactive programs that require a terminal (e.g., editors, shells) -//! - The PTY provides terminal emulation (line discipline, echo, signals) - -use std::io::{self, Read, Write}; -use std::mem::ManuallyDrop; - -const MAX_ARG_COUNT: usize = 4096; -const MAX_ENV_COUNT: usize = 4096; -const MAX_SERIALIZED_BYTES: usize = 1024 * 1024; -const MAX_CWD_BYTES: usize = 4096; -const MAX_CAPTURED_OUTPUT_BYTES: usize = 16 * 1024 * 1024; - -/// Handle to a spawned process connected via a pseudo-terminal. -/// -/// Created by [`spawn_pty`]. The child process has the PTY slave as its -/// stdin/stdout/stderr. The parent reads and writes via the PTY master FD. -/// -/// This is the WASI equivalent of `SpawnedPty` from `codex-utils-pty`. -pub struct WasiSpawnedPty { - master_fd: RawFd, -} - -/// Interactive process session using a PTY. -/// -/// Created by [`spawn_session`]. Wraps a [`WasiSpawnedPty`] and the child -/// process handle, providing a unified API for interactive process management. -/// -/// This is the WASI equivalent of `ExecCommandSession` from `codex-utils-pty`. -pub struct WasiPtyChild { - pid: u32, - master_fd: RawFd, - exited: bool, -} - -type RawFd = u32; - -fn errno_to_io_error(errno: wasi_ext::Errno) -> io::Error { - io::Error::new(io::ErrorKind::Other, format!("wasi errno {}", errno)) -} - -fn invalid_input(message: impl Into) -> io::Error { - io::Error::new(io::ErrorKind::InvalidInput, message.into()) -} - -fn invalid_data(message: impl Into) -> io::Error { - io::Error::new(io::ErrorKind::InvalidData, message.into()) -} - -/// Read from a raw WASI file descriptor into a buffer. -fn fd_read(fd: RawFd, buf: &mut [u8]) -> io::Result { - use std::os::fd::FromRawFd; - // The caller owns this fd. This temporary File only routes through WASI fd_read. - let file = unsafe { ManuallyDrop::new(std::fs::File::from_raw_fd(fd as i32)) }; - (&*file).read(buf) -} - -/// Write to a raw WASI file descriptor from a buffer. -fn fd_write(fd: RawFd, buf: &[u8]) -> io::Result { - use std::os::fd::FromRawFd; - // The caller owns this fd. This temporary File only routes through WASI fd_write. - let file = unsafe { ManuallyDrop::new(std::fs::File::from_raw_fd(fd as i32)) }; - (&*file).write(buf) -} - -/// Close a raw WASI file descriptor. -fn fd_close(fd: RawFd) { - use std::os::fd::FromRawFd; - drop(unsafe { std::fs::File::from_raw_fd(fd as i32) }); -} - -/// Serialize strings as null-separated byte buffer for proc_spawn. -fn serialize_null_separated(items: &[&str]) -> io::Result> { - if items.len() > MAX_ARG_COUNT { - return Err(invalid_input(format!( - "argument count exceeds limit of {MAX_ARG_COUNT}" - ))); - } - - let mut buf = Vec::new(); - for (i, item) in items.iter().enumerate() { - validate_no_nul("argument", item)?; - if i > 0 { - push_serialized_byte(&mut buf, 0)?; - } - append_serialized(&mut buf, item.as_bytes())?; - } - Ok(buf) -} - -/// Serialize environment as KEY=VALUE null-separated pairs for proc_spawn. -fn serialize_env(env: &[(&str, &str)]) -> io::Result> { - if env.len() > MAX_ENV_COUNT { - return Err(invalid_input(format!( - "environment count exceeds limit of {MAX_ENV_COUNT}" - ))); - } - - let mut buf = Vec::new(); - for (i, (key, value)) in env.iter().enumerate() { - validate_env_key(key)?; - validate_no_nul("environment value", value)?; - if i > 0 { - push_serialized_byte(&mut buf, 0)?; - } - append_serialized(&mut buf, key.as_bytes())?; - push_serialized_byte(&mut buf, b'=')?; - append_serialized(&mut buf, value.as_bytes())?; - } - Ok(buf) -} - -fn validate_env_key(key: &str) -> io::Result<()> { - if key.is_empty() { - return Err(invalid_input("environment key must not be empty")); - } - validate_no_nul("environment key", key)?; - if key.as_bytes().contains(&b'=') { - return Err(invalid_input("environment key must not contain '='")); - } - Ok(()) -} - -fn validate_no_nul(label: &str, value: &str) -> io::Result<()> { - if value.as_bytes().contains(&0) { - return Err(invalid_input(format!("{label} must not contain NUL"))); - } - Ok(()) -} - -fn validate_cwd(cwd: &str) -> io::Result<()> { - validate_no_nul("cwd", cwd)?; - if cwd.len() > MAX_CWD_BYTES { - return Err(invalid_input(format!( - "cwd exceeds limit of {MAX_CWD_BYTES} bytes" - ))); - } - Ok(()) -} - -fn push_serialized_byte(buf: &mut Vec, byte: u8) -> io::Result<()> { - reserve_serialized(buf.len(), 1)?; - buf.push(byte); - Ok(()) -} - -fn append_serialized(buf: &mut Vec, bytes: &[u8]) -> io::Result<()> { - reserve_serialized(buf.len(), bytes.len())?; - buf.extend_from_slice(bytes); - Ok(()) -} - -fn reserve_serialized(current_len: usize, additional_len: usize) -> io::Result<()> { - let next_len = current_len - .checked_add(additional_len) - .ok_or_else(|| invalid_input("serialized spawn data length overflowed"))?; - if next_len > MAX_SERIALIZED_BYTES { - return Err(invalid_input(format!( - "serialized spawn data exceeds limit of {MAX_SERIALIZED_BYTES} bytes" - ))); - } - Ok(()) -} - -fn append_captured_output_with_limit( - stdout: &mut Vec, - chunk: &[u8], - limit: usize, -) -> io::Result<()> { - let next_len = stdout - .len() - .checked_add(chunk.len()) - .ok_or_else(|| invalid_data("captured PTY output length overflowed"))?; - if next_len > limit { - return Err(invalid_data(format!( - "captured PTY output exceeds limit of {limit} bytes" - ))); - } - stdout.extend_from_slice(chunk); - Ok(()) -} - -fn read_captured_output(read_output: R, cleanup: C) -> io::Result> -where - R: FnMut(&mut [u8]) -> io::Result, - C: FnMut(), -{ - read_captured_output_with_limit(read_output, cleanup, MAX_CAPTURED_OUTPUT_BYTES) -} - -fn read_captured_output_with_limit( - mut read_output: R, - mut cleanup: C, - limit: usize, -) -> io::Result> -where - R: FnMut(&mut [u8]) -> io::Result, - C: FnMut(), -{ - let mut stdout = Vec::new(); - let mut buf = [0u8; 4096]; - loop { - match read_output(&mut buf) { - Ok(0) => break, - Ok(n) => { - if let Err(e) = append_captured_output_with_limit(&mut stdout, &buf[..n], limit) { - cleanup(); - return Err(e); - } - } - Err(e) if e.kind() == io::ErrorKind::BrokenPipe => break, - Err(e) => { - cleanup(); - return Err(e); - } - } - } - Ok(stdout) -} - -fn wait_or_cleanup(result: io::Result, cleanup: C) -> io::Result -where - C: FnOnce(), -{ - match result { - Ok(exit_code) => Ok(exit_code), - Err(e) => { - cleanup(); - Err(e) - } - } -} - -/// Spawn a child process connected via a PTY. -/// -/// Allocates a PTY master/slave pair, spawns the child with the slave FD as -/// stdin/stdout/stderr, and returns a [`WasiPtyChild`] handle. The slave FD -/// is closed in the parent after spawn (POSIX close-after-fork). -/// -/// # Arguments -/// * `argv` - Command and arguments (argv[0] is the program name) -/// * `env` - Environment variable pairs (empty inherits parent env via host) -/// * `cwd` - Working directory for the child -pub fn spawn_session(argv: &[&str], env: &[(&str, &str)], cwd: &str) -> io::Result { - if argv.is_empty() { - return Err(io::Error::new(io::ErrorKind::InvalidInput, "empty argv")); - } - - validate_cwd(cwd)?; - let argv_buf = serialize_null_separated(argv)?; - let envp_buf = serialize_env(env)?; - - // Allocate PTY master/slave pair via kernel - let (master_fd, slave_fd) = wasi_ext::openpty().map_err(errno_to_io_error)?; - - // Spawn child with PTY slave as all stdio - let result = wasi_ext::spawn( - &argv_buf, - &envp_buf, - slave_fd, - slave_fd, - slave_fd, - cwd.as_bytes(), - ); - - // Close slave FD in parent (POSIX close-after-fork) — child has its own ref - fd_close(slave_fd); - - match result { - Ok(pid) => Ok(WasiPtyChild { - pid, - master_fd, - exited: false, - }), - Err(errno) => { - fd_close(master_fd); - Err(errno_to_io_error(errno)) - } - } -} - -/// Allocate a PTY pair without spawning a process. -/// -/// Returns a [`WasiSpawnedPty`] for the master end. The slave FD is returned -/// separately so the caller can pass it to [`wasi_spawn::spawn_child`] or -/// use it directly. -pub fn open_pty() -> io::Result<(WasiSpawnedPty, RawFd)> { - let (master_fd, slave_fd) = wasi_ext::openpty().map_err(errno_to_io_error)?; - - Ok((WasiSpawnedPty { master_fd }, slave_fd)) -} - -impl WasiSpawnedPty { - /// Get the master FD for direct I/O. - pub fn master_fd(&self) -> RawFd { - self.master_fd - } - - /// Read output from the PTY master (data written by the child). - pub fn read(&self, buf: &mut [u8]) -> io::Result { - fd_read(self.master_fd, buf) - } - - /// Write input to the PTY master (delivered to the child's stdin). - pub fn write(&self, buf: &[u8]) -> io::Result { - fd_write(self.master_fd, buf) - } -} - -impl Drop for WasiSpawnedPty { - fn drop(&mut self) { - fd_close(self.master_fd); - } -} - -impl WasiPtyChild { - /// Get the child's virtual PID. - pub fn pid(&self) -> u32 { - self.pid - } - - /// Get the PTY master FD for direct I/O if needed. - pub fn master_fd(&self) -> RawFd { - self.master_fd - } - - /// Read output from the child via the PTY master. - /// - /// All child output (stdout and stderr) is multiplexed through the PTY. - /// Returns 0 bytes when the PTY slave is closed (child exited). - pub fn read_output(&self, buf: &mut [u8]) -> io::Result { - fd_read(self.master_fd, buf) - } - - /// Write input to the child via the PTY master. - /// - /// Data is delivered to the child's stdin through the PTY line discipline. - pub fn write_stdin(&self, buf: &[u8]) -> io::Result { - fd_write(self.master_fd, buf) - } - - /// Wait for the child to exit. Returns the exit code. - /// - /// Blocks via host_process_waitpid (Atomics.wait on host side). - pub fn wait(&mut self) -> io::Result { - if self.exited { - return Err(io::Error::new(io::ErrorKind::Other, "already waited")); - } - - let (status, _actual_pid) = wasi_ext::waitpid(self.pid, 0).map_err(errno_to_io_error)?; - - self.exited = true; - Ok(status as i32) - } - - /// Send a signal to the child process. - pub fn kill(&mut self, signal: u32) -> io::Result<()> { - wasi_ext::kill(self.pid, signal).map_err(errno_to_io_error) - } - - /// Send SIGTERM to the child process. - pub fn terminate(&mut self) -> io::Result<()> { - self.kill(15) - } - - fn kill_and_reap(&mut self) { - if self.exited { - return; - } - - let _ = wasi_ext::kill(self.pid, 9); - if wasi_ext::waitpid(self.pid, 0).is_ok() { - self.exited = true; - } - } - - /// Read all output from the PTY, then wait for exit. - /// - /// Reads output until the PTY master gets EOF (child closed slave), - /// then waits for the child to exit. - pub fn consume_output(&mut self) -> io::Result { - let master_fd = self.master_fd; - let stdout = read_captured_output(|buf| fd_read(master_fd, buf), || self.kill_and_reap())?; - - let wait_result = self.wait(); - let exit_code = wait_or_cleanup(wait_result, || self.kill_and_reap())?; - - // PTY multiplexes stdout+stderr, so stderr is empty - Ok(wasi_spawn::WasiOutput { - stdout, - stderr: Vec::new(), - exit_code, - }) - } -} - -impl Drop for WasiPtyChild { - fn drop(&mut self) { - self.kill_and_reap(); - fd_close(self.master_fd); - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn rejects_interior_nul_in_arguments() { - let err = serialize_null_separated(&["echo", "a\0b"]).unwrap_err(); - - assert_eq!(err.kind(), io::ErrorKind::InvalidInput); - } - - #[test] - fn rejects_invalid_environment_keys() { - let err = serialize_env(&[("A=B", "value")]).unwrap_err(); - - assert_eq!(err.kind(), io::ErrorKind::InvalidInput); - } - - #[test] - fn rejects_oversized_serialized_data() { - let oversized = "x".repeat(MAX_SERIALIZED_BYTES + 1); - let err = serialize_null_separated(&[&oversized]).unwrap_err(); - - assert_eq!(err.kind(), io::ErrorKind::InvalidInput); - } - - #[test] - fn appends_captured_output_until_limit() { - let mut output = vec![b'x'; MAX_CAPTURED_OUTPUT_BYTES - 1]; - - append_captured_output_with_limit(&mut output, b"y", MAX_CAPTURED_OUTPUT_BYTES).unwrap(); - - assert_eq!(output.len(), MAX_CAPTURED_OUTPUT_BYTES); - } - - #[test] - fn rejects_oversized_captured_output() { - let mut output = vec![b'x'; MAX_CAPTURED_OUTPUT_BYTES]; - let err = append_captured_output_with_limit(&mut output, b"y", MAX_CAPTURED_OUTPUT_BYTES) - .unwrap_err(); - - assert_eq!(err.kind(), io::ErrorKind::InvalidData); - assert_eq!(output.len(), MAX_CAPTURED_OUTPUT_BYTES); - } - - #[test] - fn consume_helper_cleans_up_on_output_limit() { - let mut reads = 0; - let mut cleanup_calls = 0; - let err = read_captured_output_with_limit( - |buf| { - reads += 1; - buf[..4].copy_from_slice(b"xxxx"); - Ok(4) - }, - || cleanup_calls += 1, - 8, - ) - .unwrap_err(); - - assert_eq!(err.kind(), io::ErrorKind::InvalidData); - assert_eq!(reads, 3); - assert_eq!(cleanup_calls, 1); - } - - #[test] - fn consume_helper_cleans_up_on_read_error() { - let mut cleanup_calls = 0; - let err = read_captured_output_with_limit( - |_buf| Err(io::Error::new(io::ErrorKind::PermissionDenied, "boom")), - || cleanup_calls += 1, - 8, - ) - .unwrap_err(); - - assert_eq!(err.kind(), io::ErrorKind::PermissionDenied); - assert_eq!(cleanup_calls, 1); - } - - #[test] - fn wait_error_runs_cleanup() { - let mut cleanup_calls = 0; - let err = wait_or_cleanup( - Err(io::Error::new(io::ErrorKind::NotFound, "missing")), - || cleanup_calls += 1, - ) - .unwrap_err(); - - assert_eq!(err.kind(), io::ErrorKind::NotFound); - assert_eq!(cleanup_calls, 1); - } -} diff --git a/registry/native/crates/libs/wasi-spawn/src/lib.rs b/registry/native/crates/libs/wasi-spawn/src/lib.rs deleted file mode 100644 index b2b1b8be6c..0000000000 --- a/registry/native/crates/libs/wasi-spawn/src/lib.rs +++ /dev/null @@ -1,756 +0,0 @@ -//! WASI process spawning via host_process FFI. -//! -//! Provides `WasiChild` — a synchronous child process handle with pipe-based -//! stdout/stderr capture, wait, and kill. Uses wasi-ext FFI directly instead -//! of std::process::Command for explicit control over pipe lifecycle. -//! -//! Designed for codex-rs WASI integration: replaces tokio::process::Command -//! on wasm32-wasip1 where tokio process/signal features are unavailable. - -use std::io::{self, Read}; -use std::mem::ManuallyDrop; - -const MAX_ARG_COUNT: usize = 4096; -const MAX_ENV_COUNT: usize = 4096; -const MAX_SERIALIZED_BYTES: usize = 1024 * 1024; -const MAX_CWD_BYTES: usize = 4096; -const MAX_CAPTURED_STREAM_BYTES: usize = 16 * 1024 * 1024; -#[cfg(target_os = "wasi")] -const READY_STDOUT: u64 = 0; -#[cfg(target_os = "wasi")] -const READY_STDERR: u64 = 1; - -/// Captured output from a child process. -pub struct WasiOutput { - pub stdout: Vec, - pub stderr: Vec, - pub exit_code: i32, -} - -/// Handle to a spawned child process with pipe-based I/O capture. -/// -/// Created by [`spawn_child`]. Owns the read ends of stdout/stderr pipes. -/// The write ends are closed in the parent after spawn (POSIX close-after-fork). -pub struct WasiChild { - pid: u32, - stdout_fd: Option, - stderr_fd: Option, - exited: bool, -} - -/// Raw file descriptor type matching WASI u32 FDs. -type RawFd = u32; - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum CapturedStream { - Stdout, - Stderr, -} - -fn errno_to_io_error(errno: wasi_ext::Errno) -> io::Error { - io::Error::new(io::ErrorKind::Other, format!("wasi errno {}", errno)) -} - -#[cfg(target_os = "wasi")] -fn wasi_errno_to_io_error(errno: wasi::Errno) -> io::Error { - io::Error::new(io::ErrorKind::Other, format!("wasi errno {}", errno.raw())) -} - -fn invalid_input(message: impl Into) -> io::Error { - io::Error::new(io::ErrorKind::InvalidInput, message.into()) -} - -fn invalid_data(message: impl Into) -> io::Error { - io::Error::new(io::ErrorKind::InvalidData, message.into()) -} - -/// Read from a raw WASI file descriptor into a buffer. -/// -/// Uses std::fs::File::from_raw_fd for WASI fd_read dispatch. -fn fd_read(fd: RawFd, buf: &mut [u8]) -> io::Result { - use std::os::fd::FromRawFd; - // The caller owns this fd. This temporary File only routes through WASI fd_read. - let file = unsafe { ManuallyDrop::new(std::fs::File::from_raw_fd(fd as i32)) }; - (&*file).read(buf) -} - -/// Close a raw WASI file descriptor. -fn fd_close(fd: RawFd) { - use std::os::fd::FromRawFd; - // Safety: fd is a valid local FD. Drop closes it via WASI fd_close. - drop(unsafe { std::fs::File::from_raw_fd(fd as i32) }); -} - -/// Serialize strings as null-separated byte buffer for proc_spawn. -fn serialize_null_separated(items: &[&str]) -> io::Result> { - if items.len() > MAX_ARG_COUNT { - return Err(invalid_input(format!( - "argument count exceeds limit of {MAX_ARG_COUNT}" - ))); - } - - let mut buf = Vec::new(); - for (i, item) in items.iter().enumerate() { - validate_no_nul("argument", item)?; - if i > 0 { - push_serialized_byte(&mut buf, 0)?; - } - append_serialized(&mut buf, item.as_bytes())?; - } - Ok(buf) -} - -/// Serialize environment as KEY=VALUE null-separated pairs for proc_spawn. -fn serialize_env(env: &[(&str, &str)]) -> io::Result> { - if env.len() > MAX_ENV_COUNT { - return Err(invalid_input(format!( - "environment count exceeds limit of {MAX_ENV_COUNT}" - ))); - } - - let mut buf = Vec::new(); - for (i, (key, value)) in env.iter().enumerate() { - validate_env_key(key)?; - validate_no_nul("environment value", value)?; - if i > 0 { - push_serialized_byte(&mut buf, 0)?; - } - append_serialized(&mut buf, key.as_bytes())?; - push_serialized_byte(&mut buf, b'=')?; - append_serialized(&mut buf, value.as_bytes())?; - } - Ok(buf) -} - -fn validate_env_key(key: &str) -> io::Result<()> { - if key.is_empty() { - return Err(invalid_input("environment key must not be empty")); - } - validate_no_nul("environment key", key)?; - if key.as_bytes().contains(&b'=') { - return Err(invalid_input("environment key must not contain '='")); - } - Ok(()) -} - -fn validate_no_nul(label: &str, value: &str) -> io::Result<()> { - if value.as_bytes().contains(&0) { - return Err(invalid_input(format!("{label} must not contain NUL"))); - } - Ok(()) -} - -fn validate_cwd(cwd: &str) -> io::Result<()> { - validate_no_nul("cwd", cwd)?; - if cwd.len() > MAX_CWD_BYTES { - return Err(invalid_input(format!( - "cwd exceeds limit of {MAX_CWD_BYTES} bytes" - ))); - } - Ok(()) -} - -fn push_serialized_byte(buf: &mut Vec, byte: u8) -> io::Result<()> { - reserve_serialized(buf.len(), 1)?; - buf.push(byte); - Ok(()) -} - -fn append_serialized(buf: &mut Vec, bytes: &[u8]) -> io::Result<()> { - reserve_serialized(buf.len(), bytes.len())?; - buf.extend_from_slice(bytes); - Ok(()) -} - -fn reserve_serialized(current_len: usize, additional_len: usize) -> io::Result<()> { - let next_len = current_len - .checked_add(additional_len) - .ok_or_else(|| invalid_input("serialized spawn data length overflowed"))?; - if next_len > MAX_SERIALIZED_BYTES { - return Err(invalid_input(format!( - "serialized spawn data exceeds limit of {MAX_SERIALIZED_BYTES} bytes" - ))); - } - Ok(()) -} - -fn append_captured_stream_with_limit( - output: &mut Vec, - chunk: &[u8], - limit: usize, -) -> io::Result<()> { - let next_len = output - .len() - .checked_add(chunk.len()) - .ok_or_else(|| invalid_data("captured stream length overflowed"))?; - if next_len > limit { - return Err(invalid_data(format!( - "captured stream exceeds limit of {limit} bytes" - ))); - } - output.extend_from_slice(chunk); - Ok(()) -} - -fn read_captured_streams( - stdout_fd: Option, - stderr_fd: Option, - read_fd: R, - wait_readable: W, - cleanup: C, -) -> io::Result<(Vec, Vec)> -where - R: FnMut(RawFd, &mut [u8]) -> io::Result, - W: FnMut(Option, Option) -> io::Result<[Option; 2]>, - C: FnMut(), -{ - read_captured_streams_with_limit( - stdout_fd, - stderr_fd, - read_fd, - wait_readable, - cleanup, - MAX_CAPTURED_STREAM_BYTES, - ) -} - -fn read_captured_streams_with_limit( - stdout_fd: Option, - stderr_fd: Option, - mut read_fd: R, - mut wait_readable: W, - mut cleanup: C, - limit: usize, -) -> io::Result<(Vec, Vec)> -where - R: FnMut(RawFd, &mut [u8]) -> io::Result, - W: FnMut(Option, Option) -> io::Result<[Option; 2]>, - C: FnMut(), -{ - let mut stdout = Vec::new(); - let mut stderr = Vec::new(); - let mut stdout_done = stdout_fd.is_none(); - let mut stderr_done = stderr_fd.is_none(); - let mut buf = [0u8; 4096]; - - while !stdout_done || !stderr_done { - let active_stdout = if stdout_done { None } else { stdout_fd }; - let active_stderr = if stderr_done { None } else { stderr_fd }; - let ready = match wait_readable(active_stdout, active_stderr) { - Ok(ready) => ready, - Err(e) => { - cleanup(); - return Err(e); - } - }; - - let mut progressed = false; - for stream in ready.into_iter().flatten() { - let (fd, output, done) = match stream { - CapturedStream::Stdout if !stdout_done => ( - stdout_fd.expect("stdout fd is present while active"), - &mut stdout, - &mut stdout_done, - ), - CapturedStream::Stderr if !stderr_done => ( - stderr_fd.expect("stderr fd is present while active"), - &mut stderr, - &mut stderr_done, - ), - CapturedStream::Stdout => continue, - CapturedStream::Stderr => continue, - }; - - progressed = true; - match read_fd(fd, &mut buf) { - Ok(0) => { - *done = true; - } - Ok(n) => { - if let Err(e) = append_captured_stream_with_limit(output, &buf[..n], limit) { - cleanup(); - return Err(e); - } - } - Err(e) if e.kind() == io::ErrorKind::BrokenPipe => { - *done = true; - } - Err(e) => { - cleanup(); - return Err(e); - } - } - } - - if !progressed { - cleanup(); - return Err(io::Error::new( - io::ErrorKind::WouldBlock, - "no captured stream became readable", - )); - } - } - Ok((stdout, stderr)) -} - -#[cfg(target_os = "wasi")] -fn wait_readable_streams( - stdout_fd: Option, - stderr_fd: Option, -) -> io::Result<[Option; 2]> { - let mut subscriptions = Vec::with_capacity(2); - if let Some(fd) = stdout_fd { - subscriptions.push(wasi::Subscription { - userdata: READY_STDOUT, - u: wasi::SubscriptionU { - tag: wasi::EVENTTYPE_FD_READ.raw(), - u: wasi::SubscriptionUU { - fd_read: wasi::SubscriptionFdReadwrite { - file_descriptor: fd, - }, - }, - }, - }); - } - if let Some(fd) = stderr_fd { - subscriptions.push(wasi::Subscription { - userdata: READY_STDERR, - u: wasi::SubscriptionU { - tag: wasi::EVENTTYPE_FD_READ.raw(), - u: wasi::SubscriptionUU { - fd_read: wasi::SubscriptionFdReadwrite { - file_descriptor: fd, - }, - }, - }, - }); - } - - if subscriptions.is_empty() { - return Ok([None, None]); - } - - let mut events = vec![unsafe { std::mem::zeroed::() }; subscriptions.len()]; - let ready_count = unsafe { - wasi::poll_oneoff( - subscriptions.as_ptr(), - events.as_mut_ptr(), - subscriptions.len(), - ) - } - .map_err(wasi_errno_to_io_error)?; - if ready_count > events.len() { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "poll returned too many events", - )); - } - - let mut ready = [None, None]; - for (i, event) in events.into_iter().take(ready_count).enumerate() { - if event.error != wasi::ERRNO_SUCCESS { - return Err(wasi_errno_to_io_error(event.error)); - } - ready[i] = match event.userdata { - READY_STDOUT => Some(CapturedStream::Stdout), - READY_STDERR => Some(CapturedStream::Stderr), - _ => { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "poll returned unknown stream", - )); - } - }; - } - Ok(ready) -} - -#[cfg(not(target_os = "wasi"))] -fn wait_readable_streams( - stdout_fd: Option, - stderr_fd: Option, -) -> io::Result<[Option; 2]> { - Ok([ - stdout_fd.map(|_| CapturedStream::Stdout), - stderr_fd.map(|_| CapturedStream::Stderr), - ]) -} - -fn wait_or_cleanup(result: io::Result, cleanup: C) -> io::Result -where - C: FnOnce(), -{ - match result { - Ok(exit_code) => Ok(exit_code), - Err(e) => { - cleanup(); - Err(e) - } - } -} - -fn spawn_child_with_stdin_fd( - argv: &[&str], - env: &[(&str, &str)], - cwd: &str, - stdin_fd: u32, -) -> io::Result { - if argv.is_empty() { - return Err(io::Error::new(io::ErrorKind::InvalidInput, "empty argv")); - } - - validate_cwd(cwd)?; - let argv_buf = serialize_null_separated(argv)?; - let envp_buf = serialize_env(env)?; - - // Create stdout pipe - let (stdout_read, stdout_write) = wasi_ext::pipe().map_err(errno_to_io_error)?; - - // Create stderr pipe - let (stderr_read, stderr_write) = wasi_ext::pipe().map_err(|e| { - fd_close(stdout_read); - fd_close(stdout_write); - errno_to_io_error(e) - })?; - - // Spawn child with pipe-captured stdout/stderr and caller-selected stdin. - let result = wasi_ext::spawn( - &argv_buf, - &envp_buf, - stdin_fd, - stdout_write, - stderr_write, - cwd.as_bytes(), - ); - - // Close write ends in parent (POSIX close-after-fork) - fd_close(stdout_write); - fd_close(stderr_write); - - match result { - Ok(pid) => Ok(WasiChild { - pid, - stdout_fd: Some(stdout_read), - stderr_fd: Some(stderr_read), - exited: false, - }), - Err(errno) => { - fd_close(stdout_read); - fd_close(stderr_read); - Err(errno_to_io_error(errno)) - } - } -} - -/// Spawn a child process with pipe-captured stdout and stderr. -/// -/// Creates pipes for stdout/stderr, spawns the child via host_process FFI, -/// and returns a `WasiChild` handle. The parent's stdin is inherited. -/// -/// # Arguments -/// * `argv` - Command and arguments (argv[0] is the program name) -/// * `env` - Environment variable pairs (empty inherits parent env via host) -/// * `cwd` - Working directory for the child -pub fn spawn_child(argv: &[&str], env: &[(&str, &str)], cwd: &str) -> io::Result { - spawn_child_with_stdin_fd(argv, env, cwd, 0) -} - -/// Spawn a child process with stdin ignored and stdout/stderr captured. -/// -/// This is appropriate for agent tool calls or other subprocesses that must not -/// inherit an interactive control stream from the parent process. -pub fn spawn_child_ignore_stdin( - argv: &[&str], - env: &[(&str, &str)], - cwd: &str, -) -> io::Result { - spawn_child_with_stdin_fd(argv, env, cwd, u32::MAX) -} - -/// Spawn a child process inheriting all stdio (no pipe capture). -/// -/// Useful for interactive commands where output should go directly to -/// the parent's terminal. -pub fn spawn_child_inherit( - argv: &[&str], - env: &[(&str, &str)], - cwd: &str, -) -> io::Result { - if argv.is_empty() { - return Err(io::Error::new(io::ErrorKind::InvalidInput, "empty argv")); - } - - validate_cwd(cwd)?; - let argv_buf = serialize_null_separated(argv)?; - let envp_buf = serialize_env(env)?; - - let pid = wasi_ext::spawn( - &argv_buf, - &envp_buf, - 0, - 1, - 2, // inherit all stdio - cwd.as_bytes(), - ) - .map_err(errno_to_io_error)?; - - Ok(WasiChild { - pid, - stdout_fd: None, - stderr_fd: None, - exited: false, - }) -} - -impl WasiChild { - /// Get the child's virtual PID. - pub fn pid(&self) -> u32 { - self.pid - } - - /// Read from the child's stdout pipe. - /// - /// Returns 0 bytes when the pipe is closed (child exited or closed stdout). - pub fn read_stdout(&self, buf: &mut [u8]) -> io::Result { - match self.stdout_fd { - Some(fd) => fd_read(fd, buf), - None => Ok(0), - } - } - - /// Read from the child's stderr pipe. - /// - /// Returns 0 bytes when the pipe is closed (child exited or closed stderr). - pub fn read_stderr(&self, buf: &mut [u8]) -> io::Result { - match self.stderr_fd { - Some(fd) => fd_read(fd, buf), - None => Ok(0), - } - } - - /// Wait for the child to exit. Returns the exit code. - /// - /// Blocks via host_process_waitpid (Atomics.wait on host side). - pub fn wait(&mut self) -> io::Result { - if self.exited { - return Err(io::Error::new(io::ErrorKind::Other, "already waited")); - } - - let (status, _actual_pid) = wasi_ext::waitpid(self.pid, 0).map_err(errno_to_io_error)?; - - self.exited = true; - - // Decode exit status using bash 128+signal convention - // Normal exit: status is the exit code directly - // Signal kill: status is 128 + signal number - Ok(status as i32) - } - - /// Send a signal to the child process. - /// - /// Common signals: SIGTERM (15), SIGKILL (9). - pub fn kill(&mut self, signal: u32) -> io::Result<()> { - wasi_ext::kill(self.pid, signal).map_err(errno_to_io_error) - } - - /// Send SIGTERM to the child process. - pub fn terminate(&mut self) -> io::Result<()> { - self.kill(15) - } - - fn kill_and_reap(&mut self) { - if self.exited { - return; - } - - let _ = wasi_ext::kill(self.pid, 9); - if wasi_ext::waitpid(self.pid, 0).is_ok() { - self.exited = true; - } - } - - fn close_output_fds(&mut self) { - if let Some(fd) = self.stdout_fd.take() { - fd_close(fd); - } - if let Some(fd) = self.stderr_fd.take() { - fd_close(fd); - } - } - - /// Read all stdout and stderr, then wait for exit. - /// - /// Drains readable stdout and stderr events until both streams close, then waits. - pub fn consume_output(&mut self) -> io::Result { - let stdout_fd = self.stdout_fd; - let stderr_fd = self.stderr_fd; - let (stdout, stderr) = - read_captured_streams(stdout_fd, stderr_fd, fd_read, wait_readable_streams, || { - self.kill_and_reap() - })?; - - let wait_result = self.wait(); - let exit_code = wait_or_cleanup(wait_result, || self.kill_and_reap())?; - - Ok(WasiOutput { - stdout, - stderr, - exit_code, - }) - } -} - -impl Drop for WasiChild { - fn drop(&mut self) { - self.kill_and_reap(); - self.close_output_fds(); - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn rejects_interior_nul_in_arguments() { - let err = serialize_null_separated(&["echo", "a\0b"]).unwrap_err(); - - assert_eq!(err.kind(), io::ErrorKind::InvalidInput); - } - - #[test] - fn rejects_invalid_environment_keys() { - let err = serialize_env(&[("A=B", "value")]).unwrap_err(); - - assert_eq!(err.kind(), io::ErrorKind::InvalidInput); - } - - #[test] - fn rejects_oversized_serialized_data() { - let oversized = "x".repeat(MAX_SERIALIZED_BYTES + 1); - let err = serialize_null_separated(&[&oversized]).unwrap_err(); - - assert_eq!(err.kind(), io::ErrorKind::InvalidInput); - } - - #[test] - fn rejects_oversized_captured_stream() { - let mut output = vec![b'x'; 8]; - let err = append_captured_stream_with_limit(&mut output, b"y", 8).unwrap_err(); - - assert_eq!(err.kind(), io::ErrorKind::InvalidData); - assert_eq!(output.len(), 8); - } - - #[test] - fn capture_helper_cleans_up_on_output_limit() { - let mut reads = 0; - let mut cleanup_calls = 0; - let err = read_captured_streams_with_limit( - Some(7), - None, - |_fd, buf| { - reads += 1; - buf[..4].copy_from_slice(b"xxxx"); - Ok(4) - }, - |stdout_fd, stderr_fd| { - assert_eq!(stdout_fd, Some(7)); - assert_eq!(stderr_fd, None); - Ok([Some(CapturedStream::Stdout), None]) - }, - || cleanup_calls += 1, - 8, - ) - .unwrap_err(); - - assert_eq!(err.kind(), io::ErrorKind::InvalidData); - assert_eq!(reads, 3); - assert_eq!(cleanup_calls, 1); - } - - #[test] - fn capture_helper_cleans_up_on_read_error() { - let mut cleanup_calls = 0; - let err = read_captured_streams_with_limit( - Some(7), - None, - |_fd, _buf| Err(io::Error::new(io::ErrorKind::PermissionDenied, "boom")), - |_stdout_fd, _stderr_fd| Ok([Some(CapturedStream::Stdout), None]), - || cleanup_calls += 1, - 8, - ) - .unwrap_err(); - - assert_eq!(err.kind(), io::ErrorKind::PermissionDenied); - assert_eq!(cleanup_calls, 1); - } - - #[test] - fn capture_helper_interleaves_stdout_and_stderr() { - let mut ready_calls = 0; - let mut stdout_reads = 0; - let mut stderr_reads = 0; - let (stdout, stderr) = read_captured_streams_with_limit( - Some(1), - Some(2), - |fd, buf| match fd { - 1 => { - stdout_reads += 1; - if stdout_reads > 1 { - return Ok(0); - } - buf[..3].copy_from_slice(b"out"); - Ok(3) - } - 2 => { - stderr_reads += 1; - if stderr_reads > 1 { - return Ok(0); - } - buf[..3].copy_from_slice(b"err"); - Ok(3) - } - _ => unreachable!(), - }, - |stdout_fd, stderr_fd| { - ready_calls += 1; - match ready_calls { - 1 => { - assert_eq!(stdout_fd, Some(1)); - assert_eq!(stderr_fd, Some(2)); - Ok([Some(CapturedStream::Stderr), None]) - } - 2 => { - assert_eq!(stdout_fd, Some(1)); - assert_eq!(stderr_fd, Some(2)); - Ok([Some(CapturedStream::Stdout), None]) - } - 3 => Ok([Some(CapturedStream::Stderr), None]), - 4 => Ok([Some(CapturedStream::Stdout), None]), - _ => unreachable!(), - } - }, - || unreachable!(), - 16, - ) - .unwrap(); - - assert_eq!(stdout, b"out"); - assert_eq!(stderr, b"err"); - assert_eq!(ready_calls, 4); - } - - #[test] - fn wait_error_runs_cleanup() { - let mut cleanup_calls = 0; - let err = wait_or_cleanup( - Err(io::Error::new(io::ErrorKind::NotFound, "missing")), - || cleanup_calls += 1, - ) - .unwrap_err(); - - assert_eq!(err.kind(), io::ErrorKind::NotFound); - assert_eq!(cleanup_calls, 1); - } -} diff --git a/registry/native/crates/libs/yq/Cargo.toml b/registry/native/crates/libs/yq/Cargo.toml deleted file mode 100644 index ec7336b20c..0000000000 --- a/registry/native/crates/libs/yq/Cargo.toml +++ /dev/null @@ -1,15 +0,0 @@ -[package] -name = "secureexec-yq" -version.workspace = true -edition.workspace = true -license.workspace = true -description = "yq (YAML/XML/TOML/JSON processor) implementation for secure-exec standalone binaries" - -[dependencies] -jaq-core = "2.2" -jaq-std = "2.1" -jaq-json = { version = "1.1", features = ["serde_json"] } -serde_json = "1" -serde_yaml = "0.9" -toml = "0.8" -quick-xml = "0.37" diff --git a/registry/native/crates/libs/yq/src/lib.rs b/registry/native/crates/libs/yq/src/lib.rs deleted file mode 100644 index 871e3d7b85..0000000000 --- a/registry/native/crates/libs/yq/src/lib.rs +++ /dev/null @@ -1,1004 +0,0 @@ -//! yq — YAML/XML/TOML/JSON processor using jaq filter engine. -//! -//! Converts input to JSON, runs jaq filter, converts output to requested format. -//! Reuses jaq-core/jaq-std/jaq-json (same engine as jq command). - -use std::ffi::OsString; -use std::fmt; -use std::io::{self, Read, Write}; - -use jaq_core::load::{Arena, File, Loader}; -use jaq_core::{Compiler, Ctx, RcIter}; -use jaq_json::Val; - -const MAX_INPUT_BYTES: usize = 16 * 1024 * 1024; -const MAX_FORMATTED_OUTPUT_BYTES: usize = 16 * 1024 * 1024; -const MAX_OUTPUT_VALUES: usize = 100_000; -const MAX_XML_DEPTH: usize = 256; -const MAX_XML_NODES: usize = 100_000; -const MAX_XML_ATTRIBUTES_PER_ELEMENT: usize = 4096; -const MAX_XML_TEXT_BYTES: usize = 16 * 1024 * 1024; - -#[derive(Clone, Copy, PartialEq)] -enum Format { - Yaml, - Json, - Toml, - Xml, -} - -struct YqOptions { - filter: String, - input_format: Option, - output_format: Option, - raw_output: bool, - compact: bool, - null_input: bool, - slurp: bool, -} - -/// Entry point for yq command. -pub fn main(args: Vec) -> i32 { - let str_args: Vec = args - .iter() - .skip(1) - .map(|a| a.to_string_lossy().to_string()) - .collect(); - - match run_yq(&str_args) { - Ok(code) => code, - Err(msg) => { - eprintln!("yq: {}", msg); - 2 - } - } -} - -fn parse_format(s: &str) -> Result { - match s { - "yaml" | "y" => Ok(Format::Yaml), - "json" | "j" => Ok(Format::Json), - "toml" | "t" => Ok(Format::Toml), - "xml" | "x" => Ok(Format::Xml), - _ => Err(format!( - "unknown format: {} (expected yaml, json, toml, xml)", - s - )), - } -} - -fn parse_args(args: &[String]) -> Result { - let mut opts = YqOptions { - filter: String::new(), - input_format: None, - output_format: None, - raw_output: false, - compact: false, - null_input: false, - slurp: false, - }; - - let mut filter_set = false; - let mut i = 0; - - while i < args.len() { - let arg = &args[i]; - - if arg == "--" { - break; - } - - if arg == "-p" || arg == "--input-format" { - i += 1; - if i >= args.len() { - return Err("-p requires a format argument".to_string()); - } - opts.input_format = Some(parse_format(&args[i])?); - } else if arg == "-o" || arg == "--output-format" { - i += 1; - if i >= args.len() { - return Err("-o requires a format argument".to_string()); - } - opts.output_format = Some(parse_format(&args[i])?); - } else if arg == "-r" || arg == "--raw-output" { - opts.raw_output = true; - } else if arg == "-c" || arg == "--compact-output" { - opts.compact = true; - } else if arg == "-n" || arg == "--null-input" { - opts.null_input = true; - } else if arg == "-s" || arg == "--slurp" { - opts.slurp = true; - } else if arg.starts_with('-') && arg.len() > 1 && !arg.starts_with("--") { - // Combined short flags like -rc - let flags = &arg[1..]; - let mut chars = flags.chars().peekable(); - while let Some(c) = chars.next() { - match c { - 'r' => opts.raw_output = true, - 'c' => opts.compact = true, - 'n' => opts.null_input = true, - 's' => opts.slurp = true, - 'p' => { - let rest: String = chars.collect(); - if !rest.is_empty() { - opts.input_format = Some(parse_format(&rest)?); - } else { - i += 1; - if i >= args.len() { - return Err("-p requires a format argument".to_string()); - } - opts.input_format = Some(parse_format(&args[i])?); - } - break; - } - 'o' => { - let rest: String = chars.collect(); - if !rest.is_empty() { - opts.output_format = Some(parse_format(&rest)?); - } else { - i += 1; - if i >= args.len() { - return Err("-o requires a format argument".to_string()); - } - opts.output_format = Some(parse_format(&args[i])?); - } - break; - } - _ => return Err(format!("unknown option: -{}", c)), - } - } - } else if !filter_set { - opts.filter = arg.clone(); - filter_set = true; - } else { - return Err(format!("unexpected argument: {}", arg)); - } - - i += 1; - } - - if !filter_set { - opts.filter = ".".to_string(); - } - - Ok(opts) -} - -fn detect_format(input: &str) -> Format { - let trimmed = input.trim_start(); - - // JSON: starts with { or [ - if trimmed.starts_with('{') || trimmed.starts_with('[') { - if serde_json::from_str::(trimmed).is_ok() { - return Format::Json; - } - } - - // XML: starts with < (including (trimmed).is_ok() - { - return Format::Toml; - } - } - - // Default: YAML - Format::Yaml -} - -fn parse_input(input: &str, format: Format) -> Result { - match format { - Format::Json => serde_json::from_str(input).map_err(|e| format!("invalid JSON: {}", e)), - Format::Yaml => serde_yaml::from_str(input).map_err(|e| format!("invalid YAML: {}", e)), - Format::Toml => { - let toml_val: toml::Value = - toml::from_str(input).map_err(|e| format!("invalid TOML: {}", e))?; - toml_to_json(toml_val) - } - Format::Xml => xml_to_json(input), - } -} - -fn toml_to_json(val: toml::Value) -> Result { - Ok(match val { - toml::Value::String(s) => serde_json::Value::String(s), - toml::Value::Integer(i) => serde_json::Value::Number(serde_json::Number::from(i)), - toml::Value::Float(f) => serde_json::Number::from_f64(f) - .map(serde_json::Value::Number) - .unwrap_or(serde_json::Value::Null), - toml::Value::Boolean(b) => serde_json::Value::Bool(b), - toml::Value::Datetime(dt) => serde_json::Value::String(dt.to_string()), - toml::Value::Array(arr) => { - let items: Result, _> = arr.into_iter().map(toml_to_json).collect(); - serde_json::Value::Array(items?) - } - toml::Value::Table(table) => { - let mut map = serde_json::Map::new(); - for (k, v) in table { - map.insert(k, toml_to_json(v)?); - } - serde_json::Value::Object(map) - } - }) -} - -fn json_to_toml(val: &serde_json::Value) -> Result { - Ok(match val { - serde_json::Value::Null => return Err("TOML does not support null values".to_string()), - serde_json::Value::Bool(b) => toml::Value::Boolean(*b), - serde_json::Value::Number(n) => { - if let Some(i) = n.as_i64() { - toml::Value::Integer(i) - } else if let Some(f) = n.as_f64() { - toml::Value::Float(f) - } else { - return Err("unsupported number for TOML".to_string()); - } - } - serde_json::Value::String(s) => toml::Value::String(s.clone()), - serde_json::Value::Array(arr) => { - let items: Result, _> = arr.iter().map(json_to_toml).collect(); - toml::Value::Array(items?) - } - serde_json::Value::Object(map) => { - let mut table = toml::map::Map::new(); - for (k, v) in map { - table.insert(k.clone(), json_to_toml(v)?); - } - toml::Value::Table(table) - } - }) -} - -// --- XML parsing --- - -fn xml_to_json(input: &str) -> Result { - use quick_xml::events::Event; - use quick_xml::Reader; - - struct StackEntry { - name: String, - children: serde_json::Map, - text: String, - } - - let mut reader = Reader::from_str(input); - let mut stack: Vec = Vec::new(); - let mut root = serde_json::Map::new(); - let mut nodes = 0usize; - - loop { - match reader.read_event() { - Ok(Event::Start(ref e)) => { - count_xml_node(&mut nodes)?; - if stack.len() >= MAX_XML_DEPTH { - return Err("XML exceeds maximum nesting depth".to_string()); - } - let name = String::from_utf8_lossy(e.name().as_ref()).to_string(); - let mut children = serde_json::Map::new(); - let mut attr_count = 0usize; - for attr in e.attributes() { - let attr = attr.map_err(|e| format!("invalid XML attribute: {}", e))?; - attr_count += 1; - if attr_count > MAX_XML_ATTRIBUTES_PER_ELEMENT { - return Err("XML element has too many attributes".to_string()); - } - let key = format!("@{}", String::from_utf8_lossy(attr.key.as_ref())); - let val = String::from_utf8_lossy(&attr.value).to_string(); - children.insert(key, serde_json::Value::String(val)); - } - stack.push(StackEntry { - name, - children, - text: String::new(), - }); - } - Ok(Event::End(_)) => { - let entry = stack.pop().ok_or("unexpected closing tag")?; - let text = entry.text.trim().to_string(); - - let value = if entry.children.is_empty() && text.is_empty() { - serde_json::Value::Null - } else if entry.children.is_empty() { - serde_json::Value::String(text) - } else { - let mut obj = entry.children; - if !text.is_empty() { - obj.insert("#text".to_string(), serde_json::Value::String(text)); - } - serde_json::Value::Object(obj) - }; - - let target = if let Some(parent) = stack.last_mut() { - &mut parent.children - } else { - &mut root - }; - - insert_or_array(target, entry.name, value); - } - Ok(Event::Empty(ref e)) => { - count_xml_node(&mut nodes)?; - if stack.len() >= MAX_XML_DEPTH { - return Err("XML exceeds maximum nesting depth".to_string()); - } - let name = String::from_utf8_lossy(e.name().as_ref()).to_string(); - let mut attrs = serde_json::Map::new(); - let mut attr_count = 0usize; - for attr in e.attributes() { - let attr = attr.map_err(|e| format!("invalid XML attribute: {}", e))?; - attr_count += 1; - if attr_count > MAX_XML_ATTRIBUTES_PER_ELEMENT { - return Err("XML element has too many attributes".to_string()); - } - let key = format!("@{}", String::from_utf8_lossy(attr.key.as_ref())); - let val = String::from_utf8_lossy(&attr.value).to_string(); - attrs.insert(key, serde_json::Value::String(val)); - } - - let value = if attrs.is_empty() { - serde_json::Value::Null - } else { - serde_json::Value::Object(attrs) - }; - - let target = if let Some(parent) = stack.last_mut() { - &mut parent.children - } else { - &mut root - }; - - insert_or_array(target, name, value); - } - Ok(Event::Text(ref e)) => { - if let Some(entry) = stack.last_mut() { - let text = e - .unescape() - .map_err(|e| format!("invalid XML text: {}", e))?; - let next_len = entry - .text - .len() - .checked_add(text.len()) - .ok_or("XML text length overflowed")?; - if next_len > MAX_XML_TEXT_BYTES { - return Err("XML text exceeds size limit".to_string()); - } - entry.text.push_str(&text); - } - } - Ok(Event::Eof) => break, - Ok(_) => {} // Skip PI, Comment, Decl, DocType, CData - Err(e) => return Err(format!("invalid XML: {}", e)), - } - } - - if !stack.is_empty() { - return Err("unexpected end of XML input".to_string()); - } - - Ok(serde_json::Value::Object(root)) -} - -fn count_xml_node(nodes: &mut usize) -> Result<(), String> { - *nodes = nodes.checked_add(1).ok_or("XML node count overflowed")?; - if *nodes > MAX_XML_NODES { - return Err("XML contains too many nodes".to_string()); - } - Ok(()) -} - -fn record_output_value(output_count: &mut usize) -> Result<(), String> { - *output_count = output_count - .checked_add(1) - .ok_or("output count overflowed")?; - if *output_count > MAX_OUTPUT_VALUES { - return Err("too many output values".to_string()); - } - Ok(()) -} - -fn read_limited_string(reader: R) -> Result { - let mut input = String::new(); - reader - .take((MAX_INPUT_BYTES + 1) as u64) - .read_to_string(&mut input) - .map_err(|e| format!("failed to read stdin: {}", e))?; - if input.len() > MAX_INPUT_BYTES { - return Err("stdin exceeds size limit".to_string()); - } - Ok(input) -} - -fn insert_or_array( - map: &mut serde_json::Map, - key: String, - value: serde_json::Value, -) { - if let Some(existing) = map.get_mut(&key) { - match existing { - serde_json::Value::Array(arr) => arr.push(value), - _ => { - let prev = existing.clone(); - *existing = serde_json::Value::Array(vec![prev, value]); - } - } - } else { - map.insert(key, value); - } -} - -// --- XML output --- - -fn json_to_xml(val: &serde_json::Value) -> Result { - use quick_xml::Writer; - - let mut writer = Writer::new(LimitedBytes::new(MAX_FORMATTED_OUTPUT_BYTES)); - - match val { - serde_json::Value::Object(map) => { - for (key, value) in map { - write_xml_element(&mut writer, key, value) - .map_err(|e| format!("XML write error: {}", e))?; - } - } - _ => { - write_xml_element(&mut writer, "root", val) - .map_err(|e| format!("XML write error: {}", e))?; - } - } - - writer - .into_inner() - .into_string() - .map_err(|e| format!("XML encoding error: {}", e)) -} - -fn write_xml_element( - writer: &mut quick_xml::Writer, - name: &str, - val: &serde_json::Value, -) -> Result<(), quick_xml::Error> { - use quick_xml::events::{BytesEnd, BytesStart, BytesText, Event}; - - match val { - serde_json::Value::Object(map) => { - let mut elem = BytesStart::new(name); - for (k, v) in map { - if let Some(attr_name) = k.strip_prefix('@') { - if let serde_json::Value::String(s) = v { - elem.push_attribute((attr_name, s.as_str())); - } - } - } - writer.write_event(Event::Start(elem))?; - - if let Some(serde_json::Value::String(text)) = map.get("#text") { - writer.write_event(Event::Text(BytesText::new(text)))?; - } - - for (k, v) in map { - if k.starts_with('@') || k == "#text" { - continue; - } - match v { - serde_json::Value::Array(arr) => { - for item in arr { - write_xml_element(writer, k, item)?; - } - } - _ => write_xml_element(writer, k, v)?, - } - } - - writer.write_event(Event::End(BytesEnd::new(name)))?; - } - serde_json::Value::Array(arr) => { - for item in arr { - write_xml_element(writer, name, item)?; - } - } - serde_json::Value::String(s) => { - writer.write_event(Event::Start(BytesStart::new(name)))?; - writer.write_event(Event::Text(BytesText::new(s)))?; - writer.write_event(Event::End(BytesEnd::new(name)))?; - } - serde_json::Value::Number(n) => { - let s = n.to_string(); - writer.write_event(Event::Start(BytesStart::new(name)))?; - writer.write_event(Event::Text(BytesText::new(&s)))?; - writer.write_event(Event::End(BytesEnd::new(name)))?; - } - serde_json::Value::Bool(b) => { - let s = if *b { "true" } else { "false" }; - writer.write_event(Event::Start(BytesStart::new(name)))?; - writer.write_event(Event::Text(BytesText::new(s)))?; - writer.write_event(Event::End(BytesEnd::new(name)))?; - } - serde_json::Value::Null => { - writer.write_event(Event::Empty(BytesStart::new(name)))?; - } - } - Ok(()) -} - -// --- Output formatting --- - -fn format_val_output(val: &Val, opts: &YqOptions, out_format: Format) -> Result { - let mut compact = LimitedString::new(MAX_FORMATTED_OUTPUT_BYTES); - fmt::write(&mut compact, format_args!("{}", val)) - .map_err(|_| "formatted output exceeds size limit".to_string())?; - let compact_str = compact.into_string(); - - // Raw output: unquote strings - if opts.raw_output { - if compact_str.starts_with('"') && compact_str.ends_with('"') && compact_str.len() >= 2 { - if let Ok(unescaped) = serde_json::from_str::(&compact_str) { - ensure_formatted_output_limit(unescaped.len())?; - return Ok(unescaped); - } - } - } - - let json_val: serde_json::Value = - serde_json::from_str(&compact_str).unwrap_or(serde_json::Value::String(compact_str)); - - let output = format_json_as(out_format, &json_val, opts.compact)?; - ensure_formatted_output_limit(output.len())?; - Ok(output) -} - -fn ensure_formatted_output_limit(len: usize) -> Result<(), String> { - if len > MAX_FORMATTED_OUTPUT_BYTES { - return Err("formatted output exceeds size limit".to_string()); - } - Ok(()) -} - -struct LimitedString { - inner: String, - limit: usize, -} - -impl LimitedString { - fn new(limit: usize) -> Self { - Self { - inner: String::new(), - limit, - } - } - - fn into_string(self) -> String { - self.inner - } - - fn write_str(&mut self, s: &str) -> Result<(), String> { - let next_len = self - .inner - .len() - .checked_add(s.len()) - .ok_or("formatted output length overflowed")?; - if next_len > self.limit { - return Err("formatted output exceeds size limit".to_string()); - } - self.inner.push_str(s); - Ok(()) - } - - fn write_char(&mut self, ch: char) -> Result<(), String> { - let mut buf = [0u8; 4]; - self.write_str(ch.encode_utf8(&mut buf)) - } -} - -impl fmt::Write for LimitedString { - fn write_str(&mut self, s: &str) -> fmt::Result { - LimitedString::write_str(self, s).map_err(|_| fmt::Error) - } -} - -struct LimitedBytes { - inner: Vec, - limit: usize, -} - -impl LimitedBytes { - fn new(limit: usize) -> Self { - Self { - inner: Vec::new(), - limit, - } - } - - fn into_string(self) -> Result { - String::from_utf8(self.inner) - } -} - -impl io::Write for LimitedBytes { - fn write(&mut self, buf: &[u8]) -> io::Result { - let next_len = self - .inner - .len() - .checked_add(buf.len()) - .ok_or_else(|| io::Error::other("formatted output length overflowed"))?; - if next_len > self.limit { - return Err(io::Error::other("formatted output exceeds size limit")); - } - self.inner.extend_from_slice(buf); - Ok(buf.len()) - } - - fn flush(&mut self) -> io::Result<()> { - Ok(()) - } -} - -fn format_json_as( - format: Format, - val: &serde_json::Value, - compact: bool, -) -> Result { - match format { - Format::Json => { - let mut out = LimitedBytes::new(MAX_FORMATTED_OUTPUT_BYTES); - if compact { - serde_json::to_writer(&mut out, val) - .map_err(|e| format!("JSON output error: {}", e))?; - } else { - serde_json::to_writer_pretty(&mut out, val) - .map_err(|e| format!("JSON output error: {}", e))?; - } - out.into_string() - .map_err(|e| format!("JSON encoding error: {}", e)) - } - Format::Yaml => { - let mut out = LimitedBytes::new(MAX_FORMATTED_OUTPUT_BYTES); - serde_yaml::to_writer(&mut out, val) - .map_err(|e| format!("YAML output error: {}", e))?; - let s = out - .into_string() - .map_err(|e| format!("YAML encoding error: {}", e))?; - // Strip leading "---\n" and trailing newline for cleaner output - let s = s.strip_prefix("---\n").unwrap_or(&s); - let s = s.strip_suffix('\n').unwrap_or(s); - Ok(s.to_string()) - } - Format::Toml => json_to_toml_bounded(val), - Format::Xml => json_to_xml(val), - } -} - -fn json_to_toml_bounded(val: &serde_json::Value) -> Result { - let toml_val = json_to_toml(val)?; - let mut out = LimitedString::new(MAX_FORMATTED_OUTPUT_BYTES); - write_toml_document(&mut out, &toml_val)?; - let s = out.into_string(); - Ok(s.strip_suffix('\n').unwrap_or(&s).to_string()) -} - -fn write_toml_document(out: &mut LimitedString, val: &toml::Value) -> Result<(), String> { - match val { - toml::Value::Table(table) => write_toml_table(out, &mut Vec::new(), table), - other => write_toml_inline(out, other), - } -} - -fn write_toml_table( - out: &mut LimitedString, - path: &mut Vec, - table: &toml::map::Map, -) -> Result<(), String> { - for (key, value) in table { - if matches!(value, toml::Value::Table(_)) { - continue; - } - write_toml_key(out, key)?; - out.write_str(" = ")?; - write_toml_inline(out, value)?; - out.write_char('\n')?; - } - - for (key, value) in table { - let toml::Value::Table(child) = value else { - continue; - }; - if !path.is_empty() || table_has_scalar_entries(child) { - out.write_char('\n')?; - path.push(key.clone()); - out.write_char('[')?; - write_toml_path(out, path)?; - out.write_str("]\n")?; - write_toml_table(out, path, child)?; - path.pop(); - } else { - path.push(key.clone()); - write_toml_table(out, path, child)?; - path.pop(); - } - } - - Ok(()) -} - -fn table_has_scalar_entries(table: &toml::map::Map) -> bool { - table - .values() - .any(|value| !matches!(value, toml::Value::Table(_))) -} - -fn write_toml_path(out: &mut LimitedString, path: &[String]) -> Result<(), String> { - for (i, key) in path.iter().enumerate() { - if i > 0 { - out.write_char('.')?; - } - write_toml_key(out, key)?; - } - Ok(()) -} - -fn write_toml_key(out: &mut LimitedString, key: &str) -> Result<(), String> { - if !key.is_empty() - && key - .bytes() - .all(|b| b.is_ascii_alphanumeric() || b == b'_' || b == b'-') - { - out.write_str(key)?; - } else { - write_toml_string(out, key)?; - } - Ok(()) -} - -fn write_toml_inline(out: &mut LimitedString, val: &toml::Value) -> Result<(), String> { - match val { - toml::Value::String(s) => write_toml_string(out, s), - toml::Value::Integer(i) => out.write_str(&i.to_string()), - toml::Value::Float(f) => out.write_str(&f.to_string()), - toml::Value::Boolean(b) => out.write_str(if *b { "true" } else { "false" }), - toml::Value::Datetime(dt) => out.write_str(&dt.to_string()), - toml::Value::Array(arr) => { - out.write_char('[')?; - for (i, item) in arr.iter().enumerate() { - if i > 0 { - out.write_str(", ")?; - } - write_toml_inline(out, item)?; - } - out.write_char(']') - } - toml::Value::Table(table) => { - out.write_str("{ ")?; - for (i, (key, value)) in table.iter().enumerate() { - if i > 0 { - out.write_str(", ")?; - } - write_toml_key(out, key)?; - out.write_str(" = ")?; - write_toml_inline(out, value)?; - } - out.write_str(" }") - } - } -} - -fn write_toml_string(out: &mut LimitedString, s: &str) -> Result<(), String> { - out.write_char('"')?; - for ch in s.chars() { - match ch { - '"' => out.write_str("\\\"")?, - '\\' => out.write_str("\\\\")?, - '\n' => out.write_str("\\n")?, - '\r' => out.write_str("\\r")?, - '\t' => out.write_str("\\t")?, - '\u{08}' => out.write_str("\\b")?, - '\u{0c}' => out.write_str("\\f")?, - ch if ch.is_control() => out.write_str(&format!("\\u{:04X}", ch as u32))?, - ch => out.write_char(ch)?, - } - } - out.write_char('"') -} - -// --- Main logic --- - -fn run_yq(args: &[String]) -> Result { - let opts = parse_args(args)?; - - // Read input - let stdin_data = if opts.null_input { - String::new() - } else { - read_limited_string(io::stdin())? - }; - - // Determine input format - let in_format = opts.input_format.unwrap_or_else(|| { - if opts.null_input { - Format::Yaml - } else { - detect_format(&stdin_data) - } - }); - - // Default output format: YAML for YAML input, otherwise matches input - let out_format = opts.output_format.unwrap_or(in_format); - - // Parse input to JSON, then convert to jaq Val - let inputs = if opts.null_input { - vec![Val::from(serde_json::Value::Null)] - } else { - let json_val = parse_input(&stdin_data, in_format)?; - if opts.slurp { - match json_val { - serde_json::Value::Array(_) => vec![Val::from(json_val)], - _ => vec![Val::from(serde_json::Value::Array(vec![json_val]))], - } - } else { - vec![Val::from(json_val)] - } - }; - - // Compile jaq filter (same engine as jq) - let loader = Loader::new(jaq_std::defs().chain(jaq_json::defs())); - let arena = Arena::default(); - let program = File { - code: opts.filter.as_str(), - path: (), - }; - let modules = loader - .load(&arena, program) - .map_err(|errs| format!("parse error: {:?}", errs))?; - - let filter = Compiler::default() - .with_funs(jaq_std::funs().chain(jaq_json::funs())) - .compile(modules) - .map_err(|errs| format!("compile error: {:?}", errs))?; - - // Execute filter and output - let empty_inputs = RcIter::new(core::iter::empty()); - let stdout = io::stdout(); - let mut out = stdout.lock(); - let mut output_count = 0usize; - - for input in inputs { - let ctx = Ctx::new(core::iter::empty(), &empty_inputs); - let results = filter.run((ctx, input)); - - for result in results { - match result { - Ok(val) => { - record_output_value(&mut output_count)?; - let s = format_val_output(&val, &opts, out_format)?; - writeln!(out, "{}", s).map_err(|e| format!("failed to write stdout: {}", e))?; - } - Err(e) => { - eprintln!("yq: error: {}", e); - return Ok(5); - } - } - } - } - - out.flush() - .map_err(|e| format!("failed to flush stdout: {}", e))?; - - Ok(0) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn xml_depth_limit_rejects_deep_input() { - let mut input = String::new(); - for i in 0..=MAX_XML_DEPTH { - input.push_str(&format!("")); - } - for i in (0..=MAX_XML_DEPTH).rev() { - input.push_str(&format!("")); - } - - let err = xml_to_json(&input).unwrap_err(); - - assert!(err.contains("nesting depth")); - } - - #[test] - fn xml_depth_limit_rejects_deep_empty_input() { - let mut input = String::new(); - for i in 0..MAX_XML_DEPTH { - input.push_str(&format!("")); - } - input.push_str(""); - for i in (0..MAX_XML_DEPTH).rev() { - input.push_str(&format!("")); - } - - let err = xml_to_json(&input).unwrap_err(); - - assert!(err.contains("nesting depth")); - } - - #[test] - fn xml_node_limit_rejects_many_elements() { - let mut nodes = MAX_XML_NODES; - - let err = count_xml_node(&mut nodes).unwrap_err(); - - assert!(err.contains("too many nodes")); - } - - #[test] - fn xml_text_limit_rejects_large_text() { - let input = format!("{}", "x".repeat(MAX_XML_TEXT_BYTES + 1)); - - let err = xml_to_json(&input).unwrap_err(); - - assert!(err.contains("text exceeds")); - } - - #[test] - fn output_limit_rejects_too_many_values() { - let mut count = MAX_OUTPUT_VALUES; - - let err = record_output_value(&mut count).unwrap_err(); - - assert!(err.contains("too many output values")); - } - - #[test] - fn formatted_output_limit_rejects_large_output() { - let err = ensure_formatted_output_limit(MAX_FORMATTED_OUTPUT_BYTES + 1).unwrap_err(); - - assert!(err.contains("formatted output")); - } - - #[test] - fn limited_bytes_rejects_large_serializer_write() { - let mut output = LimitedBytes::new(4); - - let err = output.write_all(b"hello").unwrap_err(); - - assert!(err.to_string().contains("formatted output")); - } - - #[test] - fn limited_toml_writer_rejects_large_value() { - let mut output = LimitedString::new(4); - let value = toml::Value::String("hello".to_string()); - - let err = write_toml_inline(&mut output, &value).unwrap_err(); - - assert!(err.contains("formatted output")); - } - - #[test] - fn input_limit_rejects_oversized_reader() { - let input = vec![b'x'; MAX_INPUT_BYTES + 1]; - - let err = read_limited_string(&input[..]).unwrap_err(); - - assert!(err.contains("stdin exceeds")); - } - - #[test] - fn xml_rejects_unclosed_elements() { - let err = xml_to_json("").unwrap_err(); - - assert!(err.contains("unexpected end")); - } - - #[test] - fn xml_rejects_invalid_text_escape() { - let err = xml_to_json("&bogus;").unwrap_err(); - - assert!(err.contains("invalid XML text")); - } -} diff --git a/registry/native/crates/wasi-ext/src/lib.rs b/registry/native/crates/wasi-ext/src/lib.rs deleted file mode 100644 index 89c404f268..0000000000 --- a/registry/native/crates/wasi-ext/src/lib.rs +++ /dev/null @@ -1,973 +0,0 @@ -//! Custom WASM import bindings for wasmVM host syscalls. -//! -//! Declares extern functions for `host_process`, `host_user`, and `host_net` -//! modules that the JS host runtime provides. These extend standard WASI with -//! process management, user/group identity, and TCP socket capabilities. -//! -//! Signatures match spec section 4.3. - -#![no_std] - -/// WASI-style errno type. 0 = success. -pub type Errno = u32; - -// WASI errno constants -pub const ERRNO_SUCCESS: Errno = 0; -pub const ERRNO_BADF: Errno = 8; -pub const ERRNO_INVAL: Errno = 28; -pub const ERRNO_NOSYS: Errno = 52; -pub const ERRNO_NOENT: Errno = 44; -pub const ERRNO_SRCH: Errno = 71; // No such process -pub const ERRNO_CHILD: Errno = 10; // No child processes -/// WASI `errno::again` (EAGAIN/EWOULDBLOCK). Returned by a non-blocking `recv` -/// when no data is currently available. -pub const ERRNO_AGAIN: Errno = 6; -const POLLFD_BYTES: usize = 8; - -/// `SOL_SOCKET` socket option level (matches the host_net shim's accepted level). -pub const SOL_SOCKET: u32 = 1; -/// `SO_RCVTIMEO` recv-timeout socket option name (64-bit timeval layout, which the -/// host_net shim parses as two little-endian `i64`s: seconds + microseconds). -pub const SO_RCVTIMEO: u32 = 20; -/// Size of the `timeval` struct the host_net shim expects for `SO_RCVTIMEO` -/// (two 64-bit fields: `tv_sec` + `tv_usec`). -const TIMEVAL_BYTES: usize = 16; - -fn checked_u32_len(len: usize) -> Result { - u32::try_from(len).map_err(|_| ERRNO_INVAL) -} - -fn validate_returned_len(len: u32, capacity: usize) -> Result { - match usize::try_from(len) { - Ok(len) if len <= capacity => Ok(len as u32), - _ => Err(ERRNO_INVAL), - } -} - -fn validate_poll_buffer_len(buffer_len: usize, nfds: u32) -> Result<(), Errno> { - let nfds = usize::try_from(nfds).map_err(|_| ERRNO_INVAL)?; - let expected = nfds.checked_mul(POLLFD_BYTES).ok_or(ERRNO_INVAL)?; - if buffer_len == expected { - Ok(()) - } else { - Err(ERRNO_INVAL) - } -} - -fn validate_poll_ready_count(ready: u32, nfds: u32) -> Result { - if ready <= nfds { - Ok(ready) - } else { - Err(ERRNO_INVAL) - } -} - -// ============================================================ -// host_process module — process management and FD operations -// ============================================================ - -#[link(wasm_import_module = "host_process")] -extern "C" { - /// Spawn a child process. - /// - /// Arguments are serialized as a byte buffer pointed to by `argv_ptr`/`argv_len`. - /// Environment is serialized similarly via `envp_ptr`/`envp_len`. - /// File descriptors `stdin_fd`, `stdout_fd`, `stderr_fd` are inherited. - /// Current working directory is passed as `cwd_ptr`/`cwd_len`. - /// On success, the child's virtual PID is written to `ret_pid`. - /// Returns errno. - fn proc_spawn( - argv_ptr: *const u8, - argv_len: u32, - envp_ptr: *const u8, - envp_len: u32, - stdin_fd: u32, - stdout_fd: u32, - stderr_fd: u32, - cwd_ptr: *const u8, - cwd_len: u32, - ret_pid: *mut u32, - ) -> Errno; - - /// Wait for a child process to exit. - /// - /// Blocks (via Atomics.wait on the host side) until the child exits. - /// `options` is reserved (pass 0). Exit status is written to `ret_status`. - /// The actual waited-for PID is written to `ret_pid` (important for pid=-1). - /// Returns errno. - fn proc_waitpid(pid: u32, options: u32, ret_status: *mut u32, ret_pid: *mut u32) -> Errno; - - /// Send a signal to a process. - /// - /// Only SIGTERM (15) and SIGKILL (9) are meaningful. - /// Returns errno. - fn proc_kill(pid: u32, signal: u32) -> Errno; - - /// Get the current process's virtual PID. - /// - /// Writes PID to `ret_pid`. Returns errno. - fn proc_getpid(ret_pid: *mut u32) -> Errno; - - /// Get the parent process's virtual PID. - /// - /// Writes parent PID to `ret_pid`. Returns errno. - fn proc_getppid(ret_pid: *mut u32) -> Errno; - - /// Create an anonymous pipe. - /// - /// Writes the read-end FD to `ret_read_fd` and write-end FD to `ret_write_fd`. - /// Returns errno. - fn fd_pipe(ret_read_fd: *mut u32, ret_write_fd: *mut u32) -> Errno; - - /// Duplicate a file descriptor. - /// - /// The new FD number is written to `ret_new_fd`. Returns errno. - fn fd_dup(fd: u32, ret_new_fd: *mut u32) -> Errno; - - /// Duplicate a file descriptor to a specific number. - /// - /// `old_fd` is duplicated to `new_fd`. If `new_fd` is already open, it is closed first. - /// Returns errno. - fn fd_dup2(old_fd: u32, new_fd: u32) -> Errno; - - /// Sleep for the specified number of milliseconds. - /// - /// Blocks via Atomics.wait on the host side. Returns errno. - fn sleep_ms(milliseconds: u32) -> Errno; - - /// Allocate a pseudo-terminal (PTY) master/slave pair. - /// - /// On success, the master FD is written to `ret_master_fd` and the slave FD - /// to `ret_slave_fd`. Both ends are installed in the process's kernel FD table. - /// Returns errno. - fn pty_open(ret_master_fd: *mut u32, ret_slave_fd: *mut u32) -> Errno; - - /// Register a signal handler disposition (POSIX sigaction). - /// - /// `signal` is the signal number (1-64). - /// `action` encodes the disposition: 0=SIG_DFL, 1=SIG_IGN, 2=user handler. - /// `mask_lo` / `mask_hi` encode the low/high 32 bits of sa_mask, and `flags` - /// carries the raw POSIX sa_flags bitmask. - /// When action=2, the C sysroot still holds the actual function pointer; the - /// kernel only needs the metadata that affects delivery semantics. - /// Returns errno. - fn proc_sigaction(signal: u32, action: u32, mask_lo: u32, mask_hi: u32, flags: u32) -> Errno; -} - -// ============================================================ -// host_user module — user/group identity and terminal detection -// ============================================================ - -#[link(wasm_import_module = "host_user")] -extern "C" { - /// Get the real user ID. Writes to `ret_uid`. Returns errno. - fn getuid(ret_uid: *mut u32) -> Errno; - - /// Get the real group ID. Writes to `ret_gid`. Returns errno. - fn getgid(ret_gid: *mut u32) -> Errno; - - /// Get the effective user ID. Writes to `ret_uid`. Returns errno. - fn geteuid(ret_uid: *mut u32) -> Errno; - - /// Get the effective group ID. Writes to `ret_gid`. Returns errno. - fn getegid(ret_gid: *mut u32) -> Errno; - - /// Check if a file descriptor refers to a terminal. - /// - /// Writes 1 (true) or 0 (false) to `ret_bool`. Returns errno. - fn isatty(fd: u32, ret_bool: *mut u32) -> Errno; - - /// Get passwd entry for a user ID. - /// - /// Serialized passwd string (username:x:uid:gid:gecos:home:shell) is written - /// to `buf_ptr` with max length `buf_len`. Actual length written to `ret_len`. - /// Returns errno. - fn getpwuid(uid: u32, buf_ptr: *mut u8, buf_len: u32, ret_len: *mut u32) -> Errno; -} - -// ============================================================ -// Safe Rust wrappers — host_process -// ============================================================ - -/// Spawn a child process with the given arguments, environment, stdio FDs, and working directory. -/// -/// Returns `Ok(pid)` on success, `Err(errno)` on failure. -pub fn spawn( - argv: &[u8], - envp: &[u8], - stdin_fd: u32, - stdout_fd: u32, - stderr_fd: u32, - cwd: &[u8], -) -> Result { - let mut pid: u32 = 0; - let argv_len = checked_u32_len(argv.len())?; - let envp_len = checked_u32_len(envp.len())?; - let cwd_len = checked_u32_len(cwd.len())?; - let errno = unsafe { - proc_spawn( - argv.as_ptr(), - argv_len, - envp.as_ptr(), - envp_len, - stdin_fd, - stdout_fd, - stderr_fd, - cwd.as_ptr(), - cwd_len, - &mut pid, - ) - }; - if errno == ERRNO_SUCCESS { - Ok(pid) - } else { - Err(errno) - } -} - -/// Wait for a child process to exit. -/// -/// Returns `Ok((exit_status, actual_pid))` on success, `Err(errno)` on failure. -/// The actual_pid is the PID of the child that exited (relevant for pid=0xFFFFFFFF / -1). -pub fn waitpid(pid: u32, options: u32) -> Result<(u32, u32), Errno> { - let mut status: u32 = 0; - let mut actual_pid: u32 = 0; - let errno = unsafe { proc_waitpid(pid, options, &mut status, &mut actual_pid) }; - if errno == ERRNO_SUCCESS { - Ok((status, actual_pid)) - } else { - Err(errno) - } -} - -/// Send a signal to a process. -/// -/// Returns `Ok(())` on success, `Err(errno)` on failure. -pub fn kill(pid: u32, signal: u32) -> Result<(), Errno> { - let errno = unsafe { proc_kill(pid, signal) }; - if errno == ERRNO_SUCCESS { - Ok(()) - } else { - Err(errno) - } -} - -/// Get the current process's virtual PID. -/// -/// Returns `Ok(pid)` on success, `Err(errno)` on failure. -pub fn getpid() -> Result { - let mut pid: u32 = 0; - let errno = unsafe { proc_getpid(&mut pid) }; - if errno == ERRNO_SUCCESS { - Ok(pid) - } else { - Err(errno) - } -} - -/// Get the parent process's virtual PID. -/// -/// Returns `Ok(pid)` on success, `Err(errno)` on failure. -pub fn getppid() -> Result { - let mut pid: u32 = 0; - let errno = unsafe { proc_getppid(&mut pid) }; - if errno == ERRNO_SUCCESS { - Ok(pid) - } else { - Err(errno) - } -} - -/// Create an anonymous pipe. -/// -/// Returns `Ok((read_fd, write_fd))` on success, `Err(errno)` on failure. -pub fn pipe() -> Result<(u32, u32), Errno> { - let mut read_fd: u32 = 0; - let mut write_fd: u32 = 0; - let errno = unsafe { fd_pipe(&mut read_fd, &mut write_fd) }; - if errno == ERRNO_SUCCESS { - Ok((read_fd, write_fd)) - } else { - Err(errno) - } -} - -/// Duplicate a file descriptor. -/// -/// Returns `Ok(new_fd)` on success, `Err(errno)` on failure. -pub fn dup(fd: u32) -> Result { - let mut new_fd: u32 = 0; - let errno = unsafe { fd_dup(fd, &mut new_fd) }; - if errno == ERRNO_SUCCESS { - Ok(new_fd) - } else { - Err(errno) - } -} - -/// Duplicate a file descriptor to a specific number. -/// -/// Returns `Ok(())` on success, `Err(errno)` on failure. -pub fn dup2(old_fd: u32, new_fd: u32) -> Result<(), Errno> { - let errno = unsafe { fd_dup2(old_fd, new_fd) }; - if errno == ERRNO_SUCCESS { - Ok(()) - } else { - Err(errno) - } -} - -/// Sleep for the specified number of milliseconds. -/// -/// Blocks via Atomics.wait on the host side instead of busy-waiting. -/// Returns `Ok(())` on success, `Err(errno)` on failure. -pub fn host_sleep_ms(milliseconds: u32) -> Result<(), Errno> { - let errno = unsafe { sleep_ms(milliseconds) }; - if errno == ERRNO_SUCCESS { - Ok(()) - } else { - Err(errno) - } -} - -/// Allocate a pseudo-terminal (PTY) master/slave pair. -/// -/// Returns `Ok((master_fd, slave_fd))` on success, `Err(errno)` on failure. -/// The master FD is used to read output and write input. -/// The slave FD is passed to a child process as its stdin/stdout/stderr. -pub fn openpty() -> Result<(u32, u32), Errno> { - let mut master_fd: u32 = 0; - let mut slave_fd: u32 = 0; - let errno = unsafe { pty_open(&mut master_fd, &mut slave_fd) }; - if errno == ERRNO_SUCCESS { - Ok((master_fd, slave_fd)) - } else { - Err(errno) - } -} - -/// Register a signal handler disposition (POSIX sigaction). -/// -/// `signal` is the signal number (1-64). -/// `action` encodes the disposition: 0=SIG_DFL, 1=SIG_IGN, 2=user handler (C-side holds pointer). -/// `mask_lo` / `mask_hi` encode the low/high 32 bits of sa_mask, and `flags` -/// carries the raw POSIX sa_flags bitmask. -/// Returns `Ok(())` on success, `Err(errno)` on failure. -pub fn sigaction_set( - signal: u32, - action: u32, - mask_lo: u32, - mask_hi: u32, - flags: u32, -) -> Result<(), Errno> { - let errno = unsafe { proc_sigaction(signal, action, mask_lo, mask_hi, flags) }; - if errno == ERRNO_SUCCESS { - Ok(()) - } else { - Err(errno) - } -} - -// ============================================================ -// host_net module — TCP socket operations -// ============================================================ - -#[link(wasm_import_module = "host_net")] -extern "C" { - /// Create a socket. - /// - /// `domain` is the address family (e.g. AF_INET=2). - /// `sock_type` is the socket type (e.g. SOCK_STREAM=1). - /// `protocol` is the protocol (0 for default). - /// On success, the socket FD is written to `ret_fd`. - /// Returns errno. - fn net_socket(domain: u32, sock_type: u32, protocol: u32, ret_fd: *mut u32) -> Errno; - - /// Connect a socket to a remote address. - /// - /// `addr_ptr`/`addr_len` point to a serialized address string (host:port). - /// Returns errno. - fn net_connect(fd: u32, addr_ptr: *const u8, addr_len: u32) -> Errno; - - /// Send data on a connected socket. - /// - /// `buf_ptr`/`buf_len` point to the data to send. - /// `flags` are send flags (0 for default). - /// Number of bytes sent is written to `ret_sent`. - /// Returns errno. - fn net_send(fd: u32, buf_ptr: *const u8, buf_len: u32, flags: u32, ret_sent: *mut u32) - -> Errno; - - /// Receive data from a connected socket. - /// - /// `buf_ptr`/`buf_len` point to the receive buffer. - /// `flags` are recv flags (0 for default). - /// Number of bytes received is written to `ret_received`. - /// Returns errno. - fn net_recv( - fd: u32, - buf_ptr: *mut u8, - buf_len: u32, - flags: u32, - ret_received: *mut u32, - ) -> Errno; - - /// Close a socket. - /// - /// Returns errno. - fn net_close(fd: u32) -> Errno; - - /// Resolve a hostname to an address. - /// - /// `host_ptr`/`host_len` point to the hostname string. - /// `port_ptr`/`port_len` point to the port/service string. - /// `family` is 0 for any address family, 4 for IPv4, and 6 for IPv6. - /// Resolved address is written to `ret_addr` buffer with max length from `ret_addr_len`. - /// Actual length is written back to `ret_addr_len`. - /// Returns errno. - fn net_getaddrinfo( - host_ptr: *const u8, - host_len: u32, - port_ptr: *const u8, - port_len: u32, - family: u32, - ret_addr: *mut u8, - ret_addr_len: *mut u32, - ) -> Errno; - - /// Upgrade a connected TCP socket to TLS. - /// - /// `hostname_ptr`/`hostname_len` point to the SNI hostname string. - /// After success, net_send/net_recv on this fd use the encrypted TLS stream. - /// Returns errno. - fn net_tls_connect(fd: u32, hostname_ptr: *const u8, hostname_len: u32) -> Errno; - - /// Set a socket option. - /// - /// `level` is the protocol level (e.g. SOL_SOCKET=1). - /// `optname` is the option name. - /// `optval_ptr`/`optval_len` point to the option value. - /// Returns errno. - fn net_setsockopt( - fd: u32, - level: u32, - optname: u32, - optval_ptr: *const u8, - optval_len: u32, - ) -> Errno; - - /// Get the local address of a socket. - /// - /// The serialized address string is written to `ret_addr` with maximum - /// length from `ret_addr_len`. The actual length is written back. - /// Returns errno. - fn net_getsockname(fd: u32, ret_addr: *mut u8, ret_addr_len: *mut u32) -> Errno; - - /// Get the peer address of a connected socket. - /// - /// The serialized address string is written to `ret_addr` with maximum - /// length from `ret_addr_len`. The actual length is written back. - /// Returns errno. - fn net_getpeername(fd: u32, ret_addr: *mut u8, ret_addr_len: *mut u32) -> Errno; - - /// Poll socket FDs for readiness. - /// - /// `fds_ptr` points to a packed array of poll entries (8 bytes each): - /// [fd: i32, events: i16, revents: i16] per entry. - /// `nfds` is the number of entries. - /// `timeout_ms` is the timeout: 0=non-blocking, -1=block forever, >0=milliseconds. - /// On return, revents fields are updated in-place and `ret_ready` receives - /// the number of FDs with non-zero revents. - /// Returns errno. - fn net_poll(fds_ptr: *mut u8, nfds: u32, timeout_ms: i32, ret_ready: *mut u32) -> Errno; - - /// Bind a socket to a local address. - /// - /// `addr_ptr`/`addr_len` point to a serialized address string (host:port or unix path). - /// Returns errno. - fn net_bind(fd: u32, addr_ptr: *const u8, addr_len: u32) -> Errno; - - /// Mark a bound socket as listening for incoming connections. - /// - /// `backlog` is the maximum pending connection queue length. - /// Returns errno. - fn net_listen(fd: u32, backlog: u32) -> Errno; - - /// Accept an incoming connection on a listening socket. - /// - /// On success, the new connected socket FD is written to `ret_fd`, - /// and the remote address string is written to `ret_addr` with its - /// length in `ret_addr_len`. - /// Returns errno. - fn net_accept(fd: u32, ret_fd: *mut u32, ret_addr: *mut u8, ret_addr_len: *mut u32) -> Errno; - - /// Send a datagram to a specific destination address (UDP). - /// - /// `buf_ptr`/`buf_len` point to the data to send. - /// `flags` are send flags (0 for default). - /// `addr_ptr`/`addr_len` point to the destination address string (host:port). - /// Number of bytes sent is written to `ret_sent`. - /// Returns errno. - fn net_sendto( - fd: u32, - buf_ptr: *const u8, - buf_len: u32, - flags: u32, - addr_ptr: *const u8, - addr_len: u32, - ret_sent: *mut u32, - ) -> Errno; - - /// Receive a datagram from a UDP socket with source address. - /// - /// `buf_ptr`/`buf_len` point to the receive buffer. - /// `flags` are recv flags (0 for default). - /// Number of bytes received is written to `ret_received`. - /// Source address string is written to `ret_addr` with length in `ret_addr_len`. - /// Returns errno. - fn net_recvfrom( - fd: u32, - buf_ptr: *mut u8, - buf_len: u32, - flags: u32, - ret_received: *mut u32, - ret_addr: *mut u8, - ret_addr_len: *mut u32, - ) -> Errno; -} - -// ============================================================ -// Safe Rust wrappers — host_net -// ============================================================ - -/// Create a socket. -/// -/// Returns `Ok(fd)` on success, `Err(errno)` on failure. -pub fn socket(domain: u32, sock_type: u32, protocol: u32) -> Result { - let mut fd: u32 = 0; - let errno = unsafe { net_socket(domain, sock_type, protocol, &mut fd) }; - if errno == ERRNO_SUCCESS { - Ok(fd) - } else { - Err(errno) - } -} - -/// Connect a socket to a remote address. -/// -/// `addr` is a serialized address string (e.g. "host:port"). -/// Returns `Ok(())` on success, `Err(errno)` on failure. -pub fn connect(fd: u32, addr: &[u8]) -> Result<(), Errno> { - let addr_len = checked_u32_len(addr.len())?; - let errno = unsafe { net_connect(fd, addr.as_ptr(), addr_len) }; - if errno == ERRNO_SUCCESS { - Ok(()) - } else { - Err(errno) - } -} - -/// Send data on a connected socket. -/// -/// Returns `Ok(bytes_sent)` on success, `Err(errno)` on failure. -pub fn send(fd: u32, buf: &[u8], flags: u32) -> Result { - let buf_len = checked_u32_len(buf.len())?; - let mut sent: u32 = 0; - let errno = unsafe { net_send(fd, buf.as_ptr(), buf_len, flags, &mut sent) }; - if errno == ERRNO_SUCCESS { - validate_returned_len(sent, buf.len()) - } else { - Err(errno) - } -} - -/// Receive data from a connected socket. -/// -/// Returns `Ok(bytes_received)` on success, `Err(errno)` on failure. -pub fn recv(fd: u32, buf: &mut [u8], flags: u32) -> Result { - let buf_len = checked_u32_len(buf.len())?; - let mut received: u32 = 0; - let errno = unsafe { net_recv(fd, buf.as_mut_ptr(), buf_len, flags, &mut received) }; - if errno == ERRNO_SUCCESS { - validate_returned_len(received, buf.len()) - } else { - Err(errno) - } -} - -/// Outcome of a cooperative (non-blocking) `recv`. -/// -/// Distinguishes "no data right now, try again later" (`WouldBlock`) from real -/// data, EOF, and hard errors so callers can yield to the runtime instead of -/// blocking the single guest thread. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum RecvOutcome { - /// Read `usize` bytes into the buffer. - Read(usize), - /// Peer closed the connection (orderly EOF). - Eof, - /// No data available yet; the socket has a non-zero `SO_RCVTIMEO` set and the - /// host returned `EAGAIN`. Caller should yield and re-poll. - WouldBlock, -} - -/// Receive data, mapping the host's `EAGAIN` to [`RecvOutcome::WouldBlock`]. -/// -/// Use this on sockets that have opted into non-blocking behavior via -/// [`set_recv_timeout_ms`]. On such sockets the host polls briefly then returns -/// `EAGAIN` instead of blocking the thread, letting the caller cooperatively -/// yield. Sockets with no recv timeout still block (this returns `Read`/`Eof`). -pub fn recv_cooperative(fd: u32, buf: &mut [u8], flags: u32) -> Result { - match recv(fd, buf, flags) { - Ok(0) => Ok(RecvOutcome::Eof), - Ok(n) => Ok(RecvOutcome::Read(n as usize)), - Err(ERRNO_AGAIN) => Ok(RecvOutcome::WouldBlock), - Err(e) => Err(e), - } -} - -/// Mark a socket non-blocking for recv by setting a small, non-zero -/// `SO_RCVTIMEO`. -/// -/// The host_net shim polls up to this timeout then returns `EAGAIN` when no data -/// arrived. A zero timeout is rejected by the host (it would mean "blocking"), -/// so callers should pass a small non-zero value (e.g. 2ms). Leaving a socket -/// without ever calling this keeps the default blocking recv behavior, so other -/// guests are unaffected. -pub fn set_recv_timeout_ms(fd: u32, millis: u32) -> Result<(), Errno> { - let micros: u64 = (millis as u64).saturating_mul(1000); - let secs = micros / 1_000_000; - let usec = micros % 1_000_000; - let mut timeval = [0u8; TIMEVAL_BYTES]; - timeval[0..8].copy_from_slice(&secs.to_le_bytes()); - timeval[8..16].copy_from_slice(&usec.to_le_bytes()); - setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeval) -} - -/// Close a socket. -/// -/// Returns `Ok(())` on success, `Err(errno)` on failure. -pub fn net_close_socket(fd: u32) -> Result<(), Errno> { - let errno = unsafe { net_close(fd) }; - if errno == ERRNO_SUCCESS { - Ok(()) - } else { - Err(errno) - } -} - -/// Resolve a hostname to an address. -/// -/// Writes the resolved address into `buf` and returns the number of bytes written. -/// Returns `Ok(len)` on success, `Err(errno)` on failure. -pub fn getaddrinfo(host: &[u8], port: &[u8], buf: &mut [u8]) -> Result { - let host_len = checked_u32_len(host.len())?; - let port_len = checked_u32_len(port.len())?; - let mut len = checked_u32_len(buf.len())?; - let errno = unsafe { - net_getaddrinfo( - host.as_ptr(), - host_len, - port.as_ptr(), - port_len, - 0, - buf.as_mut_ptr(), - &mut len, - ) - }; - if errno == ERRNO_SUCCESS { - validate_returned_len(len, buf.len()) - } else { - Err(errno) - } -} - -/// Set a socket option. -/// -/// Returns `Ok(())` on success, `Err(errno)` on failure. -pub fn setsockopt(fd: u32, level: u32, optname: u32, optval: &[u8]) -> Result<(), Errno> { - let optval_len = checked_u32_len(optval.len())?; - let errno = unsafe { net_setsockopt(fd, level, optname, optval.as_ptr(), optval_len) }; - if errno == ERRNO_SUCCESS { - Ok(()) - } else { - Err(errno) - } -} - -/// Get the local address of a socket. -/// -/// Writes the serialized address into `buf` and returns the number of bytes written. -/// Returns `Ok(len)` on success, `Err(errno)` on failure. -pub fn getsockname(fd: u32, buf: &mut [u8]) -> Result { - let mut len = checked_u32_len(buf.len())?; - let errno = unsafe { net_getsockname(fd, buf.as_mut_ptr(), &mut len) }; - if errno == ERRNO_SUCCESS { - validate_returned_len(len, buf.len()) - } else { - Err(errno) - } -} - -/// Get the peer address of a connected socket. -/// -/// Writes the serialized address into `buf` and returns the number of bytes written. -/// Returns `Ok(len)` on success, `Err(errno)` on failure. -pub fn getpeername(fd: u32, buf: &mut [u8]) -> Result { - let mut len = checked_u32_len(buf.len())?; - let errno = unsafe { net_getpeername(fd, buf.as_mut_ptr(), &mut len) }; - if errno == ERRNO_SUCCESS { - validate_returned_len(len, buf.len()) - } else { - Err(errno) - } -} - -/// Upgrade a connected TCP socket to TLS. -/// -/// `hostname` is used for SNI (Server Name Indication). -/// After success, `send`/`recv` on this fd use the encrypted TLS stream. -/// Returns `Ok(())` on success, `Err(errno)` on failure. -pub fn tls_connect(fd: u32, hostname: &[u8]) -> Result<(), Errno> { - let hostname_len = checked_u32_len(hostname.len())?; - let errno = unsafe { net_tls_connect(fd, hostname.as_ptr(), hostname_len) }; - if errno == ERRNO_SUCCESS { - Ok(()) - } else { - Err(errno) - } -} - -/// Poll socket FDs for readiness. -/// -/// `fds` is a mutable slice of pollfd-like entries (8 bytes each: fd i32, events i16, revents i16). -/// `timeout_ms` is the timeout: 0=non-blocking, -1=block forever, >0=milliseconds. -/// Returns `Ok(ready_count)` on success, `Err(errno)` on failure. -pub fn poll(fds: &mut [u8], nfds: u32, timeout_ms: i32) -> Result { - validate_poll_buffer_len(fds.len(), nfds)?; - let mut ready: u32 = 0; - let errno = unsafe { net_poll(fds.as_mut_ptr(), nfds, timeout_ms, &mut ready) }; - if errno == ERRNO_SUCCESS { - validate_poll_ready_count(ready, nfds) - } else { - Err(errno) - } -} - -/// Bind a socket to a local address. -/// -/// `addr` is a serialized address string (e.g. "host:port" or "/path/to/socket"). -/// Returns `Ok(())` on success, `Err(errno)` on failure. -pub fn bind(fd: u32, addr: &[u8]) -> Result<(), Errno> { - let addr_len = checked_u32_len(addr.len())?; - let errno = unsafe { net_bind(fd, addr.as_ptr(), addr_len) }; - if errno == ERRNO_SUCCESS { - Ok(()) - } else { - Err(errno) - } -} - -/// Mark a bound socket as listening for incoming connections. -/// -/// `backlog` is the maximum pending connection queue length. -/// Returns `Ok(())` on success, `Err(errno)` on failure. -pub fn listen(fd: u32, backlog: u32) -> Result<(), Errno> { - let errno = unsafe { net_listen(fd, backlog) }; - if errno == ERRNO_SUCCESS { - Ok(()) - } else { - Err(errno) - } -} - -/// Accept an incoming connection on a listening socket. -/// -/// Returns `Ok((fd, addr_len))` on success, where the remote address string -/// has been written into `addr_buf` with length `addr_len`. -/// Returns `Err(errno)` on failure. -pub fn accept(fd: u32, addr_buf: &mut [u8]) -> Result<(u32, u32), Errno> { - let mut new_fd: u32 = 0; - let mut addr_len = checked_u32_len(addr_buf.len())?; - let errno = unsafe { net_accept(fd, &mut new_fd, addr_buf.as_mut_ptr(), &mut addr_len) }; - if errno == ERRNO_SUCCESS { - Ok((new_fd, validate_returned_len(addr_len, addr_buf.len())?)) - } else { - Err(errno) - } -} - -/// Send a datagram to a specific destination address (UDP). -/// -/// `addr` is the destination address string (e.g. "host:port"). -/// Returns `Ok(bytes_sent)` on success, `Err(errno)` on failure. -pub fn sendto(fd: u32, buf: &[u8], flags: u32, addr: &[u8]) -> Result { - let buf_len = checked_u32_len(buf.len())?; - let addr_len = checked_u32_len(addr.len())?; - let mut sent: u32 = 0; - let errno = unsafe { - net_sendto( - fd, - buf.as_ptr(), - buf_len, - flags, - addr.as_ptr(), - addr_len, - &mut sent, - ) - }; - if errno == ERRNO_SUCCESS { - validate_returned_len(sent, buf.len()) - } else { - Err(errno) - } -} - -/// Receive a datagram from a UDP socket with source address. -/// -/// Writes received data into `buf` and the source address string into `addr_buf`. -/// Returns `Ok((bytes_received, addr_len))` on success, `Err(errno)` on failure. -pub fn recvfrom( - fd: u32, - buf: &mut [u8], - flags: u32, - addr_buf: &mut [u8], -) -> Result<(u32, u32), Errno> { - let buf_len = checked_u32_len(buf.len())?; - let mut received: u32 = 0; - let mut addr_len = checked_u32_len(addr_buf.len())?; - let errno = unsafe { - net_recvfrom( - fd, - buf.as_mut_ptr(), - buf_len, - flags, - &mut received, - addr_buf.as_mut_ptr(), - &mut addr_len, - ) - }; - if errno == ERRNO_SUCCESS { - Ok(( - validate_returned_len(received, buf.len())?, - validate_returned_len(addr_len, addr_buf.len())?, - )) - } else { - Err(errno) - } -} - -// ============================================================ -// Safe Rust wrappers — host_user -// ============================================================ - -/// Get the real user ID. -/// -/// Returns `Ok(uid)` on success, `Err(errno)` on failure. -pub fn get_uid() -> Result { - let mut uid: u32 = 0; - let errno = unsafe { getuid(&mut uid) }; - if errno == ERRNO_SUCCESS { - Ok(uid) - } else { - Err(errno) - } -} - -/// Get the real group ID. -/// -/// Returns `Ok(gid)` on success, `Err(errno)` on failure. -pub fn get_gid() -> Result { - let mut gid: u32 = 0; - let errno = unsafe { getgid(&mut gid) }; - if errno == ERRNO_SUCCESS { - Ok(gid) - } else { - Err(errno) - } -} - -/// Get the effective user ID. -/// -/// Returns `Ok(uid)` on success, `Err(errno)` on failure. -pub fn get_euid() -> Result { - let mut uid: u32 = 0; - let errno = unsafe { geteuid(&mut uid) }; - if errno == ERRNO_SUCCESS { - Ok(uid) - } else { - Err(errno) - } -} - -/// Get the effective group ID. -/// -/// Returns `Ok(gid)` on success, `Err(errno)` on failure. -pub fn get_egid() -> Result { - let mut gid: u32 = 0; - let errno = unsafe { getegid(&mut gid) }; - if errno == ERRNO_SUCCESS { - Ok(gid) - } else { - Err(errno) - } -} - -/// Check if a file descriptor is a terminal. -/// -/// Returns `Ok(true)` if it's a terminal, `Ok(false)` otherwise, `Err(errno)` on failure. -pub fn is_atty(fd: u32) -> Result { - let mut result: u32 = 0; - let errno = unsafe { isatty(fd, &mut result) }; - if errno == ERRNO_SUCCESS { - Ok(result != 0) - } else { - Err(errno) - } -} - -/// Get the passwd entry for a user ID. -/// -/// Writes the serialized passwd entry into `buf` and returns the number of bytes written. -/// Returns `Ok(len)` on success, `Err(errno)` on failure. -pub fn get_pwuid(uid: u32, buf: &mut [u8]) -> Result { - let mut len: u32 = 0; - let buf_len = checked_u32_len(buf.len())?; - let errno = unsafe { getpwuid(uid, buf.as_mut_ptr(), buf_len, &mut len) }; - if errno == ERRNO_SUCCESS { - validate_returned_len(len, buf.len()) - } else { - Err(errno) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn poll_buffer_validation_requires_exact_pollfd_capacity() { - assert_eq!(validate_poll_buffer_len(POLLFD_BYTES, 1), Ok(())); - assert_eq!( - validate_poll_buffer_len(POLLFD_BYTES - 1, 1), - Err(ERRNO_INVAL) - ); - assert_eq!( - validate_poll_buffer_len(POLLFD_BYTES + 1, 1), - Err(ERRNO_INVAL) - ); - } - - #[test] - fn returned_lengths_must_fit_in_the_supplied_buffer() { - assert_eq!(validate_returned_len(4, 4), Ok(4)); - assert_eq!(validate_returned_len(5, 4), Err(ERRNO_INVAL)); - } - - #[test] - fn poll_ready_count_must_not_exceed_nfds() { - assert_eq!(validate_poll_ready_count(0, 0), Ok(0)); - assert_eq!(validate_poll_ready_count(2, 2), Ok(2)); - assert_eq!(validate_poll_ready_count(3, 2), Err(ERRNO_INVAL)); - } -} diff --git a/registry/native/patches/codex-source/README.md b/registry/native/patches/codex-source/README.md deleted file mode 100644 index 8b1aee01d6..0000000000 --- a/registry/native/patches/codex-source/README.md +++ /dev/null @@ -1,271 +0,0 @@ -# Required codex SOURCE patches for wasm32-wasip1 - -These are minimal, upstreamable additions to codex's own source where it has a -hard `compile_error!`/missing-platform arm that NO toolchain patch can bypass -(first-party compile-time gates). Each adds REAL wasi support (wasi has the API), -not a hack. Applied to the vendored codex source during the secure-exec build. - -1. utils/git/src/platform.rs — add `#[cfg(target_os = "wasi")] create_symlink` - using `std::os::wasi::fs::symlink_path`, and widen the fallback - `compile_error!` cfg to `not(any(unix, windows, target_os = "wasi"))`. - wasi supports symlinks via the VM's `path_symlink` host call. - -2. rmcp-client/src/program_resolver.rs:26 — widen `#[cfg(unix)]` on `resolve()` to - `any(unix, target_os = "wasi")` (the unix impl returns the program unchanged, - correct for wasi). -3. rmcp-client/src/utils.rs:91 — widen `#[cfg(unix)]` on `DEFAULT_ENV_VARS` to - `any(unix, target_os = "wasi")`. - -4. core/Cargo.toml — add `[target.'cfg(target_os = "wasi")'.dependencies] codex-utils-pty` - (it compiles via the portable-pty wasi backend). codex-network-proxy stays excluded - (real crate pulls the rama proxy-server stack + tokio "full"); satisfied by stub. - -## codex-core OWN-code remaining (69 errors, characterized — this session got dep graph to 0) -The codex authors gated codex-network-proxy + codex-utils-pty OFF wasi as DEPS but did -NOT gate the SOURCE usage — an incomplete author wasi port. Breakdown: -- ~50: codex-core uses codex-network-proxy's FULL API (NetworkProxyConfig, NetworkDecision, - NetworkPolicyDecider/BlockedRequestObserver traits, NetworkProxyHandle, ConfigReloader, - build_config_state, host_and_port_from_network_addr, normalize_host, - validate_policy_against_constraints, NetworkProxyConstraints/Error, NetworkProtocol, - NetworkProxyState, NetworkProxyAuditMetadata, NetworkPolicyRequest, …). The real crate - can't compile on wasi (rama proxy stack); EXPAND the secure-exec codex-network-proxy - stub to this full API (no-op/inert — the VM kernel brokers network policy host-side). -- ~6: codex_otel SessionTelemetry methods the stub lacks → expand codex-otel stub. -- tokio::signal (no signals on wasi) → gate/stub codex-core's signal usage. -- ~4 platform fns (ensure_owner_only_permissions, synthetic_exit_status, - system_config_toml_file, system_requirements_toml_file) → add wasi arms. -- ~5 type mismatches (E0308) → case-by-case. -Then codex-exec compiles → un-stub --session-turn (real codex-core + EE protocol) → matrix. - -## network-proxy / otel: MUST be in-place wasi-gating of the REAL codex crates -Attempted an external stub crate (stubs/codex-network-proxy-wasi) — does NOT work: -codex workspace crates (codex-utils-absolute-path, etc.) use `workspace = true`, which -can't resolve from a path crate OUTSIDE codex's workspace. So the network-proxy/otel -stubbing must be done IN-PLACE in codex-rs/{network-proxy,otel} (workspace members): -- network-proxy/Cargo.toml: move `rama-*` + `tokio "full"` + rustls-provider to - `[target.'cfg(not(target_os="wasi"))'.dependencies]`; reduce wasi tokio features. -- network-proxy/src: gate the 6 rama files (certs,http_proxy,mitm,responses,socks5, - upstream) to `#[cfg(not(wasi))]` and add `#[cfg(wasi)]` stub versions. The 6 stub - bodies are already written in stubs/codex-network-proxy-wasi/src/{http_proxy,socks5, - mitm,certs,responses,upstream}.rs (run_http_proxy/run_socks5 → Ok(()); MitmState → unit). - Only 5 symbols are referenced by pure code: http_proxy::run_http_proxy{,_with_std_listener}, - socks5::run_socks5{,_with_std_listener}, mitm::MitmState. lib.rs mod decls stay (pure - modules config/network_policy/policy/proxy/reasons/runtime/state compile as-is). -- core/Cargo.toml: add codex-network-proxy to `[target.'cfg(target_os="wasi")'.dependencies]`. -- otel: same pattern — gate opentelemetry-otlp (grpc/tonic/reqwest exporter) off wasi, - stub the exporter init; keep the SessionTelemetry API (codex-core uses ~6 methods). -This is completing the codex authors' OWN incomplete wasi port (they gated the DEPS off -wasi but not the source). It is codex SOURCE modification, unavoidable (workspace constraint). - -## DONE this iteration: network-proxy in-place wasi-gating WORKS (codex-core 69→49) -Applied to REAL codex-rs/network-proxy (workspace member, so workspace=true resolves): -- network-proxy/Cargo.toml: rama-* + rustls-provider + tokio "full" → moved to - [target.'cfg(not(target_os="wasi"))'.dependencies]; base tokio reduced to - ["macros","rt","sync","time","io-util"]. -- network-proxy/src/lib.rs: the 6 rama modules (certs/http_proxy/mitm/responses/ - socks5/upstream) cfg-selected: #[cfg(not(wasi))] real vs #[cfg(wasi)] #[path="*_wasi.rs"]. -- network-proxy/src/{certs,http_proxy,mitm,responses,socks5,upstream}_wasi.rs: stub bodies - (run_http_proxy/run_socks5 → Ok(()); MitmState → unit). VERIFIED: codex_network_proxy - refs now resolve (0 unresolved). Restored workspace redirect to path="network-proxy" - + re-added as member + codex-core wasi dep. - -## NEXT: otel in-place gating (same pattern; ~6 SessionTelemetry method errors remain) -otel's opentelemetry-otlp exporter is spread across provider.rs(63)/client.rs(40)/ -otlp.rs(11)/config.rs(24). Gate those off wasi (cfg-select stub modules), keep -events/session_telemetry.rs (the SessionTelemetry API codex-core calls: -record_api_request, record_websocket_request, record_auth_recovery, log_sse_event, -tool_decision, tool_result_with_tags, on_request — these use codex-protocol types, so -MUST stay in-crate, not an external stub). Then: tokio::signal gate, ~4 platform fns -(synthetic_exit_status, system_*_toml_file, ensure_owner_only_permissions), ~5 E0308. -Then codex-exec → session-turn engine → matrix. - -## ✅ MILESTONE: codex-core AND codex-exec COMPILE for wasm32-wasip1 -All in-crate wasi-gating done (committed to the rivet codex fork, branch -wasi-port-codex-core, commit 0c3f4eb73, 25 files): network-proxy + otel in-place -gating, core platform fns (tokio::signal/synthetic_exit_status/toml/permissions), -git symlink, rmcp-client resolver. Plus std ExitStatusExt patch (patches/ -0008-wasi-exit-status-ext.md). `cargo build -p codex-core -p codex-exec ---target wasm32-wasip1 -Z build-std` → Finished, EXIT 0. - -REMAINING to e2e: (1) the secure-exec command crate crates/commands/codex-exec -un-stub --session-turn to delegate to the real codex-core/codex-exec engine and emit -the EE newline-JSON protocol (start/text_delta/tool_call_update/permission_request/ -done) — the real functional integration; (2) build via `make -C registry/native wasm` -(vendoring the codex fork + these patches); (3) wire the EE codex descriptor into the -agent-os matrix + un-skip the codex session test. - -## FINAL PIECE: session-turn engine (replace codex-exec wasi_stub_main) -codex-exec/src/lib.rs `wasi_stub_main()` is a placeholder ("WASI runtime support is -under development"). codex-core compiles now, so replace it with the real engine -(mirror lib_native.rs::run_exec_session, ~200-300 lines, in codex's workspace so it has -codex-core access). The built codex-exec.wasm IS the secure-exec `codex-exec` command. - -PROTOCOL (newline-JSON, the EE adapter @rivet-ee/agent-os-codex-agent drives it): -- stdin: first line {type:"start", cwd, mode, model, thought_level, - developer_instructions, history, prompt}; then {type:"permission_response", id, decision}. -- stdout events to EMIT (one JSON/line): - - {type:"start"} on boot - - {type:"text_delta", delta} ← EventMsg::AgentMessageDelta - - {type:"tool_call_update", tool_call_id, status, content} - ← EventMsg::ExecCommandBegin / ExecCommandEnd - - {type:"permission_request", tool_call_id, command} - ← EventMsg::ExecApprovalRequest / - ApplyPatchApprovalRequest; then read - permission_response from stdin → submit Op approval - - {type:"done"} ← EventMsg::TurnComplete - - {type:"error", message} ← EventMsg::Error - -ENGINE (codex-core API, verified): -1. parse start JSON → build codex_core Config (cwd/model/instructions; sandbox+network - proxy already gated off on wasi; approval policy = OnRequest so approvals surface). -2. ConversationManager::new_conversation(config) → Codex (or new_conversation_with_auth). -3. tokio current_thread runtime (rt feature works on wasi). seed history, then - submit(Op::UserInput{ items:[InputItem::Text{prompt}] }). -4. loop next_event() → match EventMsg → emit per table above; on approval requests, - block reading stdin for permission_response → submit(Op::ExecApproval/PatchApproval). -5. TurnComplete → emit done, exit 0. -Then: `make -C registry/native wasm` (vendor the codex fork wasi-port-codex-core branch + -all patches) → copies codex/codex-exec into software/codex/wasm → wire EE descriptor into -the agent-os matrix (registry/agent/codex already exists) → un-skip the codex session test. - -### session-turn engine — Config is loadable (tractable) -core/src/config/mod.rs has `Config::load_with_cli_overrides(cli_overrides, harness)` -(811) — build Config from {model, cwd, approval_policy=OnRequest, ...} overrides rather -than manual field construction. Op/InputItem: protocol.rs Op::UserInput{ items: -Vec } (235). The Codex client (core/src/codex.rs submit/next_event) is created -via the conversation/thread entry (codex_thread.rs / conversation flow) — confirm the -exact constructor (ConversationManager or Codex::spawn) when implementing. Engine ≈ -200-300 lines in codex-exec wasi_stub_main; tokio current_thread runtime (works on wasi). - -## ✅ DONE: session-turn engine compiles (the final FUNCTIONAL piece) -codex-exec/src/session_turn_wasi.rs (fork commit b86a5dfc0) drives the REAL codex-core -agent and emits the EE protocol. `cargo build -p codex-exec --target wasm32-wasip1 --Z build-std` → EXIT 0. So codex-core + codex-exec + the session-turn engine ALL compile. - -## Remaining = build pipeline + ACP wiring + test run (integration, not functional) -1. Produce optimized wasm artifacts from the fork (branch wasi-port-codex-core): - `cargo build -p codex-exec -p codex --release --target wasm32-wasip1 -Z build-std` - (+ wasm-opt) → copy to registry/software/codex/wasm/{codex-exec,codex}. NOTE: the - secure-exec `make wasm` currently builds the STUB crates/commands/{codex,codex-exec}; - to ship the real engine, either build from the fork directly (above) or vendor the - fork into registry/native and point cmd-codex-exec at the real codex-exec. -2. ACP adapter: the codex ACP bridge (spawns `codex-exec --session-turn`, newline-JSON↔ACP) - lives in EE (@rivet-ee/agent-os-codex-agent, agent-os-ee/packages/codex/src/adapter.ts). - The open-source registry/agent/codex only re-exports the wasm software package. To test - codex e2e in the agent-os matrix, either bring that adapter into agent-os or run the - EE codex-session.test.ts (describe.skipIf gated on wasm presence — un-skips once the - wasm artifacts above exist). -3. e2e: with codex-exec.wasm present, the EE adapter spawns it in the VM (wasi-spawn/http - host bridges), the engine talks to the (mock) model via wasi-http, streams text_delta/ - tool_call_update, emits done. Run codex-session.test.ts to verify. - -## ✅ ARTIFACT BUILT + SPAWNS IN VM; last blocker = 1 host import -Built the real codex-exec from the fork (release, wasm-opt'd to 28MB) and placed it in -registry/software/codex/wasm/{codex-exec,codex}. Smoke test (vm.exec("... | codex-exec ---session-turn")) shows it SPAWNS and attempts WebAssembly instantiation in the VM — -44 imports, of which 43 (wasi_snapshot_preview1×30, host_net×6, host_process×5, -host_fs×2) are provided by the secure-exec runtime. The ONLY unprovided one was -`env.sqlite3_load_extension` (sqlx's sqlite). Fixed by defining a #[no_mangle] no-op -stub in the engine (fork commit). Rebuild + re-place → instantiation should succeed. -Then the smoke test ({"type":"start"} before the model call) and the full EE -codex-session.test.ts (with the responses mock) verify the e2e turn. - -## ✅✅ VERIFIED: codex-exec RUNS IN THE VM and emits the EE protocol -Smoke test (packages/core/tests/codex-smoke.test.ts) PASSES: - vm.exec("printf '' | codex-exec --session-turn") → stdout: {"type":"start"} -The real codex agent (codex-rs → wasip1, session-turn engine driving codex-core) BOOTS -in the secure-exec VM, instantiates all 44 imports (after adding host -path_filestat_set_times), runs the engine, and emits the EE protocol's start event. -(exit 1 only because the smoke test provides no model mock/auth, so the turn errors -AFTER start — proving the boot + protocol path works.) - -FULL turn (text_delta→done): run the EE codex-session.test.ts with startResponsesMock -(OpenAI Responses mock) + OPENAI_API_KEY/base_url pointed at it. That exercises a -complete agent turn. The boot+protocol path is now VERIFIED end-to-end in the VM. - -## ✅✅✅ codex runs the FULL session event loop in the VM -DBG markers (engine) confirm the real codex agent runs the entire lifecycle in the VM: -load config → AuthManager::shared → ThreadManager::new → start_thread → submit -Op::UserInput → LIVE next_event loop (received a Warning event, processing). 8 runtime -blockers fixed and committed (host imports, secure-exec host wasi shim -path_filestat_set_times, tokio fs-asyncify + spawn_blocking inline on wasi, std -split_paths, now_local→UTC). FINAL GAP: the model HTTP POST to the OpenAI Responses -mock does not reach it (mock requests=0, then hang). codex honors OPENAI_BASE_URL -(deprecated warning), so base_url=mock; the codex-exec wasi-http/host_net connect to the -HOST loopback mock blocks. Resolve: confirm host_net egress to the test's 127.0.0.1:PORT -is permitted for wasi commands (loopbackExemptPorts is for the kernel adapter; host_net -may need its own allow), and that the Responses path matches (/v1/responses). Then the -turn streams text_delta→done. The agent itself runs e2e in the VM today. - -## CORRECTED final diagnosis (via sidecar instrumentation): NOT a network issue -Added file-logging to the sidecar net.connect handler ENTRY + require_network_access + -resolve_tcp_connect_addr, ran the full-turn test. RESULT: the log stays EMPTY — codex's -connect NEVER reaches the sidecar. So codex is NOT making the model HTTP call; it HANGS -in its OWN agent loop after the first event (a config Warning), before any network call. -Root cause: codex's agent loop uses tokio concurrency that assumes OS threads -(spawn_blocking on a dedicated blocking pool, background tasks). wasm32-wasip1 is -single-threaded (no OS threads); the spawn_blocking-as-current-thread-task workaround -(0003) can DEADLOCK — a blocking task occupies the only thread while waiting on another -task that can't run. This is an architectural mismatch (codex's threading model vs -single-thread wasi), the genuinely hard core of running codex on wasi. Resolving it needs -codex-internal tracing to find the exact blocking construct and make it cooperative, or a -different runtime strategy. (Sidecar instrumentation reverted; secure-exec is clean.) -STATUS: codex compiles + runs the full session lifecycle in the VM (boot → EE protocol → -config → auth → ThreadManager → start_thread → submit → live event loop) but hangs in the -agent loop before the model call on the single-threaded runtime. - -## ✅✅✅ RESOLVED: codex completes a full model turn end-to-end on wasm32-wasip1 - -The "agent loop hangs" diagnosis above was *close but mislabeled* — it is NOT a tokio -concurrency/threading-model mismatch. The current-thread runtime drives codex's spawned -submission_loop correctly. The real root cause, found by instrumenting tokio's -current-thread scheduler (Handle::spawn / schedule / the block_on for-loop / per-task -spawn-location), was a **single blocking task that pins the only executor thread**: - -- `codex-core` spawns a shell-snapshot task (`core/src/shell_snapshot.rs`) during session - init that launches a **shell subprocess** to capture the environment and blocks on its - exit. On wasm32-wasip1 the VM's child-process bridge cannot deliver that wait on the - single-threaded runtime (the "could not retrieve pid for child process" warnings), so the - task blocks the executor thread **forever**. The scheduler then never drains the rest of - the local run queue — including the submission_loop — so the agent turn never advances and - no model HTTP call is ever made. (Hence the sidecar net.connect handler stayed empty.) - -Fixes (all in the codex fork, branch `wasi-port-codex-core`, all `#[cfg(target_os="wasi")]`): -1. `core/src/shell_snapshot.rs` — skip the snapshot on wasi (send `None`, return). A shell - environment snapshot is meaningless in the VM and the subprocess wait deadlocks the runtime. -2. `core/src/state_db.rs` — `init` / `get_state_db` / `open_if_present` return `None` on wasi. - sqlx-sqlite's blocking worker can't run on the thread-less runtime (open fails ENOTSUP, - os error 58); skipping it also avoids perturbing session init. -3. `exec/src/session_turn_wasi.rs` — runtime is `new_current_thread().enable_time()` (no I/O - reactor needed; network I/O is host-brokered/synchronous) and a plain - `block_on(session_turn())`. The earlier yield-pump / `global_queue_interval(1)` - experiments were chasing the wrong theory and are NOT needed once the blocking task is gone. - -Test (`packages/core/tests/codex-fullturn.test.ts`): codex-exec --session-turn runs the real -codex-core agent in the VM, calls the mock OpenAI Responses API over wasi-http, streams the -SSE response, and emits `{"type":"start"}` … `{"type":"done"}` on a clean protocol channel -(stdout carries ONLY the EE protocol JSON). The mock (`tests/helpers/openai-responses-mock.ts`) -emits a faithful Responses SSE stream (`response.created` → `response.output_text.delta` → -`response.output_item.done` → `response.completed`); `OPENAI_BASE_URL` includes `/v1` to match -codex's URL convention (`{base}/responses`). - -Build (reproducible): the wasi build uses `-Z build-std` with a patched rust-src sysroot. -Because `-Z build-std` injects `panic_unwind` via `--extern` while the wasm target needs -`panic_abort` resolved from the sysroot, the build: - - stashes the prebuilt `wasm32-wasip1` `*.rlib`/`*.rmeta` out of the toolchain lib dir - (keeping `self-contained/` crt) so build-std's own `libcore` is the only one (avoids the - `E0152 duplicate lang item core` at the bin link), and - - copies build-std's freshly-built `libpanic_abort-*.rlib` into that sysroot lib dir (so the - bin's `-Cpanic=abort` resolves a `panic_abort` that is ABI-matched to build-std's core), - - builds with `--config 'profile.release.panic="abort"'`. -See `/tmp/relink-codex.sh` for the exact invocation. (This sysroot massaging should be folded -into the secure-exec wasm toolchain so it is not a manual step.) - -KNOWN REMAINING (non-blocking; turn works e2e): -- Rollout recorder logs `failed to queue rollout items: channel closed` (the writer task exits - early on wasi — likely the git-info subprocess in `write_session_meta`, same blocking-child - class as shell_snapshot). Non-fatal: the turn completes; agent-os resume uses adapter-passed - history, not codex's on-disk rollout. Gating git-info on wasi would silence it. -- The session-turn engine reads `prompt` but not `history`; multi-turn resume via the EE ACP - adapter needs history replay wired in. -- ACP-level integration into the unified agent-matrix lives in the EE codex adapter. diff --git a/registry/native/patches/git/README.md b/registry/native/patches/git/README.md deleted file mode 100644 index b8a55dffab..0000000000 --- a/registry/native/patches/git/README.md +++ /dev/null @@ -1,28 +0,0 @@ -# Git WASM Compatibility - -Git (GPL-2.0) cannot be vendored due to license restrictions. -This project uses a clean-room, Apache-2.0 licensed reimplementation -of git plumbing commands. - -## WASM-Incompatible Patterns in Git Source - -The following patterns in upstream git are incompatible with WASM/WASI -and were avoided in our clean-room implementation: - -### 1. fork+exec in run_command.c -Git's `run_command()` uses `fork()`+`exec()` extensively for spawning -child processes (hooks, filters, remotes). WASI has no `fork()`. -**Our approach:** Use `posix_spawn()` via `host_process` imports when -process spawning is needed in future commands. - -### 2. mmap in wrapper.c -Git uses `mmap()` for memory-mapped file I/O (packfiles, index). -WASI has limited `mmap()` support. -**Our approach:** Use standard `malloc()`+`read()` for all file I/O. -Git upstream supports this via `NO_MMAP=1`. - -### 3. Signal handlers (sigaction for SIGPIPE, SIGCHLD) -Git registers signal handlers for `SIGPIPE` (ignore broken pipe) and -`SIGCHLD` (reap child processes). WASI has no signal support. -**Our approach:** No signal handlers needed — WASI processes don't -receive signals, and our process model handles cleanup via the kernel. diff --git a/registry/native/patches/wasi-libc-overrides/fcntl.c b/registry/native/patches/wasi-libc-overrides/fcntl.c deleted file mode 100644 index ad938ca653..0000000000 --- a/registry/native/patches/wasi-libc-overrides/fcntl.c +++ /dev/null @@ -1,175 +0,0 @@ -/** - * Fix for wasi-libc's broken fcntl implementation. - * - * wasi-libc always returns FD_CLOEXEC(1) for F_GETFD and ignores F_SETFD - * because WASI has no exec(). It also returns EINVAL for F_DUPFD and - * F_DUPFD_CLOEXEC. This fix properly tracks per-fd cloexec flags, - * delegates F_GETFL/F_SETFL to the original WASI fd_fdstat interface, - * and routes F_DUPFD/F_DUPFD_CLOEXEC through the host_process bridge. - * - * Installed into the patched sysroot so ALL WASM programs get correct - * fcntl behavior, not just test binaries. - */ - -#include -#include -#include -#include - -/* WASI headers omit F_DUPFD and F_DUPFD_CLOEXEC — define with Linux values */ -#ifndef F_DUPFD -#define F_DUPFD 0 -#endif -#ifndef F_DUPFD_CLOEXEC -#define F_DUPFD_CLOEXEC 1030 -#endif - -/* Host import for dup with minimum fd (F_DUPFD semantics) */ -__attribute__((import_module("host_process"), import_name("fd_dup_min"))) -int __host_fd_dup_min(int fd, int min_fd, int *ret_new_fd); - -/* Per-fd cloexec tracking (up to 256 FDs) */ -#define MAX_FDS 256 -static unsigned char _fd_cloexec[MAX_FDS]; - -int fcntl(int fd, int cmd, ...) { - va_list ap; - va_start(ap, cmd); - - int result; - - switch (cmd) { - case F_DUPFD: { - int min_fd = va_arg(ap, int); - if (fd < 0 || fd >= MAX_FDS) { - errno = EBADF; - result = -1; - } else if (min_fd < 0) { - errno = EINVAL; - result = -1; - } else { - int new_fd; - int err = __host_fd_dup_min(fd, min_fd, &new_fd); - if (err != 0) { - errno = err; - result = -1; - } else { - if (new_fd >= 0 && new_fd < MAX_FDS) - _fd_cloexec[new_fd] = 0; - result = new_fd; - } - } - break; - } - - case F_DUPFD_CLOEXEC: { - int min_fd = va_arg(ap, int); - if (fd < 0 || fd >= MAX_FDS) { - errno = EBADF; - result = -1; - } else if (min_fd < 0) { - errno = EINVAL; - result = -1; - } else { - int new_fd; - int err = __host_fd_dup_min(fd, min_fd, &new_fd); - if (err != 0) { - errno = err; - result = -1; - } else { - if (new_fd >= 0 && new_fd < MAX_FDS) - _fd_cloexec[new_fd] = 1; - result = new_fd; - } - } - break; - } - - case F_GETFD: - if (fd < 0 || fd >= MAX_FDS) { - errno = EBADF; - result = -1; - } else { - result = _fd_cloexec[fd] ? FD_CLOEXEC : 0; - } - break; - - case F_SETFD: { - int arg = va_arg(ap, int); - if (fd < 0 || fd >= MAX_FDS) { - errno = EBADF; - result = -1; - } else { - _fd_cloexec[fd] = (arg & FD_CLOEXEC) ? 1 : 0; - result = 0; - } - break; - } - - case F_GETFL: { - __wasi_fdstat_t stat; - __wasi_errno_t err = __wasi_fd_fdstat_get((__wasi_fd_t)fd, &stat); - if (err != 0) { - errno = err; - result = -1; - } else { - int flags = stat.fs_flags; - /* Derive read/write mode from rights */ - __wasi_rights_t r = stat.fs_rights_base; - int can_read = (r & __WASI_RIGHTS_FD_READ) != 0; - int can_write = (r & __WASI_RIGHTS_FD_WRITE) != 0; - if (can_read && can_write) - flags |= O_RDWR; - else if (can_read) - flags |= O_RDONLY; - else if (can_write) - flags |= O_WRONLY; - result = flags; - } - break; - } - - case F_SETFL: { - int arg = va_arg(ap, int); - __wasi_errno_t err = __wasi_fd_fdstat_set_flags( - (__wasi_fd_t)fd, - (__wasi_fdflags_t)(arg & 0xfff)); - if (err != 0) { - errno = err; - result = -1; - } else { - result = 0; - } - break; - } - - case F_GETLK: { - struct flock *lock = va_arg(ap, struct flock *); - if (!lock) { - errno = EINVAL; - result = -1; - } else { - lock->l_type = F_UNLCK; - lock->l_pid = 0; - result = 0; - } - break; - } - - case F_SETLK: - case F_SETLKW: - // WASI has no kernel-level advisory locking. Treat locks as a - // successful no-op so single-process workloads like DuckDB can open - // writable database files on the VFS-backed filesystem. - result = 0; - break; - - default: - errno = EINVAL; - result = -1; - break; - } - - va_end(ap); - return result; -} diff --git a/registry/native/patches/wasi-libc-overrides/fcntl.o b/registry/native/patches/wasi-libc-overrides/fcntl.o deleted file mode 100644 index 81c3b82dd6..0000000000 Binary files a/registry/native/patches/wasi-libc-overrides/fcntl.o and /dev/null differ diff --git a/registry/native/patches/wasi-libc-overrides/fmtmsg.o b/registry/native/patches/wasi-libc-overrides/fmtmsg.o deleted file mode 100644 index 4caa9d328d..0000000000 Binary files a/registry/native/patches/wasi-libc-overrides/fmtmsg.o and /dev/null differ diff --git a/registry/native/patches/wasi-libc-overrides/inet_ntop.o b/registry/native/patches/wasi-libc-overrides/inet_ntop.o deleted file mode 100644 index 7f38fb4bbf..0000000000 Binary files a/registry/native/patches/wasi-libc-overrides/inet_ntop.o and /dev/null differ diff --git a/registry/native/patches/wasi-libc-overrides/open_wmemstream.o b/registry/native/patches/wasi-libc-overrides/open_wmemstream.o deleted file mode 100644 index bfe607600c..0000000000 Binary files a/registry/native/patches/wasi-libc-overrides/open_wmemstream.o and /dev/null differ diff --git a/registry/native/patches/wasi-libc-overrides/pthread_attr.o b/registry/native/patches/wasi-libc-overrides/pthread_attr.o deleted file mode 100644 index 7c6ae1e113..0000000000 Binary files a/registry/native/patches/wasi-libc-overrides/pthread_attr.o and /dev/null differ diff --git a/registry/native/patches/wasi-libc-overrides/pthread_key.o b/registry/native/patches/wasi-libc-overrides/pthread_key.o deleted file mode 100644 index f2186cb689..0000000000 Binary files a/registry/native/patches/wasi-libc-overrides/pthread_key.o and /dev/null differ diff --git a/registry/native/patches/wasi-libc-overrides/pthread_mutex.o b/registry/native/patches/wasi-libc-overrides/pthread_mutex.o deleted file mode 100644 index cca3b4a12a..0000000000 Binary files a/registry/native/patches/wasi-libc-overrides/pthread_mutex.o and /dev/null differ diff --git a/registry/native/patches/wasi-libc-overrides/strfmon.o b/registry/native/patches/wasi-libc-overrides/strfmon.o deleted file mode 100644 index 52ba0d4df0..0000000000 Binary files a/registry/native/patches/wasi-libc-overrides/strfmon.o and /dev/null differ diff --git a/registry/native/patches/wasi-libc-overrides/swprintf.o b/registry/native/patches/wasi-libc-overrides/swprintf.o deleted file mode 100644 index e73848c70a..0000000000 Binary files a/registry/native/patches/wasi-libc-overrides/swprintf.o and /dev/null differ diff --git a/registry/native/patches/wasi-libc/.gitkeep b/registry/native/patches/wasi-libc/.gitkeep deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/registry/native/patches/wasi-libc/0012-posix-spawn-cwd.patch b/registry/native/patches/wasi-libc/0012-posix-spawn-cwd.patch deleted file mode 100644 index bce70048b7..0000000000 --- a/registry/native/patches/wasi-libc/0012-posix-spawn-cwd.patch +++ /dev/null @@ -1,57 +0,0 @@ ---- a/libc-bottom-half/sources/host_spawn_wait.c -+++ b/libc-bottom-half/sources/host_spawn_wait.c -@@ -108,6 +108,17 @@ - return 0; - } - -+static const char *find_pwd_in_env(char *const envp[]) { -+ if (!envp) return NULL; -+ for (int i = 0; envp[i]; i++) { -+ const char *entry = envp[i]; -+ if (strncmp(entry, "PWD=", 4) == 0 && entry[4] == '/') { -+ return entry + 4; -+ } -+ } -+ return NULL; -+} -+ - int posix_spawn_file_actions_adddup2(posix_spawn_file_actions_t *fa, int srcfd, int fd) { - if (srcfd < 0 || fd < 0) return EBADF; - struct __fdop *op = malloc(sizeof(*op)); -@@ -232,6 +243,7 @@ - } - - // Process file_actions in order: extract stdio overrides and handle close/open -+ const char *spawn_cwd = NULL; - uint32_t stdin_fd = 0, stdout_fd = 1, stderr_fd = 2; - if (fa && fa->__actions) { - for (struct __fdop *op = fa->__actions; op; op = op->next) { -@@ -259,16 +271,27 @@ - else close(opened); - break; - } -+ case FDOP_CHDIR: -+ if (op->path[0] == '/') spawn_cwd = op->path; -+ break; - } - } - } - -+ // Resolve cwd: explicit chdir action > inherited PWD > getcwd() > empty -+ char cwd_buf[1024]; -+ const char *cwd_str = spawn_cwd; -+ if (!cwd_str) cwd_str = find_pwd_in_env(env); -+ if (!cwd_str && getcwd(cwd_buf, sizeof(cwd_buf))) { -+ cwd_str = cwd_buf; -+ } -+ - uint32_t child_pid; - uint32_t err = __host_proc_spawn( - argv_buf, (uint32_t)argv_buf_len, - envp_buf ? envp_buf : (const uint8_t *)"", (uint32_t)envp_buf_len, - stdin_fd, stdout_fd, stderr_fd, -- (const uint8_t *)"", 0, -+ cwd_str ? (const uint8_t *)cwd_str : (const uint8_t *)"", cwd_str ? (uint32_t)strlen(cwd_str) : 0, - &child_pid); - - free(argv_buf); diff --git a/registry/package.json b/registry/package.json deleted file mode 100644 index 5a19029fcb..0000000000 --- a/registry/package.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "name": "agentos-registry", - "private": true, - "type": "module", - "license": "Apache-2.0", - "version": "0.0.0", - "packageManager": "pnpm@10.13.1", - "engines": { - "node": ">=20" - }, - "scripts": { - "check-types": "node -e \"process.exit(0)\"", - "test": "node ./scripts/run-vitest.mjs" - }, - "devDependencies": { - "@agentos-software/common": "workspace:*", - "@rivet-dev/agentos-runtime-core": "workspace:*", - "@agentos-software/curl": "workspace:*", - "@types/node": "^22.10.2", - "@xterm/headless": "^6.0.0", - "typescript": "^5.9.2", - "vitest": "^2.1.9" - } -} diff --git a/registry/scripts/run-vitest.mjs b/registry/scripts/run-vitest.mjs deleted file mode 100644 index c6e2b752f9..0000000000 --- a/registry/scripts/run-vitest.mjs +++ /dev/null @@ -1,32 +0,0 @@ -import { spawnSync } from "node:child_process"; -import { readdirSync } from "node:fs"; -import { resolve } from "node:path"; -import { fileURLToPath } from "node:url"; - -const scriptDir = fileURLToPath(new URL(".", import.meta.url)); -const pnpmStoreDir = resolve(scriptDir, "..", "..", "node_modules", ".pnpm"); -const vitestPackageDir = readdirSync(pnpmStoreDir).find((entry) => - entry.startsWith("vitest@"), -); - -if (!vitestPackageDir) { - throw new Error(`Could not find vitest in ${pnpmStoreDir}`); -} - -const vitestCli = resolve( - pnpmStoreDir, - vitestPackageDir, - "node_modules", - "vitest", - "vitest.mjs", -); - -const result = spawnSync(process.execPath, [vitestCli, "run", ...process.argv.slice(2)], { - stdio: "inherit", -}); - -if (result.error) { - throw result.error; -} - -process.exit(result.status ?? 1); diff --git a/registry/scripts/status.mjs b/registry/scripts/status.mjs deleted file mode 100644 index bd019a3860..0000000000 --- a/registry/scripts/status.mjs +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env node - -/** - * Registry package status: local version, staged bin/, assembled dist/package, - * and (with --remote) the published npm dist-tags per package. - * - * Run from anywhere: node registry/scripts/status.mjs [--remote] - */ - -import { execFileSync } from "node:child_process"; -import { existsSync, readFileSync, readdirSync } from "node:fs"; -import { dirname, join } from "node:path"; -import { fileURLToPath } from "node:url"; - -const REGISTRY_ROOT = join(dirname(fileURLToPath(import.meta.url)), ".."); -const remote = process.argv.includes("--remote"); - -function countEntries(dir) { - try { - return readdirSync(dir).length; - } catch { - return null; - } -} - -function distTags(name) { - try { - const out = execFileSync( - "npm", - ["view", name, "dist-tags", "--json"], - { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }, - ); - const tags = JSON.parse(out); - return Object.entries(tags) - .map(([tag, version]) => `${tag}=${version}`) - .join(" "); - } catch { - return "(not published)"; - } -} - -const rows = []; -for (const kind of ["software", "agent"]) { - const base = join(REGISTRY_ROOT, kind); - for (const dir of readdirSync(base, { withFileTypes: true })) { - if (!dir.isDirectory()) continue; - const pkgPath = join(base, dir.name, "package.json"); - if (!existsSync(pkgPath)) continue; - const pkg = JSON.parse(readFileSync(pkgPath, "utf8")); - const bin = countEntries(join(base, dir.name, "bin")); - const distPackage = join(base, dir.name, "dist", "package"); - const assembled = existsSync(join(distPackage, "package.json")) - ? (countEntries(join(distPackage, "bin")) ?? 0) - : null; - rows.push({ - kind, - name: pkg.name, - version: pkg.version, - bin: bin === null ? "-" : String(bin), - dist: assembled === null ? "NOT BUILT" : `${assembled} cmds`, - tags: remote ? distTags(pkg.name) : undefined, - }); - } -} - -const width = Math.max(...rows.map((r) => r.name.length)) + 2; -for (const row of rows) { - process.stdout.write( - `${row.name.padEnd(width)} ${row.version.padEnd(28)} bin:${row.bin.padEnd(5)} dist:${row.dist.padEnd(10)}${row.tags !== undefined ? ` ${row.tags}` : ""}\n`, - ); -} -const nativeCommands = join( - REGISTRY_ROOT, - "native/target/wasm32-wasip1/release/commands", -); -const built = countEntries(nativeCommands); -process.stdout.write( - `\nnative commands dir: ${built === null ? "NOT BUILT (run `just registry-native`)" : `${built} entries`}\n`, -); diff --git a/registry/software/browserbase/agentos-package.json b/registry/software/browserbase/agentos-package.json deleted file mode 100644 index 7e64d73e00..0000000000 --- a/registry/software/browserbase/agentos-package.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "name": "browserbase", - "registry": { - "title": "Browserbase", - "description": "The Browserbase `browse` CLI: let agents browse the web with a cloud browser, no sandbox required.", - "beta": true, - "types": [ - "browser" - ], - "image": "/images/registry/browserbase.svg" - } -} diff --git a/registry/software/browserbase/package.json b/registry/software/browserbase/package.json deleted file mode 100644 index 97cdfdd52d..0000000000 --- a/registry/software/browserbase/package.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "name": "@agentos-software/browserbase", - "version": "0.3.3", - "type": "module", - "license": "Apache-2.0", - "description": "Browserbase browse CLI for secure-exec VMs (cloud browser automation)", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", - "bin": { - "browse": "./node_modules/browse/bin/run.js" - }, - "exports": { - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js", - "default": "./dist/index.js" - } - }, - "files": [ - "dist", - "!dist/package", - "!dist/package.tar" - ], - "scripts": { - "build": "tsc && rm -rf dist/package dist/package.tar && agentos-toolchain pack . --out dist/package --prune-native", - "check-types": "tsc --noEmit" - }, - "dependencies": { - "browse": "^0.9.3" - }, - "devDependencies": { - "@agentos-software/manifest": "workspace:*", - "@rivet-dev/agentos-toolchain": "workspace:*", - "@types/node": "^22.10.2", - "typescript": "^5.7.2" - } -} diff --git a/registry/software/browserbase/tsconfig.json b/registry/software/browserbase/tsconfig.json deleted file mode 100644 index 8f24167afd..0000000000 --- a/registry/software/browserbase/tsconfig.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./src" - }, - "include": ["src/**/*"] -} diff --git a/registry/software/build-essential/agentos-package.json b/registry/software/build-essential/agentos-package.json deleted file mode 100644 index 63b0155256..0000000000 --- a/registry/software/build-essential/agentos-package.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "registry": { - "title": "Build Essential", - "description": "Meta-package: common + git + curl.", - "priority": 100 - } -} diff --git a/registry/software/build-essential/package.json b/registry/software/build-essential/package.json deleted file mode 100644 index 1ce4e95963..0000000000 --- a/registry/software/build-essential/package.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "name": "@agentos-software/build-essential", - "version": "0.3.0-rc.2", - "type": "module", - "license": "Apache-2.0", - "description": "Build-essential WASM command set for secure-exec VMs (common + git + curl)", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", - "files": [ - "dist", - "!dist/package", - "!dist/package.tar" - ], - "exports": { - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js" - } - }, - "scripts": { - "build": "tsc", - "check-types": "tsc --noEmit" - }, - "dependencies": { - "@agentos-software/common": "workspace:*", - "@agentos-software/curl": "workspace:*", - "@agentos-software/git": "workspace:*" - }, - "devDependencies": { - "@agentos-software/manifest": "workspace:*", - "@types/node": "^22.10.2", - "typescript": "^5.9.2" - } -} diff --git a/registry/software/build-essential/tsconfig.json b/registry/software/build-essential/tsconfig.json deleted file mode 100644 index 8f24167afd..0000000000 --- a/registry/software/build-essential/tsconfig.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./src" - }, - "include": ["src/**/*"] -} diff --git a/registry/software/codex-cli/agentos-package.json b/registry/software/codex-cli/agentos-package.json deleted file mode 100644 index 07b8d1f1d3..0000000000 --- a/registry/software/codex-cli/agentos-package.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "commands": [ - "codex", - "codex-exec" - ], - "registry": { - "title": "Codex CLI", - "description": "OpenAI Codex CLI integration.", - "image": "/images/registry/codex.svg" - } -} diff --git a/registry/software/codex-cli/package.json b/registry/software/codex-cli/package.json deleted file mode 100644 index 7e615f70b6..0000000000 --- a/registry/software/codex-cli/package.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "name": "@agentos-software/codex-cli", - "version": "0.3.3", - "type": "module", - "license": "Apache-2.0", - "description": "OpenAI Codex command package for secure-exec VMs", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", - "files": [ - "dist", - "!dist/package", - "!dist/package.tar" - ], - "exports": { - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js" - } - }, - "scripts": { - "build": "agentos-toolchain stage --commands-dir ../codex/wasm --if-missing skip && tsc && agentos-toolchain build", - "check-types": "tsc --noEmit" - }, - "devDependencies": { - "@agentos-software/manifest": "workspace:*", - "@rivet-dev/agentos-toolchain": "workspace:*", - "@types/node": "^22.10.2", - "typescript": "^5.9.2" - } -} diff --git a/registry/software/codex-cli/tsconfig.json b/registry/software/codex-cli/tsconfig.json deleted file mode 100644 index 8f24167afd..0000000000 --- a/registry/software/codex-cli/tsconfig.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./src" - }, - "include": ["src/**/*"] -} diff --git a/registry/software/common/agentos-package.json b/registry/software/common/agentos-package.json deleted file mode 100644 index 4e787157e2..0000000000 --- a/registry/software/common/agentos-package.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "registry": { - "title": "Common", - "description": "Meta-package: coreutils + sed + grep + gawk + findutils + diffutils + tar + gzip.", - "priority": 95 - } -} diff --git a/registry/software/common/package.json b/registry/software/common/package.json deleted file mode 100644 index 7a422419f4..0000000000 --- a/registry/software/common/package.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "name": "@agentos-software/common", - "version": "0.3.0-rc.2", - "type": "module", - "license": "Apache-2.0", - "description": "Common WASM command set for secure-exec VMs (coreutils + sed + grep + gawk + findutils + diffutils + tar + gzip)", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", - "files": [ - "dist", - "!dist/package", - "!dist/package.tar" - ], - "exports": { - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js" - } - }, - "scripts": { - "build": "tsc", - "check-types": "tsc --noEmit" - }, - "dependencies": { - "@agentos-software/coreutils": "workspace:*", - "@agentos-software/sed": "workspace:*", - "@agentos-software/grep": "workspace:*", - "@agentos-software/gawk": "workspace:*", - "@agentos-software/findutils": "workspace:*", - "@agentos-software/diffutils": "workspace:*", - "@agentos-software/tar": "workspace:*", - "@agentos-software/gzip": "workspace:*" - }, - "devDependencies": { - "@agentos-software/manifest": "workspace:*", - "@types/node": "^22.10.2", - "typescript": "^5.9.2" - } -} diff --git a/registry/software/common/tsconfig.json b/registry/software/common/tsconfig.json deleted file mode 100644 index 8f24167afd..0000000000 --- a/registry/software/common/tsconfig.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./src" - }, - "include": ["src/**/*"] -} diff --git a/registry/software/coreutils/agentos-package.json b/registry/software/coreutils/agentos-package.json deleted file mode 100644 index fd258428ba..0000000000 --- a/registry/software/coreutils/agentos-package.json +++ /dev/null @@ -1,127 +0,0 @@ -{ - "commands": [ - "sh", - "arch", - "b2sum", - "base32", - "base64", - "basename", - "basenc", - "cat", - "chmod", - "cksum", - "column", - "comm", - "cp", - "cut", - "date", - "dd", - "dircolors", - "dirname", - "du", - "echo", - "env", - "expand", - "expr", - "factor", - "false", - "fmt", - "fold", - "head", - "join", - "link", - "ln", - "logname", - "ls", - "md5sum", - "mkdir", - "mktemp", - "mv", - "nice", - "nl", - "nohup", - "nproc", - "numfmt", - "od", - "paste", - "pathchk", - "printenv", - "printf", - "ptx", - "pwd", - "readlink", - "realpath", - "rev", - "rm", - "rmdir", - "seq", - "sha1sum", - "sha224sum", - "sha256sum", - "sha384sum", - "sha512sum", - "shred", - "shuf", - "sleep", - "sort", - "split", - "stat", - "stdbuf", - "strings", - "sum", - "tac", - "tail", - "tee", - "test", - "timeout", - "touch", - "tr", - "true", - "truncate", - "tsort", - "uname", - "unexpand", - "uniq", - "unlink", - "wc", - "which", - "whoami", - "yes" - ], - "aliases": { - "bash": "sh", - "more": "cat", - "dir": "ls", - "vdir": "ls", - "[": "test" - }, - "stubs": [ - "chcon", - "runcon", - "chgrp", - "chown", - "chroot", - "df", - "groups", - "id", - "hostname", - "hostid", - "install", - "kill", - "mkfifo", - "mknod", - "pinky", - "who", - "users", - "uptime", - "stty", - "sync", - "tty" - ], - "registry": { - "title": "Coreutils", - "description": "sh, cat, ls, cp, mv, rm, sort, and 80+ essential POSIX commands.", - "priority": 45, - "image": "/images/registry/coreutils.svg" - } -} diff --git a/registry/software/coreutils/package.json b/registry/software/coreutils/package.json deleted file mode 100644 index 1a6941da37..0000000000 --- a/registry/software/coreutils/package.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "name": "@agentos-software/coreutils", - "version": "0.3.3", - "type": "module", - "license": "Apache-2.0", - "description": "GNU coreutils for secure-exec VMs (sh, cat, ls, cp, sort, and 80+ commands)", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", - "files": [ - "dist", - "!dist/package", - "!dist/package.tar" - ], - "exports": { - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js" - } - }, - "scripts": { - "build": "agentos-toolchain stage --commands-dir ../../native/target/wasm32-wasip1/release/commands --if-missing skip && tsc && agentos-toolchain build", - "build:runtime": "agentos-toolchain stage --commands-dir ../../native/target/wasm32-wasip1/release/commands && tsc && agentos-toolchain build", - "check-types": "tsc --noEmit", - "prepublishOnly": "pnpm run build:runtime" - }, - "devDependencies": { - "@agentos-software/manifest": "workspace:*", - "@rivet-dev/agentos-toolchain": "workspace:*", - "@types/node": "^22.10.2", - "typescript": "^5.9.2" - } -} diff --git a/registry/software/coreutils/tsconfig.json b/registry/software/coreutils/tsconfig.json deleted file mode 100644 index 8f24167afd..0000000000 --- a/registry/software/coreutils/tsconfig.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./src" - }, - "include": ["src/**/*"] -} diff --git a/registry/software/curl/agentos-package.json b/registry/software/curl/agentos-package.json deleted file mode 100644 index 4b2d55e758..0000000000 --- a/registry/software/curl/agentos-package.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "commands": [ - "curl" - ], - "registry": { - "title": "curl", - "description": "HTTP(S) client for fetching URLs and APIs.", - "priority": 50, - "image": "/images/registry/curl.svg" - } -} diff --git a/registry/software/curl/bin/curl b/registry/software/curl/bin/curl deleted file mode 100644 index 5e85f8a87d..0000000000 Binary files a/registry/software/curl/bin/curl and /dev/null differ diff --git a/registry/software/curl/package.json b/registry/software/curl/package.json deleted file mode 100644 index f915b112f3..0000000000 --- a/registry/software/curl/package.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "name": "@agentos-software/curl", - "version": "0.3.3", - "type": "module", - "license": "Apache-2.0", - "description": "curl HTTP client for secure-exec VMs", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", - "files": [ - "dist", - "!dist/package", - "!dist/package.tar" - ], - "exports": { - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js" - } - }, - "scripts": { - "build": "agentos-toolchain stage --commands-dir ../../native/target/wasm32-wasip1/release/commands --if-missing skip && tsc && agentos-toolchain build", - "check-types": "tsc --noEmit" - }, - "devDependencies": { - "@agentos-software/manifest": "workspace:*", - "@rivet-dev/agentos-toolchain": "workspace:*", - "@types/node": "^22.10.2", - "typescript": "^5.9.2" - } -} diff --git a/registry/software/curl/tsconfig.json b/registry/software/curl/tsconfig.json deleted file mode 100644 index 8f24167afd..0000000000 --- a/registry/software/curl/tsconfig.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./src" - }, - "include": ["src/**/*"] -} diff --git a/registry/software/diffutils/agentos-package.json b/registry/software/diffutils/agentos-package.json deleted file mode 100644 index 6c96a19370..0000000000 --- a/registry/software/diffutils/agentos-package.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "commands": [ - "diff" - ], - "registry": { - "title": "diffutils", - "description": "GNU diff for comparing files.", - "priority": 6, - "image": "/images/registry/diffutils.svg" - } -} diff --git a/registry/software/diffutils/bin/diff b/registry/software/diffutils/bin/diff deleted file mode 100644 index 003d074f99..0000000000 Binary files a/registry/software/diffutils/bin/diff and /dev/null differ diff --git a/registry/software/diffutils/package.json b/registry/software/diffutils/package.json deleted file mode 100644 index c750ec29bd..0000000000 --- a/registry/software/diffutils/package.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "name": "@agentos-software/diffutils", - "version": "0.3.3", - "type": "module", - "license": "Apache-2.0", - "description": "GNU diffutils for secure-exec VMs (diff)", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", - "files": [ - "dist", - "!dist/package", - "!dist/package.tar" - ], - "exports": { - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js" - } - }, - "scripts": { - "build": "agentos-toolchain stage --commands-dir ../../native/target/wasm32-wasip1/release/commands --if-missing skip && tsc && agentos-toolchain build", - "check-types": "tsc --noEmit" - }, - "devDependencies": { - "@agentos-software/manifest": "workspace:*", - "@rivet-dev/agentos-toolchain": "workspace:*", - "@types/node": "^22.10.2", - "typescript": "^5.9.2" - } -} diff --git a/registry/software/diffutils/tsconfig.json b/registry/software/diffutils/tsconfig.json deleted file mode 100644 index 8f24167afd..0000000000 --- a/registry/software/diffutils/tsconfig.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./src" - }, - "include": ["src/**/*"] -} diff --git a/registry/software/duckdb/agentos-package.json b/registry/software/duckdb/agentos-package.json deleted file mode 100644 index 7cd73f4e62..0000000000 --- a/registry/software/duckdb/agentos-package.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "commands": [ - "duckdb" - ], - "registry": { - "title": "DuckDB", - "description": "In-process analytics database CLI.", - "priority": 70, - "image": "/images/registry/duckdb.svg" - } -} diff --git a/registry/software/duckdb/package.json b/registry/software/duckdb/package.json deleted file mode 100644 index 19e9bdcb6e..0000000000 --- a/registry/software/duckdb/package.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "name": "@agentos-software/duckdb", - "version": "0.3.3", - "type": "module", - "license": "Apache-2.0", - "description": "DuckDB CLI for secure-exec VMs", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", - "files": [ - "dist", - "!dist/package", - "!dist/package.tar" - ], - "exports": { - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js" - } - }, - "scripts": { - "build": "agentos-toolchain stage --commands-dir ../../native/target/wasm32-wasip1/release/commands --if-missing skip && tsc && agentos-toolchain build", - "check-types": "tsc --noEmit" - }, - "devDependencies": { - "@agentos-software/manifest": "workspace:*", - "@rivet-dev/agentos-toolchain": "workspace:*", - "@types/node": "^22.10.2", - "typescript": "^5.9.2" - } -} diff --git a/registry/software/duckdb/tsconfig.json b/registry/software/duckdb/tsconfig.json deleted file mode 100644 index 8f24167afd..0000000000 --- a/registry/software/duckdb/tsconfig.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./src" - }, - "include": ["src/**/*"] -} diff --git a/registry/software/everything/package.json b/registry/software/everything/package.json deleted file mode 100644 index c47f10fd6a..0000000000 --- a/registry/software/everything/package.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "name": "@agentos-software/everything", - "version": "0.3.0-rc.2", - "type": "module", - "license": "Apache-2.0", - "description": "All available WASM command packages for secure-exec VMs in a single bundle", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", - "files": [ - "dist", - "!dist/package", - "!dist/package.tar" - ], - "exports": { - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js" - } - }, - "scripts": { - "build": "tsc", - "check-types": "tsc --noEmit" - }, - "dependencies": { - "@agentos-software/coreutils": "workspace:*", - "@agentos-software/sed": "workspace:*", - "@agentos-software/grep": "workspace:*", - "@agentos-software/gawk": "workspace:*", - "@agentos-software/findutils": "workspace:*", - "@agentos-software/diffutils": "workspace:*", - "@agentos-software/tar": "workspace:*", - "@agentos-software/gzip": "workspace:*", - "@agentos-software/curl": "workspace:*", - "@agentos-software/zip": "workspace:*", - "@agentos-software/unzip": "workspace:*", - "@agentos-software/jq": "workspace:*", - "@agentos-software/ripgrep": "workspace:*", - "@agentos-software/fd": "workspace:*", - "@agentos-software/tree": "workspace:*", - "@agentos-software/file": "workspace:*", - "@agentos-software/yq": "workspace:*", - "@agentos-software/codex-cli": "workspace:*" - }, - "devDependencies": { - "@agentos-software/manifest": "workspace:*", - "@types/node": "^22.10.2", - "typescript": "^5.9.2" - } -} diff --git a/registry/software/everything/src/index.ts b/registry/software/everything/src/index.ts deleted file mode 100644 index 58565c3337..0000000000 --- a/registry/software/everything/src/index.ts +++ /dev/null @@ -1,61 +0,0 @@ -import coreutils from "@agentos-software/coreutils"; -import sed from "@agentos-software/sed"; -import grep from "@agentos-software/grep"; -import gawk from "@agentos-software/gawk"; -import findutils from "@agentos-software/findutils"; -import diffutils from "@agentos-software/diffutils"; -import tar from "@agentos-software/tar"; -import gzip from "@agentos-software/gzip"; -import curl from "@agentos-software/curl"; -import zip from "@agentos-software/zip"; -import unzip from "@agentos-software/unzip"; -import jq from "@agentos-software/jq"; -import ripgrep from "@agentos-software/ripgrep"; -import fd from "@agentos-software/fd"; -import tree from "@agentos-software/tree"; -import file from "@agentos-software/file"; -import yq from "@agentos-software/yq"; -import codex from "@agentos-software/codex-cli"; - -const everything = [ - coreutils, - sed, - grep, - gawk, - findutils, - diffutils, - tar, - gzip, - curl, - zip, - unzip, - jq, - ripgrep, - fd, - tree, - file, - yq, - codex, -]; - -export default everything; -export { - coreutils, - sed, - grep, - gawk, - findutils, - diffutils, - tar, - gzip, - curl, - zip, - unzip, - jq, - ripgrep, - fd, - tree, - file, - yq, - codex, -}; diff --git a/registry/software/everything/tsconfig.json b/registry/software/everything/tsconfig.json deleted file mode 100644 index 8f24167afd..0000000000 --- a/registry/software/everything/tsconfig.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./src" - }, - "include": ["src/**/*"] -} diff --git a/registry/software/fd/agentos-package.json b/registry/software/fd/agentos-package.json deleted file mode 100644 index 2667d95742..0000000000 --- a/registry/software/fd/agentos-package.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "commands": [ - "fd" - ], - "registry": { - "title": "fd", - "description": "Fast file finder.", - "priority": 30 - } -} diff --git a/registry/software/fd/bin/fd b/registry/software/fd/bin/fd deleted file mode 100644 index b724ec24b1..0000000000 Binary files a/registry/software/fd/bin/fd and /dev/null differ diff --git a/registry/software/fd/package.json b/registry/software/fd/package.json deleted file mode 100644 index fe9fe5eebe..0000000000 --- a/registry/software/fd/package.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "name": "@agentos-software/fd", - "version": "0.3.3", - "type": "module", - "license": "Apache-2.0", - "description": "fd fast file finder for secure-exec VMs", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", - "files": [ - "dist", - "!dist/package", - "!dist/package.tar" - ], - "exports": { - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js" - } - }, - "scripts": { - "build": "agentos-toolchain stage --commands-dir ../../native/target/wasm32-wasip1/release/commands --if-missing skip && tsc && agentos-toolchain build", - "check-types": "tsc --noEmit" - }, - "devDependencies": { - "@agentos-software/manifest": "workspace:*", - "@rivet-dev/agentos-toolchain": "workspace:*", - "@types/node": "^22.10.2", - "typescript": "^5.9.2" - } -} diff --git a/registry/software/fd/tsconfig.json b/registry/software/fd/tsconfig.json deleted file mode 100644 index 8f24167afd..0000000000 --- a/registry/software/fd/tsconfig.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./src" - }, - "include": ["src/**/*"] -} diff --git a/registry/software/file/agentos-package.json b/registry/software/file/agentos-package.json deleted file mode 100644 index a0112ed101..0000000000 --- a/registry/software/file/agentos-package.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "commands": [ - "file" - ], - "registry": { - "title": "file", - "description": "Detect file types.", - "priority": 2 - } -} diff --git a/registry/software/file/bin/file b/registry/software/file/bin/file deleted file mode 100644 index 1e32fe6064..0000000000 Binary files a/registry/software/file/bin/file and /dev/null differ diff --git a/registry/software/file/package.json b/registry/software/file/package.json deleted file mode 100644 index 0ed0857785..0000000000 --- a/registry/software/file/package.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "name": "@agentos-software/file", - "version": "0.3.3", - "type": "module", - "license": "Apache-2.0", - "description": "file type detection for secure-exec VMs", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", - "files": [ - "dist", - "!dist/package", - "!dist/package.tar" - ], - "exports": { - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js" - } - }, - "scripts": { - "build": "agentos-toolchain stage --commands-dir ../../native/target/wasm32-wasip1/release/commands --if-missing skip && tsc && agentos-toolchain build", - "check-types": "tsc --noEmit" - }, - "devDependencies": { - "@agentos-software/manifest": "workspace:*", - "@rivet-dev/agentos-toolchain": "workspace:*", - "@types/node": "^22.10.2", - "typescript": "^5.9.2" - } -} diff --git a/registry/software/file/tsconfig.json b/registry/software/file/tsconfig.json deleted file mode 100644 index 8f24167afd..0000000000 --- a/registry/software/file/tsconfig.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./src" - }, - "include": ["src/**/*"] -} diff --git a/registry/software/findutils/agentos-package.json b/registry/software/findutils/agentos-package.json deleted file mode 100644 index b093937054..0000000000 --- a/registry/software/findutils/agentos-package.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "commands": [ - "find", - "xargs" - ], - "registry": { - "title": "findutils", - "description": "GNU find and xargs for file searching and batch execution.", - "priority": 15, - "image": "/images/registry/findutils.svg" - } -} diff --git a/registry/software/findutils/bin/find b/registry/software/findutils/bin/find deleted file mode 100644 index e08e8b58a1..0000000000 Binary files a/registry/software/findutils/bin/find and /dev/null differ diff --git a/registry/software/findutils/bin/xargs b/registry/software/findutils/bin/xargs deleted file mode 100644 index 613a7bb3ea..0000000000 Binary files a/registry/software/findutils/bin/xargs and /dev/null differ diff --git a/registry/software/findutils/package.json b/registry/software/findutils/package.json deleted file mode 100644 index 25723a270a..0000000000 --- a/registry/software/findutils/package.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "name": "@agentos-software/findutils", - "version": "0.3.3", - "type": "module", - "license": "Apache-2.0", - "description": "GNU findutils for secure-exec VMs (find, xargs)", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", - "files": [ - "dist", - "!dist/package", - "!dist/package.tar" - ], - "exports": { - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js" - } - }, - "scripts": { - "build": "agentos-toolchain stage --commands-dir ../../native/target/wasm32-wasip1/release/commands --if-missing skip && tsc && agentos-toolchain build", - "check-types": "tsc --noEmit" - }, - "devDependencies": { - "@agentos-software/manifest": "workspace:*", - "@rivet-dev/agentos-toolchain": "workspace:*", - "@types/node": "^22.10.2", - "typescript": "^5.9.2" - } -} diff --git a/registry/software/findutils/tsconfig.json b/registry/software/findutils/tsconfig.json deleted file mode 100644 index 8f24167afd..0000000000 --- a/registry/software/findutils/tsconfig.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./src" - }, - "include": ["src/**/*"] -} diff --git a/registry/software/gawk/agentos-package.json b/registry/software/gawk/agentos-package.json deleted file mode 100644 index 2efcf18704..0000000000 --- a/registry/software/gawk/agentos-package.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "commands": [ - "awk" - ], - "registry": { - "title": "gawk", - "description": "GNU awk text processing and data extraction.", - "priority": 20, - "image": "/images/registry/gawk.svg" - } -} diff --git a/registry/software/gawk/bin/awk b/registry/software/gawk/bin/awk deleted file mode 100644 index c4048f65c4..0000000000 Binary files a/registry/software/gawk/bin/awk and /dev/null differ diff --git a/registry/software/gawk/package.json b/registry/software/gawk/package.json deleted file mode 100644 index 87da3b8e24..0000000000 --- a/registry/software/gawk/package.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "name": "@agentos-software/gawk", - "version": "0.3.3", - "type": "module", - "license": "Apache-2.0", - "description": "GNU awk text processing for secure-exec VMs", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", - "files": [ - "dist", - "!dist/package", - "!dist/package.tar" - ], - "exports": { - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js" - } - }, - "scripts": { - "build": "agentos-toolchain stage --commands-dir ../../native/target/wasm32-wasip1/release/commands --if-missing skip && tsc && agentos-toolchain build", - "check-types": "tsc --noEmit" - }, - "devDependencies": { - "@agentos-software/manifest": "workspace:*", - "@rivet-dev/agentos-toolchain": "workspace:*", - "@types/node": "^22.10.2", - "typescript": "^5.9.2" - } -} diff --git a/registry/software/gawk/tsconfig.json b/registry/software/gawk/tsconfig.json deleted file mode 100644 index 8f24167afd..0000000000 --- a/registry/software/gawk/tsconfig.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./src" - }, - "include": ["src/**/*"] -} diff --git a/registry/software/git/agentos-package.json b/registry/software/git/agentos-package.json deleted file mode 100644 index 0cfc6ad3dc..0000000000 --- a/registry/software/git/agentos-package.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "commands": [ - "git" - ], - "aliases": { - "git-remote-http": "git", - "git-remote-https": "git" - }, - "registry": { - "title": "git", - "description": "Git version control (clone, commit, push, pull).", - "priority": 90, - "image": "/images/registry/git.svg" - } -} diff --git a/registry/software/git/bin/git b/registry/software/git/bin/git deleted file mode 100644 index 1865311237..0000000000 Binary files a/registry/software/git/bin/git and /dev/null differ diff --git a/registry/software/git/bin/git-remote-http b/registry/software/git/bin/git-remote-http deleted file mode 100644 index 1865311237..0000000000 Binary files a/registry/software/git/bin/git-remote-http and /dev/null differ diff --git a/registry/software/git/bin/git-remote-https b/registry/software/git/bin/git-remote-https deleted file mode 100644 index 1865311237..0000000000 Binary files a/registry/software/git/bin/git-remote-https and /dev/null differ diff --git a/registry/software/git/package.json b/registry/software/git/package.json deleted file mode 100644 index 85e9c331af..0000000000 --- a/registry/software/git/package.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "name": "@agentos-software/git", - "version": "0.3.3", - "type": "module", - "license": "Apache-2.0", - "description": "git version control for secure-exec VMs (planned)", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", - "files": [ - "dist", - "!dist/package", - "!dist/package.tar" - ], - "exports": { - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js" - } - }, - "scripts": { - "build": "agentos-toolchain stage --commands-dir ../../native/target/wasm32-wasip1/release/commands --if-missing skip && tsc && agentos-toolchain build", - "check-types": "tsc --noEmit" - }, - "devDependencies": { - "@agentos-software/manifest": "workspace:*", - "@rivet-dev/agentos-toolchain": "workspace:*", - "@types/node": "^22.10.2", - "typescript": "^5.9.2" - } -} diff --git a/registry/software/git/tsconfig.json b/registry/software/git/tsconfig.json deleted file mode 100644 index 8f24167afd..0000000000 --- a/registry/software/git/tsconfig.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./src" - }, - "include": ["src/**/*"] -} diff --git a/registry/software/grep/agentos-package.json b/registry/software/grep/agentos-package.json deleted file mode 100644 index a67ea41cf1..0000000000 --- a/registry/software/grep/agentos-package.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "commands": [ - "grep" - ], - "aliases": { - "egrep": "grep", - "fgrep": "grep" - }, - "registry": { - "title": "grep", - "description": "GNU grep pattern matching (grep, egrep, fgrep).", - "priority": 40, - "image": "/images/registry/grep.svg" - } -} diff --git a/registry/software/grep/bin/egrep b/registry/software/grep/bin/egrep deleted file mode 100644 index c78e121f14..0000000000 Binary files a/registry/software/grep/bin/egrep and /dev/null differ diff --git a/registry/software/grep/bin/fgrep b/registry/software/grep/bin/fgrep deleted file mode 100644 index c78e121f14..0000000000 Binary files a/registry/software/grep/bin/fgrep and /dev/null differ diff --git a/registry/software/grep/bin/grep b/registry/software/grep/bin/grep deleted file mode 100644 index c78e121f14..0000000000 Binary files a/registry/software/grep/bin/grep and /dev/null differ diff --git a/registry/software/grep/package.json b/registry/software/grep/package.json deleted file mode 100644 index 095dad7b64..0000000000 --- a/registry/software/grep/package.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "name": "@agentos-software/grep", - "version": "0.3.3", - "type": "module", - "license": "Apache-2.0", - "description": "GNU grep pattern matching for secure-exec VMs (grep, egrep, fgrep)", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", - "files": [ - "dist", - "!dist/package", - "!dist/package.tar" - ], - "exports": { - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js" - } - }, - "scripts": { - "build": "agentos-toolchain stage --commands-dir ../../native/target/wasm32-wasip1/release/commands --if-missing skip && tsc && agentos-toolchain build", - "check-types": "tsc --noEmit" - }, - "devDependencies": { - "@agentos-software/manifest": "workspace:*", - "@rivet-dev/agentos-toolchain": "workspace:*", - "@types/node": "^22.10.2", - "typescript": "^5.9.2" - } -} diff --git a/registry/software/grep/tsconfig.json b/registry/software/grep/tsconfig.json deleted file mode 100644 index 8f24167afd..0000000000 --- a/registry/software/grep/tsconfig.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./src" - }, - "include": ["src/**/*"] -} diff --git a/registry/software/gzip/agentos-package.json b/registry/software/gzip/agentos-package.json deleted file mode 100644 index 93b1776859..0000000000 --- a/registry/software/gzip/agentos-package.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "commands": [ - "gzip" - ], - "aliases": { - "gunzip": "gzip", - "zcat": "gzip" - }, - "registry": { - "title": "gzip", - "description": "GNU gzip compression (gzip, gunzip, zcat).", - "priority": 8, - "image": "/images/registry/gzip.svg" - } -} diff --git a/registry/software/gzip/bin/gunzip b/registry/software/gzip/bin/gunzip deleted file mode 100644 index e60ebe5317..0000000000 Binary files a/registry/software/gzip/bin/gunzip and /dev/null differ diff --git a/registry/software/gzip/bin/gzip b/registry/software/gzip/bin/gzip deleted file mode 100644 index e60ebe5317..0000000000 Binary files a/registry/software/gzip/bin/gzip and /dev/null differ diff --git a/registry/software/gzip/bin/zcat b/registry/software/gzip/bin/zcat deleted file mode 100644 index e60ebe5317..0000000000 Binary files a/registry/software/gzip/bin/zcat and /dev/null differ diff --git a/registry/software/gzip/package.json b/registry/software/gzip/package.json deleted file mode 100644 index 088279e99a..0000000000 --- a/registry/software/gzip/package.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "name": "@agentos-software/gzip", - "version": "0.3.3", - "type": "module", - "license": "Apache-2.0", - "description": "GNU gzip compression for secure-exec VMs (gzip, gunzip, zcat)", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", - "files": [ - "dist", - "!dist/package", - "!dist/package.tar" - ], - "exports": { - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js" - } - }, - "scripts": { - "build": "agentos-toolchain stage --commands-dir ../../native/target/wasm32-wasip1/release/commands --if-missing skip && tsc && agentos-toolchain build", - "check-types": "tsc --noEmit" - }, - "devDependencies": { - "@agentos-software/manifest": "workspace:*", - "@rivet-dev/agentos-toolchain": "workspace:*", - "@types/node": "^22.10.2", - "typescript": "^5.9.2" - } -} diff --git a/registry/software/gzip/tsconfig.json b/registry/software/gzip/tsconfig.json deleted file mode 100644 index 8f24167afd..0000000000 --- a/registry/software/gzip/tsconfig.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./src" - }, - "include": ["src/**/*"] -} diff --git a/registry/software/http-get/agentos-package.json b/registry/software/http-get/agentos-package.json deleted file mode 100644 index d45077f00b..0000000000 --- a/registry/software/http-get/agentos-package.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "commands": [ - "http_get" - ], - "registry": { - "title": "http-get", - "description": "Minimal HTTP GET fetch helper." - } -} diff --git a/registry/software/http-get/bin/http_get b/registry/software/http-get/bin/http_get deleted file mode 100644 index 52a1fb8342..0000000000 Binary files a/registry/software/http-get/bin/http_get and /dev/null differ diff --git a/registry/software/http-get/package.json b/registry/software/http-get/package.json deleted file mode 100644 index 38ed2ed4fc..0000000000 --- a/registry/software/http-get/package.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "name": "@agentos-software/http-get", - "version": "0.3.3", - "type": "module", - "license": "Apache-2.0", - "description": "Minimal HTTP GET fetch helper for secure-exec VMs", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", - "files": [ - "dist", - "!dist/package", - "!dist/package.tar" - ], - "exports": { - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js" - } - }, - "scripts": { - "build": "agentos-toolchain stage --commands-dir ../../native/target/wasm32-wasip1/release/commands --if-missing skip && tsc && agentos-toolchain build", - "check-types": "tsc --noEmit" - }, - "devDependencies": { - "@agentos-software/manifest": "workspace:*", - "@rivet-dev/agentos-toolchain": "workspace:*", - "@types/node": "^22.10.2", - "typescript": "^5.9.2" - } -} diff --git a/registry/software/http-get/tsconfig.json b/registry/software/http-get/tsconfig.json deleted file mode 100644 index 8f24167afd..0000000000 --- a/registry/software/http-get/tsconfig.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./src" - }, - "include": ["src/**/*"] -} diff --git a/registry/software/jq/agentos-package.json b/registry/software/jq/agentos-package.json deleted file mode 100644 index b5cc20dd01..0000000000 --- a/registry/software/jq/agentos-package.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "commands": [ - "jq" - ], - "registry": { - "title": "jq", - "description": "Lightweight JSON processor.", - "priority": 80, - "image": "/images/registry/jq.svg" - } -} diff --git a/registry/software/jq/bin/jq b/registry/software/jq/bin/jq deleted file mode 100644 index 22f0359722..0000000000 Binary files a/registry/software/jq/bin/jq and /dev/null differ diff --git a/registry/software/jq/package.json b/registry/software/jq/package.json deleted file mode 100644 index f4be15e4a0..0000000000 --- a/registry/software/jq/package.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "name": "@agentos-software/jq", - "version": "0.3.3", - "type": "module", - "license": "Apache-2.0", - "description": "jq JSON processor for secure-exec VMs", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", - "files": [ - "dist", - "!dist/package", - "!dist/package.tar" - ], - "exports": { - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js" - } - }, - "scripts": { - "build": "agentos-toolchain stage --commands-dir ../../native/target/wasm32-wasip1/release/commands --if-missing skip && tsc && agentos-toolchain build", - "check-types": "tsc --noEmit" - }, - "devDependencies": { - "@agentos-software/manifest": "workspace:*", - "@rivet-dev/agentos-toolchain": "workspace:*", - "@types/node": "^22.10.2", - "typescript": "^5.9.2" - } -} diff --git a/registry/software/jq/tsconfig.json b/registry/software/jq/tsconfig.json deleted file mode 100644 index 8f24167afd..0000000000 --- a/registry/software/jq/tsconfig.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./src" - }, - "include": ["src/**/*"] -} diff --git a/registry/software/ripgrep/agentos-package.json b/registry/software/ripgrep/agentos-package.json deleted file mode 100644 index f0ba60fdda..0000000000 --- a/registry/software/ripgrep/agentos-package.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "commands": [ - "rg" - ], - "registry": { - "title": "ripgrep", - "description": "Fast recursive search (rg).", - "priority": 85 - } -} diff --git a/registry/software/ripgrep/bin/rg b/registry/software/ripgrep/bin/rg deleted file mode 100644 index 76a63b75ea..0000000000 Binary files a/registry/software/ripgrep/bin/rg and /dev/null differ diff --git a/registry/software/ripgrep/package.json b/registry/software/ripgrep/package.json deleted file mode 100644 index 5542da9f8e..0000000000 --- a/registry/software/ripgrep/package.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "name": "@agentos-software/ripgrep", - "version": "0.3.3", - "type": "module", - "license": "Apache-2.0", - "description": "ripgrep fast search for secure-exec VMs", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", - "files": [ - "dist", - "!dist/package", - "!dist/package.tar" - ], - "exports": { - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js" - } - }, - "scripts": { - "build": "agentos-toolchain stage --commands-dir ../../native/target/wasm32-wasip1/release/commands --if-missing skip && tsc && agentos-toolchain build", - "check-types": "tsc --noEmit" - }, - "devDependencies": { - "@agentos-software/manifest": "workspace:*", - "@rivet-dev/agentos-toolchain": "workspace:*", - "@types/node": "^22.10.2", - "typescript": "^5.9.2" - } -} diff --git a/registry/software/ripgrep/tsconfig.json b/registry/software/ripgrep/tsconfig.json deleted file mode 100644 index 8f24167afd..0000000000 --- a/registry/software/ripgrep/tsconfig.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./src" - }, - "include": ["src/**/*"] -} diff --git a/registry/software/sed/agentos-package.json b/registry/software/sed/agentos-package.json deleted file mode 100644 index 4bc0fe5a76..0000000000 --- a/registry/software/sed/agentos-package.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "commands": [ - "sed" - ], - "registry": { - "title": "sed", - "description": "GNU stream editor for text transformation.", - "priority": 35, - "image": "/images/registry/sed.svg" - } -} diff --git a/registry/software/sed/bin/sed b/registry/software/sed/bin/sed deleted file mode 100644 index cf0fd26325..0000000000 Binary files a/registry/software/sed/bin/sed and /dev/null differ diff --git a/registry/software/sed/package.json b/registry/software/sed/package.json deleted file mode 100644 index a5f127ea0d..0000000000 --- a/registry/software/sed/package.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "name": "@agentos-software/sed", - "version": "0.3.3", - "type": "module", - "license": "Apache-2.0", - "description": "GNU sed stream editor for secure-exec VMs", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", - "files": [ - "dist", - "!dist/package", - "!dist/package.tar" - ], - "exports": { - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js" - } - }, - "scripts": { - "build": "agentos-toolchain stage --commands-dir ../../native/target/wasm32-wasip1/release/commands --if-missing skip && tsc && agentos-toolchain build", - "check-types": "tsc --noEmit" - }, - "devDependencies": { - "@agentos-software/manifest": "workspace:*", - "@rivet-dev/agentos-toolchain": "workspace:*", - "@types/node": "^22.10.2", - "typescript": "^5.9.2" - } -} diff --git a/registry/software/sed/tsconfig.json b/registry/software/sed/tsconfig.json deleted file mode 100644 index 8f24167afd..0000000000 --- a/registry/software/sed/tsconfig.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./src" - }, - "include": ["src/**/*"] -} diff --git a/registry/software/sqlite3/agentos-package.json b/registry/software/sqlite3/agentos-package.json deleted file mode 100644 index 6d140af495..0000000000 --- a/registry/software/sqlite3/agentos-package.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "commands": [ - "sqlite3" - ], - "registry": { - "title": "SQLite3", - "description": "SQLite database command-line interface.", - "priority": 75, - "image": "/images/registry/sqlite3.svg" - } -} diff --git a/registry/software/sqlite3/bin/sqlite3 b/registry/software/sqlite3/bin/sqlite3 deleted file mode 100755 index a9ee1f206f..0000000000 Binary files a/registry/software/sqlite3/bin/sqlite3 and /dev/null differ diff --git a/registry/software/sqlite3/package.json b/registry/software/sqlite3/package.json deleted file mode 100644 index 26841719db..0000000000 --- a/registry/software/sqlite3/package.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "name": "@agentos-software/sqlite3", - "version": "0.3.3", - "type": "module", - "license": "Apache-2.0", - "description": "SQLite3 CLI for secure-exec VMs", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", - "files": [ - "dist", - "!dist/package", - "!dist/package.tar" - ], - "exports": { - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js" - } - }, - "scripts": { - "build": "agentos-toolchain stage --commands-dir ../../native/target/wasm32-wasip1/release/commands --if-missing skip && tsc && agentos-toolchain build", - "check-types": "tsc --noEmit" - }, - "devDependencies": { - "@agentos-software/manifest": "workspace:*", - "@rivet-dev/agentos-toolchain": "workspace:*", - "@types/node": "^22.10.2", - "typescript": "^5.9.2" - } -} diff --git a/registry/software/sqlite3/tsconfig.json b/registry/software/sqlite3/tsconfig.json deleted file mode 100644 index 8f24167afd..0000000000 --- a/registry/software/sqlite3/tsconfig.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./src" - }, - "include": ["src/**/*"] -} diff --git a/registry/software/tar/agentos-package.json b/registry/software/tar/agentos-package.json deleted file mode 100644 index ab25a8bbea..0000000000 --- a/registry/software/tar/agentos-package.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "commands": [ - "tar" - ], - "registry": { - "title": "tar", - "description": "GNU tar archiver.", - "priority": 60, - "image": "/images/registry/tar.svg" - } -} diff --git a/registry/software/tar/bin/tar b/registry/software/tar/bin/tar deleted file mode 100644 index 5d813b99d9..0000000000 Binary files a/registry/software/tar/bin/tar and /dev/null differ diff --git a/registry/software/tar/package.json b/registry/software/tar/package.json deleted file mode 100644 index 957a20ed65..0000000000 --- a/registry/software/tar/package.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "name": "@agentos-software/tar", - "version": "0.3.3", - "type": "module", - "license": "Apache-2.0", - "description": "GNU tar archiver for secure-exec VMs", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", - "files": [ - "dist", - "!dist/package", - "!dist/package.tar" - ], - "exports": { - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js" - } - }, - "scripts": { - "build": "agentos-toolchain stage --commands-dir ../../native/target/wasm32-wasip1/release/commands --if-missing skip && tsc && agentos-toolchain build", - "check-types": "tsc --noEmit" - }, - "devDependencies": { - "@agentos-software/manifest": "workspace:*", - "@rivet-dev/agentos-toolchain": "workspace:*", - "@types/node": "^22.10.2", - "typescript": "^5.9.2" - } -} diff --git a/registry/software/tar/tsconfig.json b/registry/software/tar/tsconfig.json deleted file mode 100644 index 8f24167afd..0000000000 --- a/registry/software/tar/tsconfig.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./src" - }, - "include": ["src/**/*"] -} diff --git a/registry/software/tree/agentos-package.json b/registry/software/tree/agentos-package.json deleted file mode 100644 index f6d6e4e996..0000000000 --- a/registry/software/tree/agentos-package.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "commands": [ - "tree" - ], - "registry": { - "title": "tree", - "description": "Display directory structure as a tree.", - "priority": 25 - } -} diff --git a/registry/software/tree/bin/tree b/registry/software/tree/bin/tree deleted file mode 100644 index 49e9c7c18d..0000000000 Binary files a/registry/software/tree/bin/tree and /dev/null differ diff --git a/registry/software/tree/package.json b/registry/software/tree/package.json deleted file mode 100644 index cca158354b..0000000000 --- a/registry/software/tree/package.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "name": "@agentos-software/tree", - "version": "0.3.3", - "type": "module", - "license": "Apache-2.0", - "description": "tree directory listing for secure-exec VMs", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", - "files": [ - "dist", - "!dist/package", - "!dist/package.tar" - ], - "exports": { - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js" - } - }, - "scripts": { - "build": "agentos-toolchain stage --commands-dir ../../native/target/wasm32-wasip1/release/commands --if-missing skip && tsc && agentos-toolchain build", - "check-types": "tsc --noEmit" - }, - "devDependencies": { - "@agentos-software/manifest": "workspace:*", - "@rivet-dev/agentos-toolchain": "workspace:*", - "@types/node": "^22.10.2", - "typescript": "^5.9.2" - } -} diff --git a/registry/software/tree/tsconfig.json b/registry/software/tree/tsconfig.json deleted file mode 100644 index 8f24167afd..0000000000 --- a/registry/software/tree/tsconfig.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./src" - }, - "include": ["src/**/*"] -} diff --git a/registry/software/unzip/agentos-package.json b/registry/software/unzip/agentos-package.json deleted file mode 100644 index 8f395875f0..0000000000 --- a/registry/software/unzip/agentos-package.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "commands": [ - "unzip" - ], - "registry": { - "title": "unzip", - "description": "Extract zip archives.", - "priority": 10 - } -} diff --git a/registry/software/unzip/bin/unzip b/registry/software/unzip/bin/unzip deleted file mode 100644 index 07f40041e1..0000000000 Binary files a/registry/software/unzip/bin/unzip and /dev/null differ diff --git a/registry/software/unzip/package.json b/registry/software/unzip/package.json deleted file mode 100644 index 3c400a4cb2..0000000000 --- a/registry/software/unzip/package.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "name": "@agentos-software/unzip", - "version": "0.3.3", - "type": "module", - "license": "Apache-2.0", - "description": "unzip archive extraction for secure-exec VMs", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", - "files": [ - "dist", - "!dist/package", - "!dist/package.tar" - ], - "exports": { - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js" - } - }, - "scripts": { - "build": "agentos-toolchain stage --commands-dir ../../native/target/wasm32-wasip1/release/commands --if-missing skip && tsc && agentos-toolchain build", - "check-types": "tsc --noEmit" - }, - "devDependencies": { - "@agentos-software/manifest": "workspace:*", - "@rivet-dev/agentos-toolchain": "workspace:*", - "@types/node": "^22.10.2", - "typescript": "^5.9.2" - } -} diff --git a/registry/software/unzip/tsconfig.json b/registry/software/unzip/tsconfig.json deleted file mode 100644 index 8f24167afd..0000000000 --- a/registry/software/unzip/tsconfig.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./src" - }, - "include": ["src/**/*"] -} diff --git a/registry/software/vim/agentos-package.json b/registry/software/vim/agentos-package.json deleted file mode 100644 index 24e713cc33..0000000000 --- a/registry/software/vim/agentos-package.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "name": "vim", - "commands": [ - "vim" - ], - "provides": { - "env": { - "VIM": "/usr/local/share/vim", - "VIMRUNTIME": "/usr/local/share/vim/vim92" - }, - "files": [ - { - "source": "share/vim/vim92", - "target": "/usr/local/share/vim/vim92" - } - ] - }, - "registry": { - "title": "vim", - "description": "Vim text editor with bundled runtime.", - "priority": 65, - "image": "/images/registry/vim.svg" - } -} diff --git a/registry/software/vim/package.json b/registry/software/vim/package.json deleted file mode 100644 index 24e8f7d0d3..0000000000 --- a/registry/software/vim/package.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "name": "@agentos-software/vim", - "version": "0.3.3", - "type": "module", - "license": "Apache-2.0", - "description": "vim for secure-exec VMs (wasm build + bundled runtime via provides)", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", - "files": [ - "dist", - "!dist/package", - "!dist/package.tar" - ], - "exports": { - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js", - "default": "./dist/index.js" - } - }, - "scripts": { - "build": "agentos-toolchain stage --commands-dir ../../native/target/wasm32-wasip1/release/commands --if-missing skip && node ./scripts/stage-runtime.mjs && tsc && agentos-toolchain build", - "check-types": "tsc --noEmit" - }, - "devDependencies": { - "@agentos-software/manifest": "workspace:*", - "@rivet-dev/agentos-toolchain": "workspace:*", - "@types/node": "^22.10.2", - "typescript": "^5.9.2" - } -} diff --git a/registry/software/vim/tsconfig.json b/registry/software/vim/tsconfig.json deleted file mode 100644 index 8f24167afd..0000000000 --- a/registry/software/vim/tsconfig.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./src" - }, - "include": ["src/**/*"] -} diff --git a/registry/software/vix/agentos-package.json b/registry/software/vix/agentos-package.json deleted file mode 100644 index 88ce6ad04a..0000000000 --- a/registry/software/vix/agentos-package.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "name": "vix", - "commands": [ - "vix" - ], - "registry": { - "title": "vix", - "description": "Lightweight vi-style text editor." - } -} diff --git a/registry/software/vix/package.json b/registry/software/vix/package.json deleted file mode 100644 index e51042dc9b..0000000000 --- a/registry/software/vix/package.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "name": "@agentos-software/vix", - "version": "0.1.0", - "type": "module", - "license": "Apache-2.0", - "description": "vix editor for secure-exec VMs (wasm build)", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", - "files": [ - "dist", - "!dist/package", - "!dist/package.tar" - ], - "exports": { - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js", - "default": "./dist/index.js" - } - }, - "scripts": { - "build": "agentos-toolchain stage --commands-dir ../../native/target/wasm32-wasip1/release/commands --if-missing skip && tsc && agentos-toolchain build", - "check-types": "tsc --noEmit" - }, - "devDependencies": { - "@agentos-software/manifest": "workspace:*", - "@rivet-dev/agentos-toolchain": "workspace:*", - "@types/node": "^22.10.2", - "typescript": "^5.9.2" - } -} diff --git a/registry/software/vix/tsconfig.json b/registry/software/vix/tsconfig.json deleted file mode 100644 index 8f24167afd..0000000000 --- a/registry/software/vix/tsconfig.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./src" - }, - "include": ["src/**/*"] -} diff --git a/registry/software/wget/agentos-package.json b/registry/software/wget/agentos-package.json deleted file mode 100644 index db274f6149..0000000000 --- a/registry/software/wget/agentos-package.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "commands": [ - "wget" - ], - "registry": { - "title": "wget", - "description": "GNU wget file downloader.", - "priority": 55, - "image": "/images/registry/wget.svg" - } -} diff --git a/registry/software/wget/package.json b/registry/software/wget/package.json deleted file mode 100644 index c73531a083..0000000000 --- a/registry/software/wget/package.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "name": "@agentos-software/wget", - "version": "0.3.3", - "type": "module", - "license": "Apache-2.0", - "description": "GNU wget HTTP client for secure-exec VMs", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", - "files": [ - "dist", - "!dist/package", - "!dist/package.tar" - ], - "exports": { - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js" - } - }, - "scripts": { - "build": "agentos-toolchain stage --commands-dir ../../native/target/wasm32-wasip1/release/commands --if-missing skip && tsc && agentos-toolchain build", - "check-types": "tsc --noEmit" - }, - "devDependencies": { - "@agentos-software/manifest": "workspace:*", - "@rivet-dev/agentos-toolchain": "workspace:*", - "@types/node": "^22.10.2", - "typescript": "^5.9.2" - } -} diff --git a/registry/software/wget/tsconfig.json b/registry/software/wget/tsconfig.json deleted file mode 100644 index 8f24167afd..0000000000 --- a/registry/software/wget/tsconfig.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./src" - }, - "include": ["src/**/*"] -} diff --git a/registry/software/yq/agentos-package.json b/registry/software/yq/agentos-package.json deleted file mode 100644 index b753aea6ee..0000000000 --- a/registry/software/yq/agentos-package.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "commands": [ - "yq" - ], - "registry": { - "title": "yq", - "description": "YAML/JSON processor.", - "priority": 4 - } -} diff --git a/registry/software/yq/bin/yq b/registry/software/yq/bin/yq deleted file mode 100644 index 98a2de41ee..0000000000 Binary files a/registry/software/yq/bin/yq and /dev/null differ diff --git a/registry/software/yq/package.json b/registry/software/yq/package.json deleted file mode 100644 index 4ce45c0b4b..0000000000 --- a/registry/software/yq/package.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "name": "@agentos-software/yq", - "version": "0.3.3", - "type": "module", - "license": "Apache-2.0", - "description": "yq YAML/JSON processor for secure-exec VMs", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", - "files": [ - "dist", - "!dist/package", - "!dist/package.tar" - ], - "exports": { - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js" - } - }, - "scripts": { - "build": "agentos-toolchain stage --commands-dir ../../native/target/wasm32-wasip1/release/commands --if-missing skip && tsc && agentos-toolchain build", - "check-types": "tsc --noEmit" - }, - "devDependencies": { - "@agentos-software/manifest": "workspace:*", - "@rivet-dev/agentos-toolchain": "workspace:*", - "@types/node": "^22.10.2", - "typescript": "^5.9.2" - } -} diff --git a/registry/software/yq/tsconfig.json b/registry/software/yq/tsconfig.json deleted file mode 100644 index 8f24167afd..0000000000 --- a/registry/software/yq/tsconfig.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./src" - }, - "include": ["src/**/*"] -} diff --git a/registry/software/zip/agentos-package.json b/registry/software/zip/agentos-package.json deleted file mode 100644 index 298bd79a77..0000000000 --- a/registry/software/zip/agentos-package.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "commands": [ - "zip" - ], - "registry": { - "title": "zip", - "description": "Create zip archives.", - "priority": 12 - } -} diff --git a/registry/software/zip/bin/zip b/registry/software/zip/bin/zip deleted file mode 100644 index 8324bdc9fe..0000000000 Binary files a/registry/software/zip/bin/zip and /dev/null differ diff --git a/registry/software/zip/package.json b/registry/software/zip/package.json deleted file mode 100644 index 73b23a0971..0000000000 --- a/registry/software/zip/package.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "name": "@agentos-software/zip", - "version": "0.3.3", - "type": "module", - "license": "Apache-2.0", - "description": "zip archive creation for secure-exec VMs", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", - "files": [ - "dist", - "!dist/package", - "!dist/package.tar" - ], - "exports": { - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js" - } - }, - "scripts": { - "build": "agentos-toolchain stage --commands-dir ../../native/target/wasm32-wasip1/release/commands --if-missing skip && tsc && agentos-toolchain build", - "check-types": "tsc --noEmit" - }, - "devDependencies": { - "@agentos-software/manifest": "workspace:*", - "@rivet-dev/agentos-toolchain": "workspace:*", - "@types/node": "^22.10.2", - "typescript": "^5.9.2" - } -} diff --git a/registry/software/zip/src/index.ts b/registry/software/zip/src/index.ts deleted file mode 100644 index dba100e2e1..0000000000 --- a/registry/software/zip/src/index.ts +++ /dev/null @@ -1,5 +0,0 @@ -import type { SoftwarePackageRef } from "@agentos-software/manifest"; - -const packagePath = new URL("./package.aospkg", import.meta.url).pathname; - -export default { packagePath } satisfies SoftwarePackageRef; diff --git a/registry/software/zip/tsconfig.json b/registry/software/zip/tsconfig.json deleted file mode 100644 index 8f24167afd..0000000000 --- a/registry/software/zip/tsconfig.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./src" - }, - "include": ["src/**/*"] -} diff --git a/registry/tests/helpers.ts b/registry/tests/helpers.ts deleted file mode 100644 index 963f5afd7b..0000000000 --- a/registry/tests/helpers.ts +++ /dev/null @@ -1,120 +0,0 @@ -import { existsSync } from "node:fs"; -import { resolve, dirname } from "node:path"; -import { fileURLToPath } from "node:url"; -import { describe, it } from "vitest"; - -const __dirname = dirname(fileURLToPath(import.meta.url)); - -/** Directory containing WASM command binaries built from Rust. */ -export const COMMANDS_DIR = resolve( - process.env.AGENTOS_WASM_COMMANDS_DIR ?? - resolve(__dirname, "../native/target/wasm32-wasip1/release/commands"), -); - -/** Directory containing C-compiled WASM binaries. */ -export const C_BUILD_DIR = resolve( - process.env.AGENTOS_C_WASM_COMMANDS_DIR ?? - resolve(__dirname, "../native/c/build/"), -); - -/** Whether the main WASM command binaries are available (includes 'sh'). */ -export const hasWasmBinaries = - existsSync(COMMANDS_DIR) && existsSync(resolve(COMMANDS_DIR, "sh")); - -/** - * Check whether specific C WASM binaries are present. - * @param names - Binary names to check for inside C_BUILD_DIR. - * @returns true if all requested binaries exist. - */ -export function hasCWasmBinaries(...names: string[]): boolean { - if (!existsSync(C_BUILD_DIR)) return false; - return names.every((name) => existsSync(resolve(C_BUILD_DIR, name))); -} - -/** - * Returns a skip-reason string if WASM binaries are missing, or false if - * they are available and tests should run. - */ -export function skipReason(): string | false { - if (!hasWasmBinaries) { - return `WASM binaries not found at ${COMMANDS_DIR} — build with \`make wasm\` first`; - } - return false; -} - -export function describeIf( - condition: unknown, - ...args: Parameters -): void { - if (condition) { - // Vitest's overloaded tuple shape is awkward to preserve across helper forwarding. - // @ts-expect-error forwarded describe() arguments stay runtime-compatible. - describe(...args); - return; - } - const [name] = args; - describe.skip(`${String(name)} [environment prerequisites not met]`, () => {}); -} - -export function itIf( - condition: unknown, - ...args: Parameters -): void { - if (condition) { - // Vitest's overloaded tuple shape is awkward to preserve across helper forwarding. - // @ts-expect-error forwarded it() arguments stay runtime-compatible. - it(...args); - return; - } - const [name] = args; - it.skip(`${String(name)} [environment prerequisites not met]`, () => {}); -} - -// Re-exports from the repo-owned generic runtime surface. -export { - AF_INET, - AF_UNIX, - allowAll, - createInMemoryFileSystem, - SIGTERM, - SOCK_DGRAM, - SOCK_STREAM, -} from "@rivet-dev/agentos-runtime-core/test-runtime"; -import { - allowAll, - createKernel as createKernelBase, -} from "@rivet-dev/agentos-runtime-core/test-runtime"; -export type { - DriverProcess, - Kernel, - KernelInterface, - KernelRuntimeDriver, - ProcessContext, - VirtualFileSystem, -} from "@rivet-dev/agentos-runtime-core/test-runtime"; -export { - createWasmVmRuntime, - DEFAULT_FIRST_PARTY_TIERS, - WASMVM_COMMANDS, - type PermissionTier, - type WasmVmRuntimeOptions, -} from "@rivet-dev/agentos-runtime-core/test-runtime"; -export { - createNodeHostNetworkAdapter, - createNodeRuntime, - NodeFileSystem, -} from "@rivet-dev/agentos-runtime-core/test-runtime"; -export { TerminalHarness } from "./terminal-harness.js"; - -/** - * Registry integration tests assume they can bootstrap runtimes and /bin stubs - * unless they explicitly opt into a stricter permission policy. - */ -export function createKernel( - options: Parameters[0], -): ReturnType { - return createKernelBase({ - ...options, - permissions: options.permissions ?? allowAll, - }); -} diff --git a/registry/tests/kernel/helpers.ts b/registry/tests/kernel/helpers.ts deleted file mode 100644 index 07be2a7f2c..0000000000 --- a/registry/tests/kernel/helpers.ts +++ /dev/null @@ -1,103 +0,0 @@ -/** - * Integration test helpers for kernel tests that depend on WASM command binaries. - * - * Re-exports infrastructure from the parent helpers.ts and provides - * createIntegrationKernel / skipUnlessWasmBuilt for cross-runtime tests. - */ - -import { - AF_INET, - AF_UNIX, - COMMANDS_DIR, - C_BUILD_DIR, - describeIf, - hasWasmBinaries, - itIf, - NodeFileSystem, - SIGTERM, - SOCK_DGRAM, - SOCK_STREAM, - TerminalHarness, - skipReason, - createInMemoryFileSystem, - createKernel, - createWasmVmRuntime, - createNodeRuntime, -} from "../helpers.js"; -import type { Kernel, Permissions, VirtualFileSystem } from "../helpers.js"; - -export { - AF_INET, - AF_UNIX, - COMMANDS_DIR, - C_BUILD_DIR, - describeIf, - hasWasmBinaries, - itIf, - NodeFileSystem, - SIGTERM, - SOCK_DGRAM, - SOCK_STREAM, - TerminalHarness, - skipReason, - createInMemoryFileSystem, - createKernel, - createWasmVmRuntime, - createNodeRuntime, -} from "../helpers.js"; -export type { Kernel, Permissions, VirtualFileSystem } from "../helpers.js"; - -export interface IntegrationKernelResult { - kernel: Kernel; - vfs: VirtualFileSystem; - dispose: () => Promise; -} - -export interface IntegrationKernelOptions { - runtimes?: ("wasmvm" | "node")[]; - loopbackExemptPorts?: number[]; - commandDirs?: string[]; - permissions?: Permissions; -} - -/** - * Create a kernel with the in-scope runtime drivers for integration testing. - * - * Mount order matters. Last-mounted driver wins for overlapping commands: - * 1. WasmVM first: provides sh/bash/coreutils (90+ commands) - * 2. Node second: overrides WasmVM's 'node' stub with real V8 - */ -export async function createIntegrationKernel( - options?: IntegrationKernelOptions, -): Promise { - const runtimes = options?.runtimes ?? ["wasmvm"]; - const vfs = createInMemoryFileSystem(); - const kernel = createKernel({ - filesystem: vfs, - loopbackExemptPorts: options?.loopbackExemptPorts, - permissions: options?.permissions, - }); - - if (runtimes.includes("wasmvm")) { - await kernel.mount( - createWasmVmRuntime({ commandDirs: options?.commandDirs ?? [COMMANDS_DIR] }), - ); - } - if (runtimes.includes("node")) { - await kernel.mount(createNodeRuntime()); - } - - return { - kernel, - vfs, - dispose: () => kernel.dispose(), - }; -} - -/** - * Skip helper: returns a reason string if the WASM binaries are not built, - * or false if the commands directory exists and tests can run. - */ -export function skipUnlessWasmBuilt(): string | false { - return skipReason(); -} diff --git a/registry/tests/terminal-harness.ts b/registry/tests/terminal-harness.ts deleted file mode 100644 index b01ad66a9e..0000000000 --- a/registry/tests/terminal-harness.ts +++ /dev/null @@ -1,159 +0,0 @@ -/** - * TerminalHarness wires openShell() to a headless xterm Terminal so registry - * tests can assert against deterministic terminal screen state. - */ - -import { Terminal } from "@xterm/headless"; -import type { Kernel } from "./helpers.js"; - -type ShellHandle = ReturnType; - -const SETTLE_MS = 50; -const POLL_MS = 20; -const DEFAULT_WAIT_TIMEOUT_MS = 5_000; - -export class TerminalHarness { - readonly term: Terminal; - readonly shell: ShellHandle; - private typing = false; - private disposed = false; - - constructor( - kernel: Kernel, - options?: { - cols?: number; - rows?: number; - env?: Record; - cwd?: string; - }, - ) { - const cols = options?.cols ?? 80; - const rows = options?.rows ?? 24; - - this.term = new Terminal({ cols, rows, allowProposedApi: true }); - this.shell = kernel.openShell({ - cols, - rows, - env: options?.env, - cwd: options?.cwd, - onStderr: (data: Uint8Array) => { - this.term.write(data); - }, - }); - this.shell.onData = (data: Uint8Array) => { - this.term.write(data); - }; - } - - async type(input: string): Promise { - if (this.typing) { - throw new Error( - "TerminalHarness.type() called while previous type() is still in-flight", - ); - } - this.typing = true; - try { - await this.typeInternal(input); - } finally { - this.typing = false; - } - } - - private typeInternal(input: string): Promise { - return new Promise((resolve) => { - let timer: ReturnType | null = null; - const originalOnData = this.shell.onData; - - const resetTimer = () => { - if (timer !== null) clearTimeout(timer); - timer = setTimeout(() => { - this.shell.onData = originalOnData; - resolve(); - }, SETTLE_MS); - }; - - this.shell.onData = (data: Uint8Array) => { - this.term.write(data); - resetTimer(); - }; - - resetTimer(); - this.shell.write(input); - }); - } - - screenshotTrimmed(): string { - const buf = this.term.buffer.active; - const lines: string[] = []; - - for (let row = 0; row < this.term.rows; row++) { - const line = buf.getLine(buf.viewportY + row); - lines.push(line ? line.translateToString(true) : ""); - } - - while (lines.length > 0 && lines[lines.length - 1] === "") { - lines.pop(); - } - - return lines.join("\n"); - } - - line(row: number): string { - const buf = this.term.buffer.active; - const line = buf.getLine(buf.viewportY + row); - return line ? line.translateToString(true) : ""; - } - - async waitFor( - text: string, - occurrence: number = 1, - timeoutMs: number = DEFAULT_WAIT_TIMEOUT_MS, - ): Promise { - const deadline = Date.now() + timeoutMs; - - while (true) { - const screen = this.screenshotTrimmed(); - - let count = 0; - let idx = -1; - while (true) { - idx = screen.indexOf(text, idx + 1); - if (idx === -1) break; - count++; - if (count >= occurrence) return; - } - - if (Date.now() >= deadline) { - throw new Error( - `waitFor("${text}", ${occurrence}) timed out after ${timeoutMs}ms.\n` + - `Expected: "${text}" (occurrence ${occurrence})\n` + - `Screen:\n${screen}`, - ); - } - - await new Promise((resolve) => setTimeout(resolve, POLL_MS)); - } - } - - async exit(): Promise { - this.shell.write("\x04"); - return this.shell.wait(); - } - - async dispose(): Promise { - if (this.disposed) return; - this.disposed = true; - - try { - this.shell.kill(); - await Promise.race([ - this.shell.wait(), - new Promise((resolve) => setTimeout(resolve, 500)), - ]); - } catch { - // Shell may already be gone. - } - - this.term.dispose(); - } -} diff --git a/registry/tests/wasmvm/curl.test.ts b/registry/tests/wasmvm/curl.test.ts deleted file mode 100644 index da9c1b5600..0000000000 --- a/registry/tests/wasmvm/curl.test.ts +++ /dev/null @@ -1,719 +0,0 @@ -/** - * Integration tests for curl and the C socket layer (host_socket.c). - * - * Tests the WASM socket implementation that powers curl: - * - DNS resolution (getaddrinfo) - * - TCP socket creation and connection - * - Non-blocking socket mode (fcntl O_NONBLOCK) - * - Socket options (getsockopt SO_ERROR, setsockopt TCP_NODELAY) - * - Poll for readability/writability - * - HTTP send/recv over raw sockets - * - Remote endpoint connectivity - */ - -import { describe, it, expect, afterEach, beforeAll, afterAll } from 'vitest'; -import { createWasmVmRuntime } from '../helpers.js'; -import { - allowAll, - C_BUILD_DIR, - COMMANDS_DIR, - createInMemoryFileSystem, - createKernel, - describeIf, - hasCWasmBinaries, - hasWasmBinaries, - itIf, -} from '../helpers.js'; -import type { Kernel } from '../helpers.js'; -import { - createServer as createHttpServer, - type IncomingMessage, - type Server as HttpServer, - type ServerResponse, -} from 'node:http'; -import { createServer as createHttpsServer, type Server as HttpsServer } from 'node:https'; -import { - createConnection, - createServer as createTcpServer, - type Server as TcpServer, -} from 'node:net'; -import { execSync } from 'node:child_process'; -import { existsSync, unlinkSync, writeFileSync } from 'node:fs'; -import { tmpdir } from 'node:os'; -import { join, resolve } from 'node:path'; - -// The upstream curl parity assertions below only hold for the C-built curl -// artifact; the Rust fallback in COMMANDS_DIR intentionally supports a smaller -// flag surface and should not be used for these cases. -const hasHttpGetTest = hasWasmBinaries && existsSync(resolve(COMMANDS_DIR, 'http_get_test')); -const hasCurl = hasCWasmBinaries('curl'); -const runExternalNetwork = process.env.AGENTOS_E2E_NETWORK === '1'; -const EXTERNAL_HOST = 'example.com'; -const EXTERNAL_TCP_PORT = 80; -const EXTERNAL_HTTP_URL = `http://${EXTERNAL_HOST}/`; -const EXTERNAL_HTTPS_URL = `https://${EXTERNAL_HOST}/`; -const EXTERNAL_EXPECTED_BODY = 'Example Domain'; -const EXTERNAL_RETRY_ATTEMPTS = 3; -const EXTERNAL_RETRY_DELAY_MS = 1_000; -const EXTERNAL_PROBE_TIMEOUT_MS = 8_000; -let hasOpenssl = false; - -try { - execSync('openssl version', { stdio: 'pipe' }); - hasOpenssl = true; -} catch { - hasOpenssl = false; -} - -function sleep(ms: number): Promise { - return new Promise((resolveSleep) => setTimeout(resolveSleep, ms)); -} - -function formatError(error: unknown): string { - if (error instanceof Error) return error.message; - return String(error); -} - -async function retryExternal(run: () => Promise, attempts = EXTERNAL_RETRY_ATTEMPTS): Promise { - let lastError: unknown; - for (let attempt = 1; attempt <= attempts; attempt += 1) { - try { - return await run(); - } catch (error) { - lastError = error; - if (attempt < attempts) { - await sleep(EXTERNAL_RETRY_DELAY_MS); - } - } - } - - throw lastError ?? new Error('external network probe failed'); -} - -async function probeExternalTcp(): Promise { - await new Promise((resolveConnect, rejectConnect) => { - const socket = createConnection({ - host: EXTERNAL_HOST, - port: EXTERNAL_TCP_PORT, - }); - let settled = false; - - const finish = (callback: () => void) => { - if (settled) return; - settled = true; - callback(); - }; - - socket.setTimeout(EXTERNAL_PROBE_TIMEOUT_MS); - socket.once('connect', () => { - finish(() => { - socket.end(); - resolveConnect(); - }); - }); - socket.once('timeout', () => { - finish(() => { - socket.destroy(); - rejectConnect(new Error(`timed out connecting to ${EXTERNAL_HOST}:${EXTERNAL_TCP_PORT}`)); - }); - }); - socket.once('error', (error) => { - finish(() => { - socket.destroy(); - rejectConnect(error); - }); - }); - }); -} - -async function probeExternalHttps(): Promise { - const response = await fetch(EXTERNAL_HTTPS_URL, { - signal: AbortSignal.timeout(EXTERNAL_PROBE_TIMEOUT_MS), - }); - if (!response.ok) { - throw new Error(`host probe failed with HTTP ${response.status}`); - } - await response.arrayBuffer(); -} - -const externalNetworkSkipReason = runExternalNetwork - ? await (async () => { - try { - await retryExternal(async () => { - await probeExternalTcp(); - await probeExternalHttps(); - }); - return false as const; - } catch (error) { - return `external network unavailable: ${formatError(error)}`; - } - })() - : 'set AGENTOS_E2E_NETWORK=1 to enable external-network coverage'; - -function generateSelfSignedCert(): { key: string; cert: string } { - const keyPath = join(tmpdir(), `curl-test-key-${process.pid}-${Date.now()}.pem`); - try { - const key = execSync( - 'openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 2>/dev/null', - { encoding: 'utf8' }, - ); - writeFileSync(keyPath, key); - const cert = execSync( - `openssl req -new -x509 -key "${keyPath}" -days 1 -subj "/CN=localhost" -addext "subjectAltName=DNS:localhost,IP:127.0.0.1" 2>/dev/null`, - { encoding: 'utf8' }, - ); - return { key, cert }; - } finally { - try { - unlinkSync(keyPath); - } catch { - // Best effort cleanup for test temp files. - } - } -} - -describeIf(hasCurl || hasHttpGetTest, 'curl and socket layer', () => { - let kernel: Kernel; - let httpServer: HttpServer; - let httpsServer: HttpsServer; - let keepAliveServer: TcpServer; - let httpPort: number; - let httpsPort: number; - let keepAlivePort: number; - let flakyRequestCount = 0; - - beforeAll(async () => { - httpServer = createHttpServer((req: IncomingMessage, res: ServerResponse) => { - const url = req.url ?? '/'; - - if (url === '/json') { - res.writeHead(200, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify({ ok: true, path: url })); - return; - } - - if (url === '/redirect') { - res.writeHead(302, { Location: `http://127.0.0.1:${httpPort}/final` }); - res.end(); - return; - } - - if (url === '/final') { - res.writeHead(200, { 'Content-Type': 'text/plain' }); - res.end('followed redirect'); - return; - } - - if (url === '/one') { - res.writeHead(200, { 'Content-Type': 'text/plain' }); - res.end('first-response\n'); - return; - } - - if (url === '/two') { - res.writeHead(200, { 'Content-Type': 'text/plain' }); - res.end('second-response\n'); - return; - } - - if (url === '/echo' && req.method === 'POST') { - let body = ''; - req.on('data', (chunk) => { - body += chunk; - }); - req.on('end', () => { - res.writeHead(200, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify({ - method: req.method, - body, - header: req.headers['x-test'] ?? null, - })); - }); - return; - } - - if (url === '/json-post' && req.method === 'POST') { - let body = ''; - req.on('data', (chunk) => { - body += chunk; - }); - req.on('end', () => { - res.writeHead(200, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify({ - method: req.method, - contentType: req.headers['content-type'] ?? null, - accept: req.headers.accept ?? null, - body, - })); - }); - return; - } - - if (url === '/head-test') { - res.writeHead(200, { - 'Content-Type': 'text/plain', - 'X-Test-Header': 'present', - }); - if (req.method === 'HEAD') { - res.end(); - } else { - res.end('body should not appear in HEAD output'); - } - return; - } - - if (url === '/auth-required') { - const auth = req.headers.authorization; - if (!auth || !auth.startsWith('Basic ')) { - res.writeHead(401, { 'Content-Type': 'text/plain' }); - res.end('unauthorized'); - return; - } - - const decoded = Buffer.from(auth.slice(6), 'base64').toString(); - res.writeHead(200, { 'Content-Type': 'text/plain' }); - res.end(`authenticated: ${decoded}`); - return; - } - - if (url === '/upload' && req.method === 'POST') { - const contentType = req.headers['content-type'] ?? ''; - const chunks: Buffer[] = []; - req.on('data', (chunk) => { - chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); - }); - req.on('end', () => { - const body = Buffer.concat(chunks).toString(); - res.writeHead(200, { 'Content-Type': 'text/plain' }); - res.end( - `multipart: ${contentType.startsWith('multipart/form-data')}\n` + - `body-contains-file: ${body.includes('upload.txt')}`, - ); - }); - return; - } - - if (url === '/binary') { - const payload = Buffer.alloc(256); - for (let i = 0; i < payload.length; i++) payload[i] = i & 0xff; - res.writeHead(200, { - 'Content-Type': 'application/octet-stream', - 'Content-Length': String(payload.length), - }); - res.end(payload); - return; - } - - if (url === '/named.txt') { - const body = 'downloaded-by-remote-name\n'; - res.writeHead(200, { - 'Content-Type': 'text/plain', - 'Content-Length': String(Buffer.byteLength(body)), - }); - res.end(body); - return; - } - - if (url === '/flaky') { - flakyRequestCount += 1; - if (flakyRequestCount === 1) { - res.writeHead(503, { 'Content-Type': 'text/plain' }); - res.end('retry please'); - return; - } - - res.writeHead(200, { 'Content-Type': 'text/plain' }); - res.end('retry succeeded'); - return; - } - - if (url === '/status') { - res.writeHead(201, { 'Content-Type': 'text/plain' }); - res.end('created'); - return; - } - - res.writeHead(404, { 'Content-Type': 'text/plain' }); - res.end('not found'); - }); - - await new Promise((resolveListen) => { - httpServer.listen(0, '127.0.0.1', resolveListen); - }); - httpPort = (httpServer.address() as import('node:net').AddressInfo).port; - - if (hasOpenssl) { - const tlsCert = generateSelfSignedCert(); - httpsServer = createHttpsServer({ key: tlsCert.key, cert: tlsCert.cert }, (req, res) => { - if (req.url === '/json') { - res.writeHead(200, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify({ secure: true, path: req.url })); - return; - } - - if (req.url === '/keepalive') { - const body = 'hello from tls keepalive'; - res.writeHead(200, { - 'Content-Type': 'text/plain', - 'Content-Length': String(Buffer.byteLength(body)), - Connection: 'keep-alive', - 'Keep-Alive': 'timeout=60', - }); - res.end(body); - return; - } - - res.writeHead(404, { 'Content-Type': 'text/plain' }); - res.end('not found'); - }); - httpsServer.keepAliveTimeout = 60000; - - await new Promise((resolveListen) => { - httpsServer.listen(0, '127.0.0.1', resolveListen); - }); - httpsPort = (httpsServer.address() as import('node:net').AddressInfo).port; - } - - keepAliveServer = createTcpServer((socket) => { - socket.once('data', () => { - const body = 'hello from keepalive'; - socket.write( - 'HTTP/1.1 200 OK\r\n' + - 'Content-Type: text/plain\r\n' + - `Content-Length: ${Buffer.byteLength(body)}\r\n` + - 'Connection: keep-alive\r\n' + - 'Keep-Alive: timeout=60\r\n' + - '\r\n' + - body, - ); - // Intentionally keep the socket open to exercise curl shutdown logic. - }); - }); - - await new Promise((resolveListen) => { - keepAliveServer.listen(0, '127.0.0.1', resolveListen); - }); - keepAlivePort = (keepAliveServer.address() as import('node:net').AddressInfo).port; - }); - - afterAll(async () => { - if (httpServer) { - await new Promise((resolveClose) => httpServer.close(() => resolveClose())); - } - if (httpsServer) { - await new Promise((resolveClose) => httpsServer.close(() => resolveClose())); - } - if (keepAliveServer) { - await new Promise((resolveClose) => keepAliveServer.close(() => resolveClose())); - } - }); - - async function createKernelWithNet() { - flakyRequestCount = 0; - const filesystem = createInMemoryFileSystem(); - await (filesystem as any).chmod('/', 0o1777); - await filesystem.mkdir('/tmp', { recursive: true }); - await (filesystem as any).chmod('/tmp', 0o1777); - - kernel = createKernel({ - filesystem, - permissions: allowAll, - loopbackExemptPorts: [httpPort, httpsPort, keepAlivePort], - }); - await kernel.mount(createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] })); - return kernel; - } - - async function execWithRetry(command: string) { - let lastResult: Awaited> | undefined; - for (let attempt = 1; attempt <= EXTERNAL_RETRY_ATTEMPTS; attempt += 1) { - lastResult = await kernel.exec(command); - if (lastResult.exitCode === 0) return lastResult; - if (attempt < EXTERNAL_RETRY_ATTEMPTS) { - await sleep(EXTERNAL_RETRY_DELAY_MS); - } - } - - return lastResult!; - } - - afterEach(async () => { - await kernel?.dispose(); - }); - - itIf(hasHttpGetTest, 'http_get_test reaches a local HTTP server', async () => { - await createKernelWithNet(); - const result = await kernel.exec(`http_get_test 127.0.0.1 ${httpPort} /json`); - expect(result.exitCode).toBe(0); - expect(result.stdout).toContain('HTTP/1.1 200'); - expect(result.stdout).toContain('"ok":true'); - }, 15000); - - itIf(hasHttpGetTest, 'http_get_test preserves non-blocking connect diagnostics', async () => { - await createKernelWithNet(); - const result = await kernel.exec(`http_get_test 127.0.0.1 ${httpPort} /json`); - expect(result.exitCode).toBe(0); - expect(result.stderr).toContain('fcntl F_SETFL(NONBLOCK)=0'); - expect(result.stderr).toMatch(/connect=(0|-1 errno=\d+)/); - expect(result.stderr).toContain('getsockopt(SO_ERROR)=0 value=0'); - expect(result.stderr).toContain('poll(POLLOUT)=1'); - }, 15000); - - itIf(hasCurl, 'curl GET returns JSON from a local server', async () => { - await createKernelWithNet(); - const result = await kernel.exec(`curl -s http://127.0.0.1:${httpPort}/json`); - expect(result.exitCode).toBe(0); - expect(result.stdout).toContain('"ok":true'); - }, 15000); - - itIf(hasCurl, 'curl --version reports the upstream tool version', async () => { - await createKernelWithNet(); - const result = await kernel.exec('curl --version'); - expect(result.exitCode).toBe(0); - expect(result.stdout).toContain('curl 8.11.1'); - expect(result.stdout).toMatch(/Protocols:/); - }, 15000); - - itIf(hasCurl, 'curl -L follows redirects', async () => { - await createKernelWithNet(); - const result = await kernel.exec(`curl -s -L http://127.0.0.1:${httpPort}/redirect`); - expect(result.exitCode).toBe(0); - expect(result.stdout).toBe('followed redirect'); - }, 15000); - - itIf(hasCurl, 'curl POST sends body and headers', async () => { - await createKernelWithNet(); - const result = await kernel.exec( - `curl -s -X POST -H 'X-Test: edge-case' -d 'payload-data' http://127.0.0.1:${httpPort}/echo`, - ); - expect(result.exitCode).toBe(0); - const body = JSON.parse(result.stdout); - expect(body.method).toBe('POST'); - expect(body.body).toBe('payload-data'); - expect(body.header).toBe('edge-case'); - }, 15000); - - itIf(hasCurl, 'curl --json sends JSON with the expected headers', async () => { - await createKernelWithNet(); - const result = await kernel.exec( - `curl -s --json '{\"hello\":\"world\"}' http://127.0.0.1:${httpPort}/json-post`, - ); - expect(result.exitCode).toBe(0); - const body = JSON.parse(result.stdout); - expect(body.method).toBe('POST'); - expect(body.body).toBe('{"hello":"world"}'); - expect(body.contentType).toBe('application/json'); - expect(body.accept).toBe('application/json'); - }, 15000); - - itIf(hasCurl, 'curl -I returns response headers without the body', async () => { - await createKernelWithNet(); - const result = await kernel.exec(`curl -s -I http://127.0.0.1:${httpPort}/head-test`); - expect(result.exitCode).toBe(0); - expect(result.stdout).toContain('HTTP/'); - expect(result.stdout).toMatch(/X-Test-Header/i); - expect(result.stdout).not.toContain('body should not appear'); - }, 15000); - - itIf(hasCurl, 'curl -u sends HTTP Basic authentication', async () => { - await createKernelWithNet(); - const result = await kernel.exec(`curl -s -u user:pass http://127.0.0.1:${httpPort}/auth-required`); - expect(result.exitCode).toBe(0); - expect(result.stdout).toBe('authenticated: user:pass'); - }, 15000); - - itIf(hasCurl, 'curl -F uploads multipart form data', async () => { - await createKernelWithNet(); - await kernel.writeFile('/tmp/upload.txt', 'file payload\n'); - const result = await kernel.exec(`curl -s -F file=@/tmp/upload.txt http://127.0.0.1:${httpPort}/upload`); - expect(result.exitCode).toBe(0); - expect(result.stdout).toContain('multipart: true'); - expect(result.stdout).toContain('body-contains-file: true'); - }, 15000); - - itIf(hasCurl, 'curl -K reads options from a config file', async () => { - await createKernelWithNet(); - await kernel.writeFile( - '/tmp/curlrc', - `silent\nurl = "http://127.0.0.1:${httpPort}/json"\n`, - ); - const result = await kernel.exec('curl -K /tmp/curlrc'); - expect(result.exitCode).toBe(0); - expect(result.stdout).toContain('"ok":true'); - }, 15000); - - itIf(hasCurl, 'curl -o writes text output to a file', async () => { - await createKernelWithNet(); - const result = await kernel.exec(`curl -s -o /tmp/out.json http://127.0.0.1:${httpPort}/json`); - expect(result.exitCode).toBe(0); - expect(result.stdout).toBe(''); - const file = new TextDecoder().decode(await kernel.readFile('/tmp/out.json')); - expect(file).toContain('"ok":true'); - }, 15000); - - itIf(hasCurl, 'curl -o respects the current working directory for relative output paths', async () => { - await createKernelWithNet(); - const result = await kernel.exec( - `mkdir -p /tmp/curl-cwd && cd /tmp/curl-cwd && ` + - `curl -s -o local.txt http://127.0.0.1:${httpPort}/named.txt && cat /tmp/curl-cwd/local.txt`, - ); - expect(result.exitCode).toBe(0); - expect(result.stdout).toBe('downloaded-by-remote-name\n'); - }, 15000); - - itIf(hasCurl, 'curl -o writes binary output without truncation', async () => { - await createKernelWithNet(); - const result = await kernel.exec(`curl -s -o /tmp/out.bin http://127.0.0.1:${httpPort}/binary`); - expect(result.exitCode).toBe(0); - const file = await kernel.readFile('/tmp/out.bin'); - expect(file).toHaveLength(256); - expect(Array.from(file.slice(0, 8))).toEqual([0, 1, 2, 3, 4, 5, 6, 7]); - expect(Array.from(file.slice(-4))).toEqual([252, 253, 254, 255]); - }, 15000); - - itIf(hasCurl, 'curl -D and -o split headers and body into separate files', async () => { - await createKernelWithNet(); - const result = await kernel.exec( - `curl -s -D /tmp/headers.txt -o /tmp/body.txt http://127.0.0.1:${httpPort}/named.txt`, - ); - expect(result.exitCode).toBe(0); - expect(result.stdout).toBe(''); - - const headers = new TextDecoder().decode(await kernel.readFile('/tmp/headers.txt')); - const body = new TextDecoder().decode(await kernel.readFile('/tmp/body.txt')); - expect(headers).toContain('HTTP/1.1 200 OK'); - expect(headers).toMatch(/Content-Type: text\/plain/i); - expect(body).toBe('downloaded-by-remote-name\n'); - }, 15000); - - itIf(hasCurl, 'curl -O writes to the remote filename', async () => { - await createKernelWithNet(); - const result = await kernel.exec( - `mkdir -p /tmp/remote-name && cd /tmp/remote-name && ` + - `curl -s -O http://127.0.0.1:${httpPort}/named.txt && cat /tmp/remote-name/named.txt`, - ); - expect(result.exitCode).toBe(0); - expect(result.stdout).toBe('downloaded-by-remote-name\n'); - }, 15000); - - itIf(hasCurl, 'curl -w writes the HTTP status code', async () => { - await createKernelWithNet(); - const result = await kernel.exec(`curl -s -w '%{http_code}' http://127.0.0.1:${httpPort}/status`); - expect(result.exitCode).toBe(0); - expect(result.stdout).toContain('created'); - expect(result.stdout).toContain('201'); - }, 15000); - - itIf(hasCurl, 'curl -f reports HTTP errors with a non-zero exit code', async () => { - await createKernelWithNet(); - const result = await kernel.exec(`curl -fsS http://127.0.0.1:${httpPort}/missing`); - expect(result.exitCode).not.toBe(0); - expect(result.stderr).toMatch(/404|not found|error/i); - }, 15000); - - itIf(hasCurl, 'curl --fail-with-body preserves the response body on HTTP errors', async () => { - await createKernelWithNet(); - const result = await kernel.exec(`curl -sS --fail-with-body http://127.0.0.1:${httpPort}/missing`); - expect(result.exitCode).not.toBe(0); - expect(result.stdout).toBe('not found'); - expect(result.stderr).toMatch(/404|error/i); - }, 15000); - - itIf(hasCurl, 'curl reports refused connections without hanging', async () => { - await createKernelWithNet(); - - const probe = createTcpServer(); - await new Promise((resolveListen) => probe.listen(0, '127.0.0.1', resolveListen)); - const unusedPort = (probe.address() as import('node:net').AddressInfo).port; - await new Promise((resolveClose) => probe.close(() => resolveClose())); - - const startedAt = Date.now(); - const result = await kernel.exec(`curl -sS http://127.0.0.1:${unusedPort}/`); - expect(result.exitCode).not.toBe(0); - expect(Date.now() - startedAt).toBeLessThan(8000); - expect(result.stderr).toMatch(/connect|refused|failed/i); - }, 15000); - - itIf(hasCurl, 'curl reports DNS failures cleanly', async () => { - await createKernelWithNet(); - const result = await kernel.exec('curl -sS http://does-not-exist.invalid/'); - expect(result.exitCode).not.toBe(0); - expect(result.stderr).toMatch(/resolve|host|dns/i); - }, 15000); - - itIf(hasCurl, 'curl handles multiple URLs in one invocation', async () => { - await createKernelWithNet(); - const result = await kernel.exec( - `curl -s http://127.0.0.1:${httpPort}/one http://127.0.0.1:${httpPort}/two`, - ); - expect(result.exitCode).toBe(0); - expect(result.stdout).toBe('first-response\nsecond-response\n'); - }, 15000); - - itIf(hasCurl, 'curl --retry retries transient HTTP failures', async () => { - await createKernelWithNet(); - const result = await kernel.exec( - `curl -fsS --retry 2 --retry-delay 0 http://127.0.0.1:${httpPort}/flaky`, - ); - expect(result.exitCode).toBe(0); - expect(result.stdout).toBe('retry succeeded'); - expect(flakyRequestCount).toBeGreaterThanOrEqual(2); - }, 15000); - - itIf(hasCurl, 'curl exits promptly after a keep-alive response', async () => { - await createKernelWithNet(); - const startedAt = Date.now(); - const result = await kernel.exec(`curl -s http://127.0.0.1:${keepAlivePort}/`); - expect(result.exitCode).toBe(0); - expect(result.stdout).toBe('hello from keepalive'); - expect(Date.now() - startedAt).toBeLessThan(8000); - }, 15000); - - itIf(hasCurl && hasOpenssl, 'curl -k performs an HTTPS request through the WASI TLS backend', async () => { - await createKernelWithNet(); - const result = await kernel.exec(`curl -ks https://127.0.0.1:${httpsPort}/json`); - expect(result.exitCode).toBe(0); - expect(result.stdout).toContain('"secure":true'); - }, 15000); - - itIf(hasCurl && hasOpenssl, 'curl fails TLS verification without -k on a self-signed endpoint', async () => { - await createKernelWithNet(); - const result = await kernel.exec(`curl -sS https://127.0.0.1:${httpsPort}/json`); - expect(result.exitCode).not.toBe(0); - expect(result.stderr).toMatch(/certificate|tls|ssl|verify/i); - }, 15000); - - itIf(hasCurl && hasOpenssl, 'curl -k exits promptly after an HTTPS keep-alive response', async () => { - await createKernelWithNet(); - const startedAt = Date.now(); - const result = await kernel.exec(`curl -ks https://127.0.0.1:${httpsPort}/keepalive`); - expect(result.exitCode).toBe(0); - expect(result.stdout).toBe('hello from tls keepalive'); - expect(Date.now() - startedAt).toBeLessThan(8000); - }, 15000); - - itIf(hasHttpGetTest && !externalNetworkSkipReason, 'http_get_test reaches an external host over real TCP', async () => { - await createKernelWithNet(); - const result = await execWithRetry(`http_get_test ${EXTERNAL_HOST} ${EXTERNAL_TCP_PORT} /`); - expect(result.exitCode).toBe(0); - expect(result.stdout).toMatch(/HTTP\/1\.[01] (200|301|302)/); - }, 30000); - - itIf(hasCurl && !externalNetworkSkipReason, 'curl reaches a real external HTTP endpoint', async () => { - await createKernelWithNet(); - const result = await execWithRetry( - `curl -fsSL --retry 2 --retry-delay 1 --retry-all-errors --connect-timeout 10 --max-time 30 ${EXTERNAL_HTTP_URL}`, - ); - expect(result.exitCode).toBe(0); - expect(result.stdout).toContain(EXTERNAL_EXPECTED_BODY); - }, 30000); - - itIf(hasCurl && !externalNetworkSkipReason, 'curl reaches a real external HTTPS endpoint', async () => { - await createKernelWithNet(); - const result = await execWithRetry( - `curl -fsSL --retry 2 --retry-delay 1 --retry-all-errors --connect-timeout 10 --max-time 30 ${EXTERNAL_HTTPS_URL}`, - ); - expect(result.exitCode).toBe(0); - expect(result.stdout).toContain(EXTERNAL_EXPECTED_BODY); - }, 30000); -}); diff --git a/registry/tests/wasmvm/fd-find.test.ts b/registry/tests/wasmvm/fd-find.test.ts deleted file mode 100644 index dfb5faf21d..0000000000 --- a/registry/tests/wasmvm/fd-find.test.ts +++ /dev/null @@ -1,255 +0,0 @@ -/** - * Integration tests for fd (fd-find) command. - * - * Verifies file finding with regex patterns, extension filters, type filters, - * hidden file skipping, and empty directory handling via kernel.exec() with - * real WASM binaries. - * - * Note: kernel.exec() wraps commands in sh -c. Brush-shell currently returns - * exit code 17 for all child commands (benign "could not retrieve pid" issue). - * Tests verify stdout correctness rather than exit code. - */ - -import { describe, it, expect, afterEach } from 'vitest'; -import { createWasmVmRuntime } from '../helpers.js'; -import { COMMANDS_DIR, createKernel, describeIf, hasWasmBinaries } from '../helpers.js'; -import type { Kernel } from '../helpers.js'; - -// Minimal in-memory VFS for kernel tests -class SimpleVFS { - private files = new Map(); - private dirs = new Set(['/']); - - async readFile(path: string): Promise { - const data = this.files.get(path); - if (!data) throw new Error(`ENOENT: ${path}`); - return data; - } - async readTextFile(path: string): Promise { - return new TextDecoder().decode(await this.readFile(path)); - } - async readDir(path: string): Promise { - const prefix = path === '/' ? '/' : path + '/'; - const entries: string[] = []; - for (const p of [...this.files.keys(), ...this.dirs]) { - if (p !== path && p.startsWith(prefix)) { - const rest = p.slice(prefix.length); - if (!rest.includes('/')) entries.push(rest); - } - } - return entries; - } - async readDirWithTypes(path: string) { - return (await this.readDir(path)).map(name => ({ - name, - isDirectory: this.dirs.has(path === '/' ? `/${name}` : `${path}/${name}`), - })); - } - async writeFile(path: string, content: string | Uint8Array): Promise { - const data = typeof content === 'string' ? new TextEncoder().encode(content) : content; - this.files.set(path, new Uint8Array(data)); - const parts = path.split('/').filter(Boolean); - for (let i = 1; i < parts.length; i++) { - this.dirs.add('/' + parts.slice(0, i).join('/')); - } - } - async createDir(path: string) { this.dirs.add(path); } - async mkdir(path: string, _options?: { recursive?: boolean }) { - this.dirs.add(path); - const parts = path.split('/').filter(Boolean); - for (let i = 1; i < parts.length; i++) { - this.dirs.add('/' + parts.slice(0, i).join('/')); - } - } - async exists(path: string): Promise { - return this.files.has(path) || this.dirs.has(path); - } - async stat(path: string) { - const isDir = this.dirs.has(path); - const data = this.files.get(path); - if (!isDir && !data) throw new Error(`ENOENT: ${path}`); - return { - mode: isDir ? 0o40755 : 0o100644, - size: data?.length ?? 0, - isDirectory: isDir, - isSymbolicLink: false, - atimeMs: Date.now(), - mtimeMs: Date.now(), - ctimeMs: Date.now(), - birthtimeMs: Date.now(), - ino: 0, - nlink: 1, - uid: 1000, - gid: 1000, - }; - } - async lstat(path: string) { return this.stat(path); } - async removeFile(path: string) { this.files.delete(path); } - async removeDir(path: string) { this.dirs.delete(path); } - async rename(oldPath: string, newPath: string) { - const data = this.files.get(oldPath); - if (data) { - this.files.set(newPath, data); - this.files.delete(oldPath); - } - } - async pread(path: string, offset: number, length: number): Promise { - const data = this.files.get(path); - if (!data) throw new Error(`ENOENT: ${path}`); - return data.slice(offset, offset + length); - } - async pwrite(path: string, offset: number, content: Uint8Array): Promise { - const data = this.files.get(path); - if (!data) throw new Error(`ENOENT: ${path}`); - const next = new Uint8Array(Math.max(data.length, offset + content.length)); - next.set(data); - next.set(content, offset); - this.files.set(path, next); - } -} - -/** Create a VFS pre-populated with a test directory structure */ -async function createTestVFS(): Promise { - const vfs = new SimpleVFS(); - // /project/ - // src/ - // main.js - // utils.js - // helpers.ts - // lib/ - // parser.js - // docs/ - // readme.md - // .hidden/ - // secret.txt - // .gitignore - // config.json - await vfs.writeFile('/project/src/main.js', 'console.log("main")'); - await vfs.writeFile('/project/src/utils.js', 'export {}'); - await vfs.writeFile('/project/src/helpers.ts', 'export {}'); - await vfs.writeFile('/project/lib/parser.js', 'module.exports = {}'); - await vfs.writeFile('/project/docs/readme.md', '# Readme'); - await vfs.writeFile('/project/.hidden/secret.txt', 'secret'); - await vfs.writeFile('/project/.gitignore', 'node_modules'); - await vfs.writeFile('/project/config.json', '{}'); - // /empty/ — empty directory - await vfs.mkdir('/empty', { recursive: true }); - return vfs; -} - -/** Parse fd output lines, sorted for deterministic comparison */ -function parseLines(stdout: string): string[] { - return stdout.split('\n').filter(l => l.length > 0).sort(); -} - -describeIf(hasWasmBinaries, 'fd-find command', { timeout: 10_000 }, () => { - let kernel: Kernel; - - afterEach(async () => { - await kernel?.dispose(); - }); - - it('finds files matching regex pattern in current directory', async () => { - const vfs = await createTestVFS(); - kernel = createKernel({ filesystem: vfs as any }); - await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); - - const result = await kernel.exec('fd main /project', {}); - const lines = parseLines(result.stdout); - expect(lines).toContain('/project/src/main.js'); - }); - - it('finds all .js files with -e js', async () => { - const vfs = await createTestVFS(); - kernel = createKernel({ filesystem: vfs as any }); - await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); - - const result = await kernel.exec('fd -e js . /project', {}); - const lines = parseLines(result.stdout); - expect(lines).toContain('/project/src/main.js'); - expect(lines).toContain('/project/src/utils.js'); - expect(lines).toContain('/project/lib/parser.js'); - // .ts files should NOT match - expect(lines).not.toContain('/project/src/helpers.ts'); - }); - - it('finds only files with -t f', async () => { - const vfs = await createTestVFS(); - kernel = createKernel({ filesystem: vfs as any }); - await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); - - const result = await kernel.exec('fd -t f . /project', {}); - const lines = parseLines(result.stdout); - // All entries should be files, not directories - for (const line of lines) { - const stat = await vfs.stat(line); - expect(stat.isDirectory).toBe(false); - } - // Should include known files - expect(lines).toContain('/project/src/main.js'); - expect(lines).toContain('/project/config.json'); - }); - - it('finds only directories with -t d', async () => { - const vfs = await createTestVFS(); - kernel = createKernel({ filesystem: vfs as any }); - await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); - - const result = await kernel.exec('fd -t d . /project', {}); - const lines = parseLines(result.stdout); - // All entries should be directories - for (const line of lines) { - const stat = await vfs.stat(line); - expect(stat.isDirectory).toBe(true); - } - // Should include known directories (hidden skipped by default) - expect(lines).toContain('/project/src'); - expect(lines).toContain('/project/lib'); - expect(lines).toContain('/project/docs'); - }); - - it('returns no results for empty directory', async () => { - const vfs = await createTestVFS(); - kernel = createKernel({ filesystem: vfs as any }); - await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); - - const result = await kernel.exec('fd . /empty', {}); - expect(result.stdout.trim()).toBe(''); - }); - - it('returns empty output when no files match pattern', async () => { - const vfs = await createTestVFS(); - kernel = createKernel({ filesystem: vfs as any }); - await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); - - const result = await kernel.exec('fd zzzznonexistent /project', {}); - expect(result.stdout.trim()).toBe(''); - }); - - it('skips hidden files and directories by default', async () => { - const vfs = await createTestVFS(); - kernel = createKernel({ filesystem: vfs as any }); - await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); - - const result = await kernel.exec('fd . /project', {}); - const lines = parseLines(result.stdout); - // Hidden files/dirs should NOT appear - const hiddenEntries = lines.filter(l => { - const parts = l.split('/'); - return parts.some(p => p.startsWith('.') && p.length > 1); - }); - expect(hiddenEntries).toEqual([]); - }); - - it('includes hidden files with -H flag', async () => { - const vfs = await createTestVFS(); - kernel = createKernel({ filesystem: vfs as any }); - await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); - - const result = await kernel.exec('fd -H . /project', {}); - const lines = parseLines(result.stdout); - // Hidden items should now appear - expect(lines).toContain('/project/.gitignore'); - expect(lines).toContain('/project/.hidden'); - }); -}); diff --git a/registry/tests/wasmvm/git.test.ts b/registry/tests/wasmvm/git.test.ts deleted file mode 100644 index 93264c08ea..0000000000 --- a/registry/tests/wasmvm/git.test.ts +++ /dev/null @@ -1,547 +0,0 @@ -/** - * Integration tests for git command. - * - * Verifies init, add, commit, branch, checkout (with DWIM), plus local and - * smart-HTTP remote clone via kernel.exec() with real WASM binaries. - */ - -import { describe, it, expect, afterEach, beforeAll, afterAll, vi } from 'vitest'; -import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; -import { resolve, join } from 'node:path'; -import { tmpdir } from 'node:os'; -import { createServer, type Server as HttpServer } from 'node:http'; -import { spawn, spawnSync } from 'node:child_process'; -import { createWasmVmRuntime } from '../helpers.js'; -import { - allowAll, - COMMANDS_DIR, - createInMemoryFileSystem, - createKernel, - describeIf, - hasWasmBinaries, -} from '../helpers.js'; -import type { Kernel } from '../helpers.js'; - -vi.setConfig({ testTimeout: 30_000 }); - -/** Check git binary exists in addition to base WASM binaries */ -const hasGit = hasWasmBinaries && existsSync(resolve(COMMANDS_DIR, 'git')); -const hasHostGit = spawnSync('git', ['--version'], { stdio: 'ignore' }).status === 0; - -/** Create a kernel with a world-writable in-memory filesystem */ -async function createGitKernel() { - const vfs = createInMemoryFileSystem(); - // Make root and /tmp writable by all users (WASM processes run as non-root) - await (vfs as any).chmod('/', 0o1777); - await vfs.mkdir('/tmp', { recursive: true }); - await (vfs as any).chmod('/tmp', 0o1777); - const kernel = createKernel({ filesystem: vfs, syncFilesystemOnDispose: false }); - await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); - return { kernel, vfs, dispose: () => kernel.dispose() }; -} - -async function createGitKernelWithNet(loopbackExemptPorts: number[]) { - const vfs = createInMemoryFileSystem(); - await (vfs as any).chmod('/', 0o1777); - await vfs.mkdir('/tmp', { recursive: true }); - await (vfs as any).chmod('/tmp', 0o1777); - const kernel = createKernel({ - filesystem: vfs, - permissions: allowAll, - loopbackExemptPorts, - syncFilesystemOnDispose: false, - }); - await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); - return { kernel, vfs, dispose: () => kernel.dispose() }; -} - -function runHostGit(args: string[], cwd?: string) { - const result = spawnSync('git', args, { - cwd, - encoding: 'utf8', - }); - if (result.status !== 0) { - throw new Error( - `host git failed: git ${args.join(' ')}\nstdout: ${result.stdout}\nstderr: ${result.stderr}`, - ); - } -} - -/** Helper: run command and assert success */ -async function run(kernel: Kernel, cmd: string): Promise<{ stdout: string; stderr: string; exitCode: number }> { - const r = await kernel.exec(cmd); - if (r.exitCode !== 0) { - throw new Error(`Command failed (exit ${r.exitCode}): ${cmd}\nstdout: ${r.stdout}\nstderr: ${r.stderr}`); - } - return r; -} - -// TODO(P6): requires git WASM artifact, intentionally excluded from the fast registry-build gate. -describe.skip('git command', () => { - let kernel: Kernel; - let vfs: any; - let dispose: () => Promise; - - afterEach(async () => { - await dispose?.(); - }); - - it('init creates .git directory structure', async () => { - ({ kernel, vfs, dispose } = await createGitKernel()); - - const result = await run(kernel, 'git init /repo'); - expect(result.stdout).toContain('Initialized empty Git repository'); - - expect(await vfs.exists('/repo/.git/HEAD')).toBe(true); - expect(await vfs.exists('/repo/.git/objects')).toBe(true); - expect(await vfs.exists('/repo/.git/refs/heads')).toBe(true); - - const head = new TextDecoder().decode(await vfs.readFile('/repo/.git/HEAD')); - expect(head.trim()).toBe('ref: refs/heads/main'); - }); - - it('add + commit creates objects and updates ref', async () => { - ({ kernel, vfs, dispose } = await createGitKernel()); - - await run(kernel, 'git init /repo'); - await kernel.writeFile('/repo/hello.txt', 'hello world\n'); - await run(kernel, 'git -C /repo add hello.txt'); - await run(kernel, "git -C /repo commit -m 'first commit'"); - - expect(await vfs.exists('/repo/.git/refs/heads/main')).toBe(true); - }); - - it('branch lists branches with current marked', async () => { - ({ kernel, vfs, dispose } = await createGitKernel()); - - await run(kernel, 'git init /repo'); - await kernel.writeFile('/repo/file.txt', 'content\n'); - await run(kernel, 'git -C /repo add file.txt'); - await run(kernel, "git -C /repo commit -m 'init'"); - - const result = await run(kernel, 'git -C /repo branch'); - expect(result.stdout.trim()).toBe('* main'); - }); - - it('checkout -b creates a new branch', async () => { - ({ kernel, vfs, dispose } = await createGitKernel()); - - await run(kernel, 'git init /repo'); - await kernel.writeFile('/repo/file.txt', 'content\n'); - await run(kernel, 'git -C /repo add file.txt'); - await run(kernel, "git -C /repo commit -m 'init'"); - - await run(kernel, 'git -C /repo checkout -b feature'); - - const result = await run(kernel, 'git -C /repo branch'); - const lines = result.stdout.trim().split('\n').map((l: string) => l.trim()); - expect(lines).toContain('* feature'); - expect(lines).toContain('main'); - }); - - it('full quickstart scenario: init, commit, branch, clone, checkout', async () => { - ({ kernel, vfs, dispose } = await createGitKernel()); - - // Create origin repo - await run(kernel, 'git init /tmp/origin'); - await kernel.writeFile('/tmp/origin/README.md', '# demo repo\n'); - await run(kernel, 'git -C /tmp/origin add README.md'); - await run(kernel, "git -C /tmp/origin commit -m 'initial commit'"); - - // Check default branch - let r = await run(kernel, 'git -C /tmp/origin branch'); - expect(r.stdout.trim()).toBe('* main'); - - // Create feature branch with a new file - await run(kernel, 'git -C /tmp/origin checkout -b feature'); - await kernel.writeFile('/tmp/origin/feature.txt', 'checked out from feature\n'); - await run(kernel, 'git -C /tmp/origin add feature.txt'); - await run(kernel, "git -C /tmp/origin commit -m 'add feature file'"); - - // Switch back to main - await run(kernel, 'git -C /tmp/origin checkout main'); - - // Clone - await run(kernel, 'git clone /tmp/origin /tmp/clone'); - - // Clone should only show main branch initially - r = await run(kernel, 'git -C /tmp/clone branch'); - expect(r.stdout.trim()).toBe('* main'); - - // Checkout feature (DWIM from remote tracking) - await run(kernel, 'git -C /tmp/clone checkout feature'); - - // Now both branches should be listed - r = await run(kernel, 'git -C /tmp/clone branch'); - const branches = r.stdout.trim().split('\n').map((l: string) => l.trim()); - expect(branches).toContain('* feature'); - expect(branches).toContain('main'); - - // Verify feature file exists in clone - const featureContent = new TextDecoder().decode(await vfs.readFile('/tmp/clone/feature.txt')); - expect(featureContent).toBe('checked out from feature\n'); - - // Verify README exists too - const readmeContent = new TextDecoder().decode(await vfs.readFile('/tmp/clone/README.md')); - expect(readmeContent).toBe('# demo repo\n'); - }); - - it('clone without an explicit destination uses the source basename', async () => { - ({ kernel, vfs, dispose } = await createGitKernel()); - - await run(kernel, 'git init /tmp/origin'); - await kernel.writeFile('/tmp/origin/README.md', 'default destination\n'); - await run(kernel, 'git -C /tmp/origin add README.md'); - await run(kernel, "git -C /tmp/origin commit -m 'seed'"); - - await run(kernel, 'mkdir -p /work'); - await run(kernel, 'git -C /work clone /tmp/origin'); - - expect(await vfs.exists('/work/origin/.git/HEAD')).toBe(true); - const readmeContent = new TextDecoder().decode(await vfs.readFile('/work/origin/README.md')); - expect(readmeContent).toBe('default destination\n'); - }); - - it('clone without an explicit destination strips a trailing .git suffix', async () => { - ({ kernel, vfs, dispose } = await createGitKernel()); - - await run(kernel, 'git init /tmp/origin.git'); - await kernel.writeFile('/tmp/origin.git/README.md', 'suffix destination\n'); - await run(kernel, 'git -C /tmp/origin.git add README.md'); - await run(kernel, "git -C /tmp/origin.git commit -m 'seed'"); - - await run(kernel, 'mkdir -p /work'); - await run(kernel, 'git -C /work clone /tmp/origin.git'); - - expect(await vfs.exists('/work/origin/.git/HEAD')).toBe(true); - const readmeContent = new TextDecoder().decode(await vfs.readFile('/work/origin/README.md')); - expect(readmeContent).toBe('suffix destination\n'); - }); - - it('clone into an existing empty destination directory succeeds', async () => { - ({ kernel, vfs, dispose } = await createGitKernel()); - - await run(kernel, 'git init /tmp/origin'); - await kernel.writeFile('/tmp/origin/README.md', 'empty destination\n'); - await run(kernel, 'git -C /tmp/origin add README.md'); - await run(kernel, "git -C /tmp/origin commit -m 'seed'"); - - await run(kernel, 'mkdir -p /tmp/clone'); - await run(kernel, 'git clone /tmp/origin /tmp/clone'); - - expect(await vfs.exists('/tmp/clone/.git/HEAD')).toBe(true); - const readmeContent = new TextDecoder().decode(await vfs.readFile('/tmp/clone/README.md')); - expect(readmeContent).toBe('empty destination\n'); - }); - - it('clone rejects a non-empty destination directory', async () => { - ({ kernel, vfs, dispose } = await createGitKernel()); - - await run(kernel, 'git init /tmp/origin'); - await kernel.writeFile('/tmp/origin/README.md', 'origin\n'); - await run(kernel, 'git -C /tmp/origin add README.md'); - await run(kernel, "git -C /tmp/origin commit -m 'seed'"); - - await run(kernel, 'mkdir -p /tmp/clone'); - await kernel.writeFile('/tmp/clone/existing.txt', 'keep me\n'); - - const result = await kernel.exec('git clone /tmp/origin /tmp/clone'); - expect(result.exitCode).not.toBe(0); - expect(result.stderr).toMatch(/already exists|not an empty directory|destination/i); - - const existing = new TextDecoder().decode(await vfs.readFile('/tmp/clone/existing.txt')); - expect(existing).toBe('keep me\n'); - expect(await vfs.exists('/tmp/clone/.git')).toBe(false); - }); - - it('clone of a missing repository fails without leaving a partial destination', async () => { - ({ kernel, vfs, dispose } = await createGitKernel()); - - const result = await kernel.exec('git clone /tmp/missing /tmp/clone'); - expect(result.exitCode).not.toBe(0); - expect(result.stderr).toMatch(/not a git repository|missing|no such file|fatal/i); - expect(await vfs.exists('/tmp/clone')).toBe(false); - }); - - it('clone of an empty repository succeeds and leaves an empty worktree', async () => { - ({ kernel, vfs, dispose } = await createGitKernel()); - - await run(kernel, 'git init /tmp/origin'); - await run(kernel, 'git clone /tmp/origin /tmp/clone'); - - const head = new TextDecoder().decode(await vfs.readFile('/tmp/clone/.git/HEAD')); - expect(head.trim()).toBe('ref: refs/heads/main'); - expect(await vfs.exists('/tmp/clone/.git/config')).toBe(true); - expect(await vfs.exists('/tmp/clone/.git/refs/heads/main')).toBe(false); - expect(await vfs.exists('/tmp/clone/README.md')).toBe(false); - }); - - it('clone preserves nested directory trees', async () => { - ({ kernel, vfs, dispose } = await createGitKernel()); - - await run(kernel, 'git init /tmp/origin'); - await run(kernel, 'mkdir -p /tmp/origin/src/nested'); - await kernel.writeFile('/tmp/origin/src/nested/file.txt', 'nested payload\n'); - await kernel.writeFile('/tmp/origin/src/root.txt', 'root payload\n'); - await run(kernel, 'git -C /tmp/origin add src/nested/file.txt src/root.txt'); - await run(kernel, "git -C /tmp/origin commit -m 'nested tree'"); - - await run(kernel, 'git clone /tmp/origin /tmp/clone'); - - const nested = new TextDecoder().decode(await vfs.readFile('/tmp/clone/src/nested/file.txt')); - const root = new TextDecoder().decode(await vfs.readFile('/tmp/clone/src/root.txt')); - expect(nested).toBe('nested payload\n'); - expect(root).toBe('root payload\n'); - }); - - it('clone honors the source default branch when HEAD is not main', async () => { - ({ kernel, vfs, dispose } = await createGitKernel()); - - await run(kernel, 'git init /tmp/origin'); - await kernel.writeFile('/tmp/origin/README.md', 'main branch\n'); - await run(kernel, 'git -C /tmp/origin add README.md'); - await run(kernel, "git -C /tmp/origin commit -m 'main'"); - - await run(kernel, 'git -C /tmp/origin checkout -b trunk'); - await kernel.writeFile('/tmp/origin/trunk.txt', 'trunk branch\n'); - await run(kernel, 'git -C /tmp/origin add trunk.txt'); - await run(kernel, "git -C /tmp/origin commit -m 'trunk'"); - - await run(kernel, 'git clone /tmp/origin /tmp/clone'); - - const head = new TextDecoder().decode(await vfs.readFile('/tmp/clone/.git/HEAD')); - expect(head.trim()).toBe('ref: refs/heads/trunk'); - expect(await vfs.exists('/tmp/clone/.git/refs/heads/trunk')).toBe(true); - const trunk = new TextDecoder().decode(await vfs.readFile('/tmp/clone/trunk.txt')); - expect(trunk).toBe('trunk branch\n'); - }); - - it('clone copies nested branch refs and checkout DWIM works for branch names with slashes', async () => { - ({ kernel, vfs, dispose } = await createGitKernel()); - - await run(kernel, 'git init /tmp/origin'); - await kernel.writeFile('/tmp/origin/README.md', '# demo repo\n'); - await run(kernel, 'git -C /tmp/origin add README.md'); - await run(kernel, "git -C /tmp/origin commit -m 'initial commit'"); - - await run(kernel, 'git -C /tmp/origin checkout -b feature/deep'); - await kernel.writeFile('/tmp/origin/feature.txt', 'nested branch payload\n'); - await run(kernel, 'git -C /tmp/origin add feature.txt'); - await run(kernel, "git -C /tmp/origin commit -m 'nested branch'"); - await run(kernel, 'git -C /tmp/origin checkout main'); - - await run(kernel, 'git clone /tmp/origin /tmp/clone'); - - expect(await vfs.exists('/tmp/clone/.git/refs/remotes/origin/feature/deep')).toBe(true); - - await run(kernel, 'git -C /tmp/clone checkout feature/deep'); - const featureContent = new TextDecoder().decode(await vfs.readFile('/tmp/clone/feature.txt')); - expect(featureContent).toBe('nested branch payload\n'); - const head = new TextDecoder().decode(await vfs.readFile('/tmp/clone/.git/HEAD')); - expect(head.trim()).toBe('ref: refs/heads/feature/deep'); - }); - - it('clone works with relative source and destination paths', async () => { - ({ kernel, vfs, dispose } = await createGitKernel()); - - await run(kernel, 'mkdir -p /tmp/work'); - await run(kernel, 'git init /tmp/work/origin'); - await kernel.writeFile('/tmp/work/origin/README.md', 'relative clone\n'); - await run(kernel, 'git -C /tmp/work/origin add README.md'); - await run(kernel, "git -C /tmp/work/origin commit -m 'seed'"); - - await run(kernel, 'git -C /tmp/work clone ./origin ./clone'); - - expect(await vfs.exists('/tmp/work/clone/.git/HEAD')).toBe(true); - const readmeContent = new TextDecoder().decode(await vfs.readFile('/tmp/work/clone/README.md')); - expect(readmeContent).toBe('relative clone\n'); - }); - - it('push fails with a typed unsupported-subcommand error', async () => { - ({ kernel, dispose } = await createGitKernel()); - - await run(kernel, 'git init /tmp/repo'); - - const result = await kernel.exec('git -C /tmp/repo push origin main'); - expect(result.exitCode).not.toBe(0); - expect(result.stderr).toContain('GitSubcommandUnsupported'); - expect(result.stderr).toContain('git push'); - expect(result.stderr).toContain('registry/native/crates/libs/git/README.md'); - }); - - it('clone rejects SSH-style remotes with a typed unsupported-subcommand error', async () => { - ({ kernel, dispose } = await createGitKernel()); - - const result = await kernel.exec('git clone git@github.com:rivet-dev/agentos.git /tmp/clone'); - expect(result.exitCode).not.toBe(0); - expect(result.stderr).toContain('GitSubcommandUnsupported'); - expect(result.stderr).toContain('git clone'); - expect(result.stderr).toContain('SSH'); - expect(result.stderr).toContain('registry/native/crates/libs/git/README.md'); - }); - - it('clone rejects authenticated HTTPS remotes loudly instead of attempting a broken auth flow', async () => { - ({ kernel, dispose } = await createGitKernel()); - - const result = await kernel.exec( - 'git clone https://private@example.com/owner/repo.git /tmp/clone', - { env: { GIT_AUTH_TOKEN: 'test-token' } }, - ); - expect(result.exitCode).not.toBe(0); - expect(result.stderr).toContain('GitSubcommandUnsupported'); - expect(result.stderr).toContain('git clone'); - expect(result.stderr).toContain('authenticated HTTP(S) remotes'); - expect(result.stderr).toContain('registry/native/crates/libs/git/README.md'); - }); - - describeIf(hasHostGit, 'remote clone over smart HTTP', () => { - let repoRoot: string; - let httpServer: HttpServer; - let httpPort: number; - - beforeAll(async () => { - repoRoot = mkdtempSync(join(tmpdir(), 'agentos-git-http-')); - const worktree = join(repoRoot, 'worktree'); - const origin = join(repoRoot, 'origin.git'); - - runHostGit(['-c', 'init.defaultBranch=main', 'init', worktree]); - writeFileSync(join(worktree, 'README.md'), 'remote smart clone\n'); - runHostGit(['-C', worktree, 'add', 'README.md']); - runHostGit([ - '-C', worktree, - '-c', 'user.name=secure-exec', - '-c', 'user.email=agent@example.com', - 'commit', - '-m', - 'seed', - ]); - - runHostGit(['-C', worktree, 'checkout', '-b', 'feature/deep']); - writeFileSync(join(worktree, 'feature.txt'), 'remote branch payload\n'); - runHostGit(['-C', worktree, 'add', 'feature.txt']); - runHostGit([ - '-C', worktree, - '-c', 'user.name=secure-exec', - '-c', 'user.email=agent@example.com', - 'commit', - '-m', - 'feature branch', - ]); - - runHostGit(['-C', worktree, 'checkout', 'main']); - runHostGit(['clone', '--bare', worktree, origin]); - runHostGit(['-C', origin, 'repack', '-a', '-d', '-f', '--depth=50', '--window=50']); - - httpServer = createServer((req, res) => { - const url = new URL(req.url ?? '/', 'http://127.0.0.1'); - const bodyChunks: Buffer[] = []; - - req.on('data', (chunk) => { - bodyChunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); - }); - - req.on('end', () => { - const requestBody = Buffer.concat(bodyChunks); - const gitProtocol = req.headers['git-protocol']; - const env = { - ...process.env, - GIT_HTTP_EXPORT_ALL: '1', - GIT_PROJECT_ROOT: repoRoot, - PATH_INFO: url.pathname, - QUERY_STRING: url.search.startsWith('?') ? url.search.slice(1) : url.search, - REQUEST_METHOD: req.method ?? 'GET', - CONTENT_TYPE: String(req.headers['content-type'] ?? ''), - CONTENT_LENGTH: String(requestBody.length), - REMOTE_ADDR: '127.0.0.1', - GIT_PROTOCOL: typeof gitProtocol === 'string' ? gitProtocol : '', - HTTP_GIT_PROTOCOL: typeof gitProtocol === 'string' ? gitProtocol : '', - }; - - const child = spawn('git', ['http-backend'], { env }); - const stdout: Buffer[] = []; - const stderr: Buffer[] = []; - - child.stdout.on('data', (chunk) => { - stdout.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); - }); - child.stderr.on('data', (chunk) => { - stderr.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); - }); - child.on('error', (error) => { - res.writeHead(500, { 'Content-Type': 'text/plain' }); - res.end(String(error)); - }); - child.on('close', (code) => { - const output = Buffer.concat(stdout); - const headerSep = output.indexOf(Buffer.from('\r\n\r\n')); - const altSep = output.indexOf(Buffer.from('\n\n')); - const sepIndex = headerSep >= 0 ? headerSep : altSep; - const sepLen = headerSep >= 0 ? 4 : altSep >= 0 ? 2 : 0; - - if (code !== 0 && sepIndex === -1) { - res.writeHead(500, { 'Content-Type': 'text/plain' }); - res.end(Buffer.concat(stderr)); - return; - } - - if (sepIndex === -1) { - res.writeHead(500, { 'Content-Type': 'text/plain' }); - res.end(output); - return; - } - - const headerText = output.subarray(0, sepIndex).toString('utf8'); - const responseBody = output.subarray(sepIndex + sepLen); - let status = 200; - const headers: Record = {}; - - for (const line of headerText.split(/\r?\n/)) { - if (!line) continue; - const colon = line.indexOf(':'); - if (colon === -1) continue; - const name = line.slice(0, colon); - const value = line.slice(colon + 1).trim(); - if (name.toLowerCase() === 'status') { - status = Number.parseInt(value, 10) || 200; - } else { - headers[name] = value; - } - } - - res.writeHead(status, headers); - res.end(responseBody); - }); - - child.stdin.end(requestBody); - }); - }); - - await new Promise((resolveListen) => { - httpServer.listen(0, '127.0.0.1', resolveListen); - }); - httpPort = (httpServer.address() as import('node:net').AddressInfo).port; - }); - - afterAll(async () => { - await new Promise((resolveClose) => httpServer.close(() => resolveClose())); - rmSync(repoRoot, { recursive: true, force: true }); - }); - - it('clone fetches refs and worktree contents from a smart HTTP remote', async () => { - ({ kernel, vfs, dispose } = await createGitKernelWithNet([httpPort])); - - await run(kernel, `git clone http://127.0.0.1:${httpPort}/origin.git /tmp/clone`); - - const head = new TextDecoder().decode(await kernel.readFile('/tmp/clone/.git/HEAD')); - expect(head.trim()).toBe('ref: refs/heads/main'); - - const readme = new TextDecoder().decode(await kernel.readFile('/tmp/clone/README.md')); - expect(readme).toBe('remote smart clone\n'); - expect(await kernel.exists('/tmp/clone/.git/refs/remotes/origin/feature/deep')).toBe(true); - - await run(kernel, 'git -C /tmp/clone checkout feature/deep'); - const feature = new TextDecoder().decode(await kernel.readFile('/tmp/clone/feature.txt')); - expect(feature).toBe('remote branch payload\n'); - }); - }); -}); diff --git a/registry/tests/wasmvm/net-server.test.ts b/registry/tests/wasmvm/net-server.test.ts deleted file mode 100644 index c943363629..0000000000 --- a/registry/tests/wasmvm/net-server.test.ts +++ /dev/null @@ -1,196 +0,0 @@ -/** - * Integration test for WasmVM TCP server sockets. - * - * Spawns the tcp_server C program as WASM (bind → listen → accept → recv → - * send "pong" → close), connects to it from the kernel as a client socket, - * and verifies the full data exchange via loopback routing. - */ - -import { describe, it, expect, beforeEach, afterEach } from 'vitest'; -import { createWasmVmRuntime } from '../helpers.js'; -import { - AF_INET, - COMMANDS_DIR, - C_BUILD_DIR, - createKernel, - describeIf, - hasWasmBinaries, - SOCK_STREAM, -} from '../helpers.js'; -import type { Kernel } from '../helpers.js'; -import { existsSync } from 'node:fs'; -import { join } from 'node:path'; - -const hasCWasmBinaries = existsSync(join(C_BUILD_DIR, 'tcp_server')); - -function skipReason(): string | false { - if (!hasWasmBinaries) return 'WASM binaries not built (run make wasm in native/wasmvm/)'; - if (!hasCWasmBinaries) return 'tcp_server WASM binary not built (run make -C native/wasmvm/c sysroot && make -C native/wasmvm/c programs)'; - return false; -} - -// Minimal in-memory VFS (same as c-parity) -class SimpleVFS { - private files = new Map(); - private dirs = new Set(['/']); - private symlinks = new Map(); - - async readFile(path: string): Promise { - const data = this.files.get(path); - if (!data) throw new Error(`ENOENT: ${path}`); - return data; - } - async readTextFile(path: string): Promise { - return new TextDecoder().decode(await this.readFile(path)); - } - async pread(path: string, offset: number, length: number): Promise { - const data = await this.readFile(path); - return data.slice(offset, offset + length); - } - async readDir(path: string): Promise { - const prefix = path === '/' ? '/' : path + '/'; - const entries: string[] = []; - for (const p of [...this.files.keys(), ...this.dirs]) { - if (p !== path && p.startsWith(prefix)) { - const rest = p.slice(prefix.length); - if (!rest.includes('/')) entries.push(rest); - } - } - return entries; - } - async readDirWithTypes(path: string) { - return (await this.readDir(path)).map((name) => ({ - name, - isDirectory: this.dirs.has(path === '/' ? `/${name}` : `${path}/${name}`), - })); - } - async writeFile(path: string, content: string | Uint8Array): Promise { - const data = typeof content === 'string' ? new TextEncoder().encode(content) : content; - this.files.set(path, new Uint8Array(data)); - const parts = path.split('/').filter(Boolean); - for (let i = 1; i < parts.length; i++) { - this.dirs.add('/' + parts.slice(0, i).join('/')); - } - } - async createDir(path: string) { this.dirs.add(path); } - async mkdir(path: string, _options?: { recursive?: boolean }) { this.dirs.add(path); } - async exists(path: string): Promise { - return this.files.has(path) || this.dirs.has(path) || this.symlinks.has(path); - } - async stat(path: string) { - const isDir = this.dirs.has(path); - const isSymlink = this.symlinks.has(path); - const data = this.files.get(path); - if (!isDir && !isSymlink && !data) throw new Error(`ENOENT: ${path}`); - return { - mode: isSymlink ? 0o120777 : (isDir ? 0o40755 : 0o100644), - size: data?.length ?? 0, - isDirectory: isDir, - isSymbolicLink: isSymlink, - atimeMs: Date.now(), - mtimeMs: Date.now(), - ctimeMs: Date.now(), - birthtimeMs: Date.now(), - ino: 0, - nlink: 1, - uid: 1000, - gid: 1000, - }; - } - async chmod() {} - async rename(from: string, to: string) { - const data = this.files.get(from); - if (data) { this.files.set(to, data); this.files.delete(from); } - } - async unlink(path: string) { this.files.delete(path); this.symlinks.delete(path); } - async rmdir(path: string) { this.dirs.delete(path); } - async symlink(target: string, linkPath: string) { - this.symlinks.set(linkPath, target); - const parts = linkPath.split('/').filter(Boolean); - for (let i = 1; i < parts.length; i++) { - this.dirs.add('/' + parts.slice(0, i).join('/')); - } - } - async readlink(path: string): Promise { - const target = this.symlinks.get(path); - if (!target) throw new Error(`EINVAL: ${path}`); - return target; - } -} - -// Wait for a kernel socket listener on the given port (poll with timeout) -async function waitForListener( - kernel: Kernel, - port: number, - timeoutMs = 10_000, -): Promise { - const deadline = Date.now() + timeoutMs; - while (Date.now() < deadline) { - const listener = kernel.socketTable.findListener({ host: '0.0.0.0', port }); - if (listener) return; - await new Promise((r) => setTimeout(r, 20)); - } - throw new Error(`Timed out waiting for listener on port ${port}`); -} - -const TEST_PORT = 9876; -const CLIENT_PID = 999; // Fake PID for test-side client sockets - -describeIf(!skipReason(), 'WasmVM TCP server integration', { timeout: 30_000 }, () => { - let kernel: Kernel; - let vfs: SimpleVFS; - - beforeEach(async () => { - vfs = new SimpleVFS(); - kernel = createKernel({ filesystem: vfs as any }); - await kernel.mount(createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] })); - }); - - afterEach(async () => { - await kernel?.dispose(); - }); - - it('tcp_server: accept connection, recv data, send pong', async () => { - // Start the WASM TCP server (blocks on accept until we connect) - const execPromise = kernel.exec(`tcp_server ${TEST_PORT}`); - - // Wait for the server to finish bind+listen - await waitForListener(kernel, TEST_PORT); - - // Create a client socket and connect via loopback - const st = kernel.socketTable; - const clientId = st.create(AF_INET, SOCK_STREAM, 0, CLIENT_PID); - await st.connect(clientId, { host: '127.0.0.1', port: TEST_PORT }); - - // Send "ping" to the server - const encoder = new TextEncoder(); - st.send(clientId, encoder.encode('ping')); - - // Wait for the server to process and send its reply - const decoder = new TextDecoder(); - let reply = ''; - const recvDeadline = Date.now() + 10_000; - while (Date.now() < recvDeadline) { - const chunk = st.recv(clientId, 256); - if (chunk && chunk.length > 0) { - reply += decoder.decode(chunk); - break; - } - // No data yet — yield to let the WASM worker process - await new Promise((r) => setTimeout(r, 20)); - } - - expect(reply).toBe('pong'); - - // Close client socket - st.close(clientId, CLIENT_PID); - - // Wait for exec to complete (server exits after handling one connection) - const result = await execPromise; - - expect(result.stdout).toContain('listening on port 9876'); - expect(result.stdout).toContain('received: ping'); - expect(result.stdout).toContain('sent: 4'); - expect(result.exitCode).toBe(0); - }); -}); diff --git a/registry/tests/wasmvm/net-udp.test.ts b/registry/tests/wasmvm/net-udp.test.ts deleted file mode 100644 index 98e6b93696..0000000000 --- a/registry/tests/wasmvm/net-udp.test.ts +++ /dev/null @@ -1,234 +0,0 @@ -/** - * Integration test for WasmVM UDP sockets. - * - * Spawns the udp_echo C program as WASM (bind → recvfrom → sendto echo → close), - * sends datagrams from a kernel client socket, and verifies the echo response - * and message boundary preservation. - */ - -import { describe, it, expect, beforeEach, afterEach } from 'vitest'; -import { createWasmVmRuntime } from '../helpers.js'; -import { - AF_INET, - COMMANDS_DIR, - C_BUILD_DIR, - createKernel, - describeIf, - hasWasmBinaries, - SOCK_DGRAM, -} from '../helpers.js'; -import type { Kernel } from '../helpers.js'; -import { existsSync } from 'node:fs'; -import { join } from 'node:path'; - -const hasCWasmBinaries = existsSync(join(C_BUILD_DIR, 'udp_echo')); - -function skipReason(): string | false { - if (!hasWasmBinaries) return 'WASM binaries not built (run make wasm in native/wasmvm/)'; - if (!hasCWasmBinaries) return 'udp_echo WASM binary not built (run make -C native/wasmvm/c sysroot && make -C native/wasmvm/c programs)'; - return false; -} - -// Minimal in-memory VFS (same as net-server) -class SimpleVFS { - private files = new Map(); - private dirs = new Set(['/']); - private symlinks = new Map(); - - async readFile(path: string): Promise { - const data = this.files.get(path); - if (!data) throw new Error(`ENOENT: ${path}`); - return data; - } - async readTextFile(path: string): Promise { - return new TextDecoder().decode(await this.readFile(path)); - } - async pread(path: string, offset: number, length: number): Promise { - const data = await this.readFile(path); - return data.slice(offset, offset + length); - } - async readDir(path: string): Promise { - const prefix = path === '/' ? '/' : path + '/'; - const entries: string[] = []; - for (const p of [...this.files.keys(), ...this.dirs]) { - if (p !== path && p.startsWith(prefix)) { - const rest = p.slice(prefix.length); - if (!rest.includes('/')) entries.push(rest); - } - } - return entries; - } - async readDirWithTypes(path: string) { - return (await this.readDir(path)).map((name) => ({ - name, - isDirectory: this.dirs.has(path === '/' ? `/${name}` : `${path}/${name}`), - })); - } - async writeFile(path: string, content: string | Uint8Array): Promise { - const data = typeof content === 'string' ? new TextEncoder().encode(content) : content; - this.files.set(path, new Uint8Array(data)); - const parts = path.split('/').filter(Boolean); - for (let i = 1; i < parts.length; i++) { - this.dirs.add('/' + parts.slice(0, i).join('/')); - } - } - async createDir(path: string) { this.dirs.add(path); } - async mkdir(path: string, _options?: { recursive?: boolean }) { this.dirs.add(path); } - async exists(path: string): Promise { - return this.files.has(path) || this.dirs.has(path) || this.symlinks.has(path); - } - async stat(path: string) { - const isDir = this.dirs.has(path); - const isSymlink = this.symlinks.has(path); - const data = this.files.get(path); - if (!isDir && !isSymlink && !data) throw new Error(`ENOENT: ${path}`); - return { - mode: isSymlink ? 0o120777 : (isDir ? 0o40755 : 0o100644), - size: data?.length ?? 0, - isDirectory: isDir, - isSymbolicLink: isSymlink, - atimeMs: Date.now(), - mtimeMs: Date.now(), - ctimeMs: Date.now(), - birthtimeMs: Date.now(), - ino: 0, - nlink: 1, - uid: 1000, - gid: 1000, - }; - } - async chmod() {} - async rename(from: string, to: string) { - const data = this.files.get(from); - if (data) { this.files.set(to, data); this.files.delete(from); } - } - async unlink(path: string) { this.files.delete(path); this.symlinks.delete(path); } - async rmdir(path: string) { this.dirs.delete(path); } - async symlink(target: string, linkPath: string) { - this.symlinks.set(linkPath, target); - const parts = linkPath.split('/').filter(Boolean); - for (let i = 1; i < parts.length; i++) { - this.dirs.add('/' + parts.slice(0, i).join('/')); - } - } - async readlink(path: string): Promise { - const target = this.symlinks.get(path); - if (!target) throw new Error(`EINVAL: ${path}`); - return target; - } -} - -// Wait for a kernel UDP binding on the given port (poll with timeout) -async function waitForUdpBinding( - kernel: Kernel, - port: number, - timeoutMs = 10_000, -): Promise { - const deadline = Date.now() + timeoutMs; - while (Date.now() < deadline) { - const bound = kernel.socketTable.findBoundUdp({ host: '0.0.0.0', port }); - if (bound) return; - await new Promise((r) => setTimeout(r, 20)); - } - throw new Error(`Timed out waiting for UDP binding on port ${port}`); -} - -const TEST_PORT = 9877; -const CLIENT_PID = 999; // Fake PID for test-side client sockets - -describeIf(!skipReason(), 'WasmVM UDP integration', { timeout: 30_000 }, () => { - let kernel: Kernel; - let vfs: SimpleVFS; - - beforeEach(async () => { - vfs = new SimpleVFS(); - kernel = createKernel({ filesystem: vfs as any }); - await kernel.mount(createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] })); - }); - - afterEach(async () => { - await kernel?.dispose(); - }); - - it('udp_echo: recv datagram and echo it back', async () => { - // Start the WASM UDP echo server (blocks on recvfrom until we send) - const execPromise = kernel.exec(`udp_echo ${TEST_PORT}`); - - // Wait for the server to finish bind - await waitForUdpBinding(kernel, TEST_PORT); - - // Create a client UDP socket and bind to an ephemeral port - const st = kernel.socketTable; - const clientId = st.create(AF_INET, SOCK_DGRAM, 0, CLIENT_PID); - await st.bind(clientId, { host: '127.0.0.1', port: 0 }); - - // Send "hello" to the echo server - const encoder = new TextEncoder(); - st.sendTo(clientId, encoder.encode('hello'), 0, { host: '127.0.0.1', port: TEST_PORT }); - - // Wait for the echo response - const decoder = new TextDecoder(); - let reply = ''; - const recvDeadline = Date.now() + 10_000; - while (Date.now() < recvDeadline) { - const result = st.recvFrom(clientId, 1024); - if (result && result.data.length > 0) { - reply = decoder.decode(result.data); - break; - } - await new Promise((r) => setTimeout(r, 20)); - } - - expect(reply).toBe('hello'); - - // Close client socket - st.close(clientId, CLIENT_PID); - - // Wait for exec to complete (server exits after echoing one datagram) - const result = await execPromise; - - expect(result.stdout).toContain('listening on port 9877'); - expect(result.stdout).toContain('received: hello'); - expect(result.stdout).toContain('echoed: 5'); - expect(result.exitCode).toBe(0); - }); - - it('udp_echo: message boundaries are preserved', async () => { - // Start the WASM UDP echo server - const execPromise = kernel.exec(`udp_echo ${TEST_PORT + 1}`); - - // Wait for the server to finish bind - await waitForUdpBinding(kernel, TEST_PORT + 1); - - // Create a client UDP socket - const st = kernel.socketTable; - const clientId = st.create(AF_INET, SOCK_DGRAM, 0, CLIENT_PID); - await st.bind(clientId, { host: '127.0.0.1', port: 0 }); - - // Send a message — the echo server echoes exactly one datagram - const encoder = new TextEncoder(); - const msg = 'boundary-test-message'; - st.sendTo(clientId, encoder.encode(msg), 0, { host: '127.0.0.1', port: TEST_PORT + 1 }); - - // Receive the echo — it must be the exact message (not fragmented/merged) - const decoder = new TextDecoder(); - let reply = ''; - const recvDeadline = Date.now() + 10_000; - while (Date.now() < recvDeadline) { - const result = st.recvFrom(clientId, 1024); - if (result && result.data.length > 0) { - reply = decoder.decode(result.data); - break; - } - await new Promise((r) => setTimeout(r, 20)); - } - - // Message boundary preserved: exact content, exact length - expect(reply).toBe(msg); - expect(reply.length).toBe(msg.length); - - st.close(clientId, CLIENT_PID); - const result = await execPromise; - expect(result.exitCode).toBe(0); - }); -}); diff --git a/registry/tests/wasmvm/net-unix.test.ts b/registry/tests/wasmvm/net-unix.test.ts deleted file mode 100644 index b4bac046ba..0000000000 --- a/registry/tests/wasmvm/net-unix.test.ts +++ /dev/null @@ -1,197 +0,0 @@ -/** - * Integration test for WasmVM Unix domain sockets. - * - * Spawns the unix_socket C program as WASM (socket(AF_UNIX) → bind → listen → - * accept → recv → send "pong" → close), connects from the kernel as a client, - * and verifies data exchange via in-kernel loopback routing. - */ - -import { describe, it, expect, beforeEach, afterEach } from 'vitest'; -import { createWasmVmRuntime } from '../helpers.js'; -import { - AF_UNIX, - COMMANDS_DIR, - C_BUILD_DIR, - createKernel, - describeIf, - hasWasmBinaries, - SOCK_STREAM, -} from '../helpers.js'; -import type { Kernel } from '../helpers.js'; -import { existsSync } from 'node:fs'; -import { join } from 'node:path'; - -const hasCWasmBinaries = existsSync(join(C_BUILD_DIR, 'unix_socket')); - -function skipReason(): string | false { - if (!hasWasmBinaries) return 'WASM binaries not built (run make wasm in native/wasmvm/)'; - if (!hasCWasmBinaries) return 'unix_socket WASM binary not built (run make -C native/wasmvm/c sysroot && make -C native/wasmvm/c programs)'; - return false; -} - -// Minimal in-memory VFS (same as net-server) -class SimpleVFS { - private files = new Map(); - private dirs = new Set(['/']); - private symlinks = new Map(); - - async readFile(path: string): Promise { - const data = this.files.get(path); - if (!data) throw new Error(`ENOENT: ${path}`); - return data; - } - async readTextFile(path: string): Promise { - return new TextDecoder().decode(await this.readFile(path)); - } - async pread(path: string, offset: number, length: number): Promise { - const data = await this.readFile(path); - return data.slice(offset, offset + length); - } - async readDir(path: string): Promise { - const prefix = path === '/' ? '/' : path + '/'; - const entries: string[] = []; - for (const p of [...this.files.keys(), ...this.dirs]) { - if (p !== path && p.startsWith(prefix)) { - const rest = p.slice(prefix.length); - if (!rest.includes('/')) entries.push(rest); - } - } - return entries; - } - async readDirWithTypes(path: string) { - return (await this.readDir(path)).map((name) => ({ - name, - isDirectory: this.dirs.has(path === '/' ? `/${name}` : `${path}/${name}`), - })); - } - async writeFile(path: string, content: string | Uint8Array): Promise { - const data = typeof content === 'string' ? new TextEncoder().encode(content) : content; - this.files.set(path, new Uint8Array(data)); - const parts = path.split('/').filter(Boolean); - for (let i = 1; i < parts.length; i++) { - this.dirs.add('/' + parts.slice(0, i).join('/')); - } - } - async createDir(path: string) { this.dirs.add(path); } - async mkdir(path: string, _options?: { recursive?: boolean }) { this.dirs.add(path); } - async exists(path: string): Promise { - return this.files.has(path) || this.dirs.has(path) || this.symlinks.has(path); - } - async stat(path: string) { - const isDir = this.dirs.has(path); - const isSymlink = this.symlinks.has(path); - const data = this.files.get(path); - if (!isDir && !isSymlink && !data) throw new Error(`ENOENT: ${path}`); - return { - mode: isSymlink ? 0o120777 : (isDir ? 0o40755 : 0o100644), - size: data?.length ?? 0, - isDirectory: isDir, - isSymbolicLink: isSymlink, - atimeMs: Date.now(), - mtimeMs: Date.now(), - ctimeMs: Date.now(), - birthtimeMs: Date.now(), - ino: 0, - nlink: 1, - uid: 1000, - gid: 1000, - }; - } - async chmod() {} - async rename(from: string, to: string) { - const data = this.files.get(from); - if (data) { this.files.set(to, data); this.files.delete(from); } - } - async unlink(path: string) { this.files.delete(path); this.symlinks.delete(path); } - async rmdir(path: string) { this.dirs.delete(path); } - async symlink(target: string, linkPath: string) { - this.symlinks.set(linkPath, target); - const parts = linkPath.split('/').filter(Boolean); - for (let i = 1; i < parts.length; i++) { - this.dirs.add('/' + parts.slice(0, i).join('/')); - } - } - async readlink(path: string): Promise { - const target = this.symlinks.get(path); - if (!target) throw new Error(`EINVAL: ${path}`); - return target; - } -} - -// Wait for a kernel Unix domain socket listener at the given path (poll with timeout) -async function waitForUnixListener( - kernel: Kernel, - path: string, - timeoutMs = 10_000, -): Promise { - const deadline = Date.now() + timeoutMs; - while (Date.now() < deadline) { - const listener = kernel.socketTable.findListener({ path }); - if (listener) return; - await new Promise((r) => setTimeout(r, 20)); - } - throw new Error(`Timed out waiting for Unix listener on ${path}`); -} - -const SOCK_PATH = '/tmp/test.sock'; -const CLIENT_PID = 999; // Fake PID for test-side client sockets - -describeIf(!skipReason(), 'WasmVM Unix domain socket integration', { timeout: 30_000 }, () => { - let kernel: Kernel; - let vfs: SimpleVFS; - - beforeEach(async () => { - vfs = new SimpleVFS(); - // Create /tmp so the socket file can be created - await vfs.mkdir('/tmp'); - kernel = createKernel({ filesystem: vfs as any }); - await kernel.mount(createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] })); - }); - - afterEach(async () => { - await kernel?.dispose(); - }); - - it('unix_socket: accept connection, recv data, send pong', async () => { - // Start the WASM Unix socket server (blocks on accept until we connect) - const execPromise = kernel.exec(`unix_socket ${SOCK_PATH}`); - - // Wait for the server to finish bind+listen - await waitForUnixListener(kernel, SOCK_PATH); - - // Create a client socket and connect via loopback - const st = kernel.socketTable; - const clientId = st.create(AF_UNIX, SOCK_STREAM, 0, CLIENT_PID); - await st.connect(clientId, { path: SOCK_PATH }); - - // Send "ping" to the server - const encoder = new TextEncoder(); - st.send(clientId, encoder.encode('ping')); - - // Wait for the server to process and send its reply - const decoder = new TextDecoder(); - let reply = ''; - const recvDeadline = Date.now() + 10_000; - while (Date.now() < recvDeadline) { - const chunk = st.recv(clientId, 256); - if (chunk && chunk.length > 0) { - reply += decoder.decode(chunk); - break; - } - await new Promise((r) => setTimeout(r, 20)); - } - - expect(reply).toBe('pong'); - - // Close client socket - st.close(clientId, CLIENT_PID); - - // Wait for exec to complete (server exits after handling one connection) - const result = await execPromise; - - expect(result.stdout).toContain(`listening on ${SOCK_PATH}`); - expect(result.stdout).toContain('received: ping'); - expect(result.stdout).toContain('sent: 4'); - expect(result.exitCode).toBe(0); - }); -}); diff --git a/registry/tests/wasmvm/signal-handler.test.ts b/registry/tests/wasmvm/signal-handler.test.ts deleted file mode 100644 index 9feeed21c8..0000000000 --- a/registry/tests/wasmvm/signal-handler.test.ts +++ /dev/null @@ -1,164 +0,0 @@ -/** - * Integration test for WasmVM cooperative signal handling. - * - * Spawns the signal_handler C program as WASM (sigaction(SIGINT, ...) → - * busy-loop with sleep → verify handler called), delivers SIGINT via - * kernel.kill(), and verifies the handler fires at a syscall boundary. - */ - -import { describe, it, expect, beforeEach, afterEach } from 'vitest'; -import { createWasmVmRuntime } from '../helpers.js'; -import { - COMMANDS_DIR, - C_BUILD_DIR, - createKernel, - describeIf, - hasWasmBinaries, - SIGTERM, -} from '../helpers.js'; -import type { Kernel } from '../helpers.js'; -import { existsSync } from 'node:fs'; -import { join } from 'node:path'; - -const hasCWasmBinaries = existsSync(join(C_BUILD_DIR, 'signal_handler')); -const EXPECTED_SIGACTION_FLAGS = (0x10000000 | 0x80000000) >>> 0; - -function skipReason(): string | false { - if (!hasWasmBinaries) return 'WASM binaries not built (run make wasm in native/wasmvm/)'; - if (!hasCWasmBinaries) return 'signal_handler WASM binary not built (run make -C native/wasmvm/c sysroot && make -C native/wasmvm/c programs)'; - return false; -} - -// Minimal in-memory VFS -class SimpleVFS { - private files = new Map(); - private dirs = new Set(['/']); - private symlinks = new Map(); - - async readFile(path: string): Promise { - const data = this.files.get(path); - if (!data) throw new Error(`ENOENT: ${path}`); - return data; - } - async readTextFile(path: string): Promise { - return new TextDecoder().decode(await this.readFile(path)); - } - async pread(path: string, offset: number, length: number): Promise { - const data = await this.readFile(path); - return data.slice(offset, offset + length); - } - async readDir(path: string): Promise { - const prefix = path === '/' ? '/' : path + '/'; - const entries: string[] = []; - for (const p of [...this.files.keys(), ...this.dirs]) { - if (p !== path && p.startsWith(prefix)) { - const rest = p.slice(prefix.length); - if (!rest.includes('/')) entries.push(rest); - } - } - return entries; - } - async readDirWithTypes(path: string) { - return (await this.readDir(path)).map((name) => ({ - name, - isDirectory: this.dirs.has(path === '/' ? `/${name}` : `${path}/${name}`), - })); - } - async writeFile(path: string, content: string | Uint8Array): Promise { - const data = typeof content === 'string' ? new TextEncoder().encode(content) : content; - this.files.set(path, new Uint8Array(data)); - const parts = path.split('/').filter(Boolean); - for (let i = 1; i < parts.length; i++) { - this.dirs.add('/' + parts.slice(0, i).join('/')); - } - } - async createDir(path: string) { this.dirs.add(path); } - async mkdir(path: string, _options?: { recursive?: boolean }) { this.dirs.add(path); } - async exists(path: string): Promise { - return this.files.has(path) || this.dirs.has(path) || this.symlinks.has(path); - } - async stat(path: string) { - const isDir = this.dirs.has(path); - const isSymlink = this.symlinks.has(path); - const data = this.files.get(path); - if (!isDir && !isSymlink && !data) throw new Error(`ENOENT: ${path}`); - return { - mode: isSymlink ? 0o120777 : (isDir ? 0o40755 : 0o100644), - size: data?.length ?? 0, - isDirectory: isDir, - isSymbolicLink: isSymlink, - atimeMs: Date.now(), - mtimeMs: Date.now(), - ctimeMs: Date.now(), - birthtimeMs: Date.now(), - ino: 0, - nlink: 1, - uid: 1000, - gid: 1000, - }; - } - lstat(path: string) { return this.stat(path); } - async chmod() {} - async rename(from: string, to: string) { - const data = this.files.get(from); - if (data) { this.files.set(to, data); this.files.delete(from); } - } - async unlink(path: string) { this.files.delete(path); this.symlinks.delete(path); } - async rmdir(path: string) { this.dirs.delete(path); } - async symlink(target: string, linkPath: string) { - this.symlinks.set(linkPath, target); - const parts = linkPath.split('/').filter(Boolean); - for (let i = 1; i < parts.length; i++) { - this.dirs.add('/' + parts.slice(0, i).join('/')); - } - } - async readlink(path: string): Promise { - const target = this.symlinks.get(path); - if (!target) throw new Error(`EINVAL: ${path}`); - return target; - } -} - -describeIf(!skipReason(), 'WasmVM signal handler integration', { timeout: 30_000 }, () => { - let kernel: Kernel; - let vfs: SimpleVFS; - - beforeEach(async () => { - vfs = new SimpleVFS(); - kernel = createKernel({ filesystem: vfs as any }); - await kernel.mount(createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] })); - }); - - afterEach(async () => { - await kernel?.dispose(); - }); - - it('signal_handler: sigaction registration preserves mask/flags and fires at syscall boundary', async () => { - // Spawn the WASM signal_handler program (registers SIGINT handler, then loops) - let stdout = ''; - const proc = kernel.spawn('signal_handler', [], { - onStdout: (data) => { stdout += new TextDecoder().decode(data); }, - }); - - // Wait for the program to register its handler and start waiting - const deadline = Date.now() + 10_000; - while (Date.now() < deadline && !stdout.includes('waiting')) { - await new Promise((r) => setTimeout(r, 20)); - } - expect(stdout).toContain('handler_registered'); - expect(stdout).toContain('waiting'); - - const registration = kernel.processTable.getSignalState(proc.pid).handlers.get(2); - expect(registration?.mask).toEqual(new Set([SIGTERM])); - expect(registration?.flags).toBe(EXPECTED_SIGACTION_FLAGS); - - // Deliver SIGINT via ManagedProcess.kill() — routes through kernel process table - proc.kill(2 /* SIGINT */); - - // Wait for the program to handle the signal and exit - const exitCode = await proc.wait(); - - expect(stdout).toContain('caught_signal=2'); - expect(exitCode).toBe(0); - }); -}); diff --git a/registry/tests/wasmvm/sqlite3.test.ts b/registry/tests/wasmvm/sqlite3.test.ts deleted file mode 100644 index 2457ce3e1a..0000000000 --- a/registry/tests/wasmvm/sqlite3.test.ts +++ /dev/null @@ -1,332 +0,0 @@ -/** - * Integration tests for sqlite3 C command. - * - * Verifies SQLite CLI operations via kernel.exec() with real WASM binaries: - * - In-memory databases (:memory:) - * - Stdin pipe mode for simple queries - * - SQL from command line arguments for multi-statement operations - * - Meta-commands (.dump, .schema, .tables) - * - * Note: kernel.exec() wraps commands in sh -c. Brush-shell currently returns - * exit code 17 for all child commands. Tests verify stdout correctness. - * - * Multi-statement SQL via stdin is not yet reliable in WASM (fgetc buffering - * issues with the WASI polyfill). Tests use SQL-as-argument for complex cases. - */ - -import { describe, it, expect, afterEach } from 'vitest'; -import { createWasmVmRuntime } from '../helpers.js'; -import { - C_BUILD_DIR, - COMMANDS_DIR, - createKernel, - describeIf, - hasCWasmBinaries, -} from '../helpers.js'; -import type { Kernel } from '../helpers.js'; - -// Minimal in-memory VFS for kernel tests -class SimpleVFS { - private files = new Map(); - private dirs = new Set(['/']); - - async readFile(path: string): Promise { - const data = this.files.get(path); - if (!data) throw new Error(`ENOENT: ${path}`); - return data; - } - async readTextFile(path: string): Promise { - return new TextDecoder().decode(await this.readFile(path)); - } - async readDir(path: string): Promise { - const prefix = path === '/' ? '/' : path + '/'; - const entries: string[] = []; - for (const p of [...this.files.keys(), ...this.dirs]) { - if (p !== path && p.startsWith(prefix)) { - const rest = p.slice(prefix.length); - if (!rest.includes('/')) entries.push(rest); - } - } - return entries; - } - async readDirWithTypes(path: string) { - return (await this.readDir(path)).map(name => ({ - name, - isDirectory: this.dirs.has(path === '/' ? `/${name}` : `${path}/${name}`), - })); - } - async writeFile(path: string, content: string | Uint8Array): Promise { - const data = typeof content === 'string' ? new TextEncoder().encode(content) : content; - this.files.set(path, new Uint8Array(data)); - const parts = path.split('/').filter(Boolean); - for (let i = 1; i < parts.length; i++) { - this.dirs.add('/' + parts.slice(0, i).join('/')); - } - } - async createDir(path: string) { this.dirs.add(path); } - async mkdir(path: string, _options?: { recursive?: boolean }) { - this.dirs.add(path); - const parts = path.split('/').filter(Boolean); - for (let i = 1; i < parts.length; i++) { - this.dirs.add('/' + parts.slice(0, i).join('/')); - } - } - async exists(path: string): Promise { - return this.files.has(path) || this.dirs.has(path); - } - async stat(path: string) { - const isDir = this.dirs.has(path); - const data = this.files.get(path); - if (!isDir && !data) throw new Error(`ENOENT: ${path}`); - return { - mode: isDir ? 0o40755 : 0o100644, - size: data?.length ?? 0, - isDirectory: isDir, - isSymbolicLink: false, - atimeMs: Date.now(), - mtimeMs: Date.now(), - ctimeMs: Date.now(), - birthtimeMs: Date.now(), - ino: 0, - nlink: 1, - uid: 1000, - gid: 1000, - }; - } - async chmod(_path: string, _mode: number) {} - async lstat(path: string) { return this.stat(path); } - async removeFile(path: string) { this.files.delete(path); } - async removeDir(path: string) { this.dirs.delete(path); } - async rename(oldPath: string, newPath: string) { - const data = this.files.get(oldPath); - if (data) { - this.files.set(newPath, data); - this.files.delete(oldPath); - } - } - async pread(path: string, buffer: Uint8Array, offset: number, length: number, position: number): Promise { - const data = this.files.get(path); - if (!data) throw new Error(`ENOENT: ${path}`); - const available = Math.min(length, data.length - position); - if (available <= 0) return 0; - buffer.set(data.subarray(position, position + available), offset); - return available; - } -} - -describeIf(hasCWasmBinaries('sqlite3'), 'sqlite3 command', () => { - let kernel: Kernel; - - afterEach(async () => { - await kernel?.dispose(); - }); - - it('executes SQL from stdin pipe on in-memory database', async () => { - const vfs = new SimpleVFS(); - kernel = createKernel({ filesystem: vfs as any }); - await kernel.mount(createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] })); - - const result = await kernel.exec('sqlite3 :memory:', { - stdin: 'SELECT 1+1 AS result;\n', - }); - expect(result.stdout.trim()).toBe('2'); - }); - - it('executes multi-statement SQL as command argument', async () => { - const vfs = new SimpleVFS(); - kernel = createKernel({ filesystem: vfs as any }); - await kernel.mount(createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] })); - - // Multi-statement SQL passed as command argument (more reliable than stdin in WASM) - const sql = 'CREATE TABLE t(x INTEGER); INSERT INTO t VALUES(10); INSERT INTO t VALUES(20); INSERT INTO t VALUES(30); SELECT * FROM t ORDER BY x;'; - const result = await kernel.exec(`sqlite3 :memory: "${sql}"`); - expect(result.stdout.trim()).toBe('10\n20\n30'); - }); - - it('supports .tables meta-command via SQL setup', async () => { - const vfs = new SimpleVFS(); - kernel = createKernel({ filesystem: vfs as any }); - await kernel.mount(createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] })); - - // Create tables via SQL argument, then query sqlite_master - const sql = "CREATE TABLE alpha(x); CREATE TABLE beta(y); SELECT name FROM sqlite_master WHERE type='table' ORDER BY 1;"; - const result = await kernel.exec(`sqlite3 :memory: "${sql}"`); - const tables = result.stdout.trim().split('\n').sort(); - expect(tables).toEqual(['alpha', 'beta']); - }); - - it('supports .schema via sqlite_master query', async () => { - const vfs = new SimpleVFS(); - kernel = createKernel({ filesystem: vfs as any }); - await kernel.mount(createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] })); - - const sql = "CREATE TABLE users(id INTEGER PRIMARY KEY, name TEXT NOT NULL); SELECT sql FROM sqlite_master WHERE name='users';"; - const result = await kernel.exec(`sqlite3 :memory: "${sql}"`); - expect(result.stdout.trim()).toContain('CREATE TABLE users'); - }); - - it('supports .dump style output via SQL', async () => { - const vfs = new SimpleVFS(); - kernel = createKernel({ filesystem: vfs as any }); - await kernel.mount(createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] })); - - const sql = "CREATE TABLE t(x INTEGER, y TEXT); INSERT INTO t VALUES(1,'hello'); SELECT sql FROM sqlite_master; SELECT * FROM t;"; - const result = await kernel.exec(`sqlite3 :memory: "${sql}"`); - const output = result.stdout.trim(); - expect(output).toContain('CREATE TABLE t'); - expect(output).toContain("1|hello"); - }); - - it('handles SELECT with multiple columns', async () => { - const vfs = new SimpleVFS(); - kernel = createKernel({ filesystem: vfs as any }); - await kernel.mount(createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] })); - - const result = await kernel.exec('sqlite3 :memory:', { - stdin: "SELECT 'hello' AS greeting, 42 AS number, 3.14 AS pi;\n", - }); - expect(result.stdout.trim()).toBe('hello|42|3.14'); - }); - - it('handles NULL values in output', async () => { - const vfs = new SimpleVFS(); - kernel = createKernel({ filesystem: vfs as any }); - await kernel.mount(createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] })); - - const result = await kernel.exec('sqlite3 :memory:', { - stdin: 'SELECT NULL;\n', - }); - // SQLite CLI outputs empty string for NULL - expect(result.stdout.trim()).toBe(''); - }); - - it('reports SQL errors on stderr', async () => { - const vfs = new SimpleVFS(); - kernel = createKernel({ filesystem: vfs as any }); - await kernel.mount(createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] })); - - const result = await kernel.exec(`sqlite3 :memory: "SELECT * FROM nonexistent_table;"`); - expect(result.stderr).toContain('no such table'); - }); - - it('defaults to :memory: when no database specified', async () => { - const vfs = new SimpleVFS(); - kernel = createKernel({ filesystem: vfs as any }); - await kernel.mount(createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] })); - - const result = await kernel.exec('sqlite3', { - stdin: 'SELECT 99;\n', - }); - expect(result.stdout.trim()).toBe('99'); - }); - - it('CREATE TABLE, INSERT, SELECT roundtrip via piped SQL', async () => { - const vfs = new SimpleVFS(); - kernel = createKernel({ filesystem: vfs as any }); - await kernel.mount(createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] })); - - // Multi-statement SQL via command arg (stdin multi-statement has fgetc buffering issues in WASM) - const sql = "CREATE TABLE items(id INTEGER PRIMARY KEY, name TEXT); INSERT INTO items VALUES(1,'apple'); INSERT INTO items VALUES(2,'banana'); SELECT id, name FROM items ORDER BY id;"; - const result = await kernel.exec(`sqlite3 :memory: "${sql}"`); - expect(result.stdout.trim()).toBe('1|apple\n2|banana'); - }); - - it('.tables meta-command lists created tables', async () => { - const vfs = new SimpleVFS(); - kernel = createKernel({ filesystem: vfs as any }); - await kernel.mount(createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] })); - - // Multi-statement stdin has fgetc buffering limitations in WASM, - // use SQL command arg to verify table listing behavior - const sql = "CREATE TABLE alpha(x); CREATE TABLE beta(y); SELECT name FROM sqlite_master WHERE type='table' ORDER BY 1;"; - const result = await kernel.exec(`sqlite3 :memory: "${sql}"`); - const tables = result.stdout.trim().split('\n').sort(); - expect(tables).toEqual(['alpha', 'beta']); - }); - - it('.schema meta-command shows CREATE TABLE statements', async () => { - const vfs = new SimpleVFS(); - kernel = createKernel({ filesystem: vfs as any }); - await kernel.mount(createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] })); - - // Query schema via sqlite_master (equivalent to .schema output) - const sql = "CREATE TABLE users(id INTEGER PRIMARY KEY, name TEXT NOT NULL); SELECT sql FROM sqlite_master WHERE name='users';"; - const result = await kernel.exec(`sqlite3 :memory: "${sql}"`); - expect(result.stdout).toContain('CREATE TABLE users'); - expect(result.stdout).toContain('id INTEGER PRIMARY KEY'); - }); - - it('.dump meta-command outputs INSERT statements for data', async () => { - const vfs = new SimpleVFS(); - kernel = createKernel({ filesystem: vfs as any }); - await kernel.mount(createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] })); - - // Verify dump-equivalent output: schema + data via SQL queries - const sql = "CREATE TABLE t(x INTEGER, y TEXT); INSERT INTO t VALUES(1,'hello'); INSERT INTO t VALUES(2,'world'); SELECT sql FROM sqlite_master WHERE name='t'; SELECT '---'; SELECT x||','||y FROM t ORDER BY x;"; - const result = await kernel.exec(`sqlite3 :memory: "${sql}"`); - const output = result.stdout; - // Schema is preserved - expect(output).toContain('CREATE TABLE t'); - // Data is preserved and retrievable - expect(output).toContain("1,hello"); - expect(output).toContain("2,world"); - }); - - it('file-based DB persists data across separate exec calls', async () => { - const vfs = new SimpleVFS(); - await vfs.mkdir('/tmp', { recursive: true }); - kernel = createKernel({ filesystem: vfs as any }); - await kernel.mount(createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] })); - - // Use shell pipe to create table and insert data, then query back - // Note: file-based DB uses WASI VFS (open/write/fstat/ftruncate) which - // requires full POSIX file I/O support through the kernel - const createSql = "CREATE TABLE t(x INTEGER); INSERT INTO t VALUES(42); INSERT INTO t VALUES(99);"; - const createResult = await kernel.exec(`sqlite3 /tmp/test.db "${createSql}"`); - - // Check if file-based DB is supported (fstat/ftruncate may not be available) - const hasError = createResult.stderr.includes('disk I/O error') || - createResult.stderr.includes('unable to open database'); - if (hasError) { - // Fall back: verify file-based behavior via VFS write/read simulation - // Write a pre-populated DB, then verify sqlite3 can read from VFS-provided data - // For now, verify in-memory DB persistence within single exec - const result = await kernel.exec( - 'sqlite3 :memory: "CREATE TABLE t(x INTEGER); INSERT INTO t VALUES(42); INSERT INTO t VALUES(99); SELECT * FROM t ORDER BY x;"' - ); - expect(result.stdout.trim()).toBe('42\n99'); - return; - } - - // Verify file was created in VFS - const dbData = await vfs.readFile('/tmp/test.db'); - expect(dbData.length).toBeGreaterThan(0); - - // Second exec: reopen and query persisted data via stdin - const result = await kernel.exec('sqlite3 /tmp/test.db', { - stdin: 'SELECT * FROM t ORDER BY x;\n', - }); - expect(result.stdout.trim()).toBe('42\n99'); - }); - - it('multi-statement input separated by semicolons', async () => { - const vfs = new SimpleVFS(); - kernel = createKernel({ filesystem: vfs as any }); - await kernel.mount(createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] })); - - // Multi-statement SQL via command arg (semicolons separate statements) - const sql = "CREATE TABLE nums(v); INSERT INTO nums VALUES(10); INSERT INTO nums VALUES(20); SELECT v FROM nums ORDER BY v;"; - const result = await kernel.exec(`sqlite3 :memory: "${sql}"`); - expect(result.stdout.trim()).toBe('10\n20'); - }); - - it('SQL syntax error produces error on stderr with non-zero exit', async () => { - const vfs = new SimpleVFS(); - kernel = createKernel({ filesystem: vfs as any }); - await kernel.mount(createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] })); - - // Syntax error via command arg (reliable error output) - const result = await kernel.exec('sqlite3 :memory: "SELEC INVALID SYNTAX;"'); - expect(result.stderr).toContain('Error'); - }); -}); diff --git a/registry/tests/wasmvm/terminal-harness.ts b/registry/tests/wasmvm/terminal-harness.ts deleted file mode 100644 index d5d80772f0..0000000000 --- a/registry/tests/wasmvm/terminal-harness.ts +++ /dev/null @@ -1,172 +0,0 @@ -/** - * TerminalHarness — wires openShell() to a headless xterm Terminal for - * deterministic screen-state assertions in tests. - * - * Duplicated from packages/kernel/test/terminal-harness.ts because cross-package - * test imports aren't supported. - */ - -import { Terminal } from "@xterm/headless"; -import type { Kernel } from "../helpers.js"; - -type ShellHandle = ReturnType; - -/** Settlement window: resolve type() after this many ms of no new output. */ -const SETTLE_MS = 50; -/** Poll interval for waitFor(). */ -const POLL_MS = 20; -/** Default waitFor() timeout. */ -const DEFAULT_WAIT_TIMEOUT_MS = 10_000; - -export class TerminalHarness { - readonly term: Terminal; - readonly shell: ShellHandle; - private typing = false; - private disposed = false; - - constructor(kernel: Kernel, options?: { cols?: number; rows?: number; env?: Record; cwd?: string }) { - const cols = options?.cols ?? 80; - const rows = options?.rows ?? 24; - - this.term = new Terminal({ cols, rows, allowProposedApi: true }); - - this.shell = kernel.openShell({ cols, rows, env: options?.env, cwd: options?.cwd }); - - // Wire shell output → xterm - this.shell.onData = (data: Uint8Array) => { - this.term.write(data); - }; - } - - /** - * Send input through the PTY. Resolves after output settles (no new bytes - * received for SETTLE_MS). - */ - async type(input: string): Promise { - if (this.typing) { - throw new Error("TerminalHarness.type() called while previous type() is still in-flight"); - } - this.typing = true; - try { - await this.typeInternal(input); - } finally { - this.typing = false; - } - } - - private typeInternal(input: string): Promise { - return new Promise((resolve) => { - let timer: ReturnType | null = null; - - const resetTimer = () => { - if (timer !== null) clearTimeout(timer); - timer = setTimeout(() => { - this.shell.onData = originalOnData; - resolve(); - }, SETTLE_MS); - }; - - const originalOnData = this.shell.onData; - this.shell.onData = (data: Uint8Array) => { - this.term.write(data); - resetTimer(); - }; - - resetTimer(); - this.shell.write(input); - }); - } - - /** - * Full screen as a string: viewport rows only (not scrollback), trailing - * whitespace trimmed per line, trailing empty lines dropped, joined with '\n'. - */ - screenshotTrimmed(): string { - const buf = this.term.buffer.active; - const rows = this.term.rows; - const lines: string[] = []; - - for (let y = 0; y < rows; y++) { - const line = buf.getLine(buf.viewportY + y); - lines.push(line ? line.translateToString(true) : ""); - } - - while (lines.length > 0 && lines[lines.length - 1] === "") { - lines.pop(); - } - - return lines.join("\n"); - } - - /** - * Single trimmed row from the screen buffer (0-indexed from viewport top). - */ - line(row: number): string { - const buf = this.term.buffer.active; - const line = buf.getLine(buf.viewportY + row); - return line ? line.translateToString(true) : ""; - } - - /** - * Poll screen buffer every POLL_MS until `text` is found. Throws a - * descriptive error on timeout. - */ - async waitFor( - text: string, - occurrence: number = 1, - timeoutMs: number = DEFAULT_WAIT_TIMEOUT_MS, - ): Promise { - const deadline = Date.now() + timeoutMs; - - while (true) { - const screen = this.screenshotTrimmed(); - - let count = 0; - let idx = -1; - while (true) { - idx = screen.indexOf(text, idx + 1); - if (idx === -1) break; - count++; - if (count >= occurrence) return; - } - - if (Date.now() >= deadline) { - throw new Error( - `waitFor("${text}", ${occurrence}) timed out after ${timeoutMs}ms.\n` + - `Expected: "${text}" (occurrence ${occurrence})\n` + - `Screen:\n${screen}`, - ); - } - - await new Promise((r) => setTimeout(r, POLL_MS)); - } - } - - /** - * Send ^D on empty line and await shell exit. Returns exit code. - */ - async exit(): Promise { - this.shell.write("\x04"); - return this.shell.wait(); - } - - /** - * Kill shell and dispose terminal. Safe to call multiple times. - */ - async dispose(): Promise { - if (this.disposed) return; - this.disposed = true; - - try { - this.shell.kill(); - await Promise.race([ - this.shell.wait(), - new Promise((r) => setTimeout(r, 500)), - ]); - } catch { - // Shell may already be dead - } - - this.term.dispose(); - } -} diff --git a/registry/tests/wasmvm/wget.test.ts b/registry/tests/wasmvm/wget.test.ts deleted file mode 100644 index e1121f6b36..0000000000 --- a/registry/tests/wasmvm/wget.test.ts +++ /dev/null @@ -1,239 +0,0 @@ -/** - * Integration tests for wget C command (libcurl-based). - * - * Verifies HTTP download operations via kernel.exec() with real WASM binaries: - * - Basic GET download to file - * - Download to specified file (-O) - * - Quiet mode (-q) - * - Error handling for 404 URLs - * - Follow redirects (default behavior) - * - * Tests start a local HTTP server in beforeAll and make wget requests against it. - */ - -import { describe, it, expect, afterEach, beforeAll, afterAll } from 'vitest'; -import { createWasmVmRuntime } from '../helpers.js'; -import { C_BUILD_DIR, COMMANDS_DIR, createKernel, describeIf, hasCWasmBinaries } from '../helpers.js'; -import type { Kernel } from '../helpers.js'; -import { createServer, type Server, type IncomingMessage, type ServerResponse } from 'node:http'; - -// Minimal in-memory VFS for kernel tests -class SimpleVFS { - private files = new Map(); - private dirs = new Set(['/']); - - async readFile(path: string): Promise { - const data = this.files.get(path); - if (!data) throw new Error(`ENOENT: ${path}`); - return data; - } - async readTextFile(path: string): Promise { - return new TextDecoder().decode(await this.readFile(path)); - } - async readDir(path: string): Promise { - const prefix = path === '/' ? '/' : path + '/'; - const entries: string[] = []; - for (const p of [...this.files.keys(), ...this.dirs]) { - if (p !== path && p.startsWith(prefix)) { - const rest = p.slice(prefix.length); - if (!rest.includes('/')) entries.push(rest); - } - } - return entries; - } - async readDirWithTypes(path: string) { - return (await this.readDir(path)).map(name => ({ - name, - isDirectory: this.dirs.has(path === '/' ? `/${name}` : `${path}/${name}`), - })); - } - async writeFile(path: string, content: string | Uint8Array): Promise { - const data = typeof content === 'string' ? new TextEncoder().encode(content) : content; - this.files.set(path, new Uint8Array(data)); - const parts = path.split('/').filter(Boolean); - for (let i = 1; i < parts.length; i++) { - this.dirs.add('/' + parts.slice(0, i).join('/')); - } - } - async createDir(path: string) { this.dirs.add(path); } - async mkdir(path: string, _options?: { recursive?: boolean }) { - this.dirs.add(path); - const parts = path.split('/').filter(Boolean); - for (let i = 1; i < parts.length; i++) { - this.dirs.add('/' + parts.slice(0, i).join('/')); - } - } - async exists(path: string): Promise { - return this.files.has(path) || this.dirs.has(path); - } - async stat(path: string) { - const isDir = this.dirs.has(path); - const data = this.files.get(path); - if (!isDir && !data) throw new Error(`ENOENT: ${path}`); - return { - mode: isDir ? 0o40755 : 0o100644, - size: data?.length ?? 0, - isDirectory: isDir, - isSymbolicLink: false, - atimeMs: Date.now(), - mtimeMs: Date.now(), - ctimeMs: Date.now(), - birthtimeMs: Date.now(), - ino: 0, - nlink: 1, - uid: 1000, - gid: 1000, - }; - } - async chmod(_path: string, _mode: number) {} - async lstat(path: string) { return this.stat(path); } - async removeFile(path: string) { this.files.delete(path); } - async removeDir(path: string) { this.dirs.delete(path); } - async rename(oldPath: string, newPath: string) { - const data = this.files.get(oldPath); - if (data) { - this.files.set(newPath, data); - this.files.delete(oldPath); - } - } - async pread(path: string, buffer: Uint8Array, offset: number, length: number, position: number): Promise { - const data = this.files.get(path); - if (!data) throw new Error(`ENOENT: ${path}`); - const available = Math.min(length, data.length - position); - if (available <= 0) return 0; - buffer.set(data.subarray(position, position + available), offset); - return available; - } - - has(path: string): boolean { - return this.files.has(path); - } - getContent(path: string): string | undefined { - const data = this.files.get(path); - return data ? new TextDecoder().decode(data) : undefined; - } - getRawContent(path: string): Uint8Array | undefined { - return this.files.get(path); - } -} - -// TODO(P6): requires wget WASM artifact, intentionally excluded from the fast registry-build gate. -describe.skip('wget command', () => { - let kernel: Kernel; - let server: Server; - let port: number; - - beforeAll(async () => { - server = createServer((req: IncomingMessage, res: ServerResponse) => { - const url = req.url ?? '/'; - - if (url === '/file.txt') { - res.writeHead(200, { 'Content-Type': 'text/plain' }); - res.end('downloaded content'); - return; - } - - if (url === '/data.json') { - res.writeHead(200, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify({ status: 'ok' })); - return; - } - - if (url === '/redirect') { - const addr = server.address() as import('node:net').AddressInfo; - res.writeHead(302, { 'Location': `http://127.0.0.1:${addr.port}/redirected` }); - res.end(); - return; - } - - if (url === '/redirected') { - res.writeHead(200, { 'Content-Type': 'text/plain' }); - res.end('arrived after redirect'); - return; - } - - if (url === '/binary') { - const buf = Buffer.alloc(1024); - for (let i = 0; i < buf.length; i++) buf[i] = i & 0xff; - res.writeHead(200, { - 'Content-Type': 'application/octet-stream', - 'Content-Length': String(buf.length), - }); - res.end(buf); - return; - } - - res.writeHead(404, { 'Content-Type': 'text/plain' }); - res.end('not found'); - }); - - await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); - port = (server.address() as import('node:net').AddressInfo).port; - }); - - afterAll(async () => { - await new Promise((resolve) => server.close(() => resolve())); - }); - - afterEach(async () => { - await kernel?.dispose(); - }); - - it('downloads file to VFS using URL basename', async () => { - const vfs = new SimpleVFS(); - kernel = createKernel({ filesystem: vfs as any }); - await kernel.mount(createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] })); - - await kernel.exec(`wget http://127.0.0.1:${port}/file.txt`); - - const content = vfs.getContent('/file.txt'); - expect(content).toBe('downloaded content'); - }); - - it('-O saves to specified filename', async () => { - const vfs = new SimpleVFS(); - kernel = createKernel({ filesystem: vfs as any }); - await kernel.mount(createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] })); - - await kernel.exec(`wget -O /output.txt http://127.0.0.1:${port}/data.json`); - - const content = vfs.getContent('/output.txt'); - expect(content).toBeDefined(); - expect(content).toContain('"status":"ok"'); - }); - - it('-q suppresses progress output', async () => { - const vfs = new SimpleVFS(); - kernel = createKernel({ filesystem: vfs as any }); - await kernel.mount(createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] })); - - const result = await kernel.exec(`wget -q -O /output.txt http://127.0.0.1:${port}/file.txt`); - - // Quiet mode should produce no stderr - expect(result.stderr).toBe(''); - // File should still be downloaded - expect(vfs.getContent('/output.txt')).toBe('downloaded content'); - }); - - it('returns non-zero exit code for 404 URL', async () => { - const vfs = new SimpleVFS(); - kernel = createKernel({ filesystem: vfs as any }); - await kernel.mount(createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] })); - - const result = await kernel.exec(`wget http://127.0.0.1:${port}/nonexistent`); - - // Should report error on stderr - expect(result.stderr).toMatch(/wget|404|error|server/i); - }); - - it('follows redirects by default', async () => { - const vfs = new SimpleVFS(); - kernel = createKernel({ filesystem: vfs as any }); - await kernel.mount(createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] })); - - await kernel.exec(`wget -O /output.txt http://127.0.0.1:${port}/redirect`); - - const content = vfs.getContent('/output.txt'); - expect(content).toBe('arrived after redirect'); - }); -}); diff --git a/registry/tests/wasmvm/zip-unzip.test.ts b/registry/tests/wasmvm/zip-unzip.test.ts deleted file mode 100644 index db330c1e59..0000000000 --- a/registry/tests/wasmvm/zip-unzip.test.ts +++ /dev/null @@ -1,249 +0,0 @@ -/** - * Integration tests for zip and unzip C commands. - * - * Verifies zip/unzip roundtrip, recursive compression, list mode, - * and extract-to-directory via kernel.exec() with real WASM binaries. - */ - -import { describe, it, expect, afterEach } from 'vitest'; -import { createInMemoryFileSystem, createWasmVmRuntime } from '../helpers.js'; -import { C_BUILD_DIR, COMMANDS_DIR, createKernel } from '../helpers.js'; -import type { Kernel } from '../helpers.js'; - -interface HostileEntry { - name: string; - method: number; // 0 = store, 8 = deflate - compressedSize: number; - uncompressedSize: number; - localOffset: number; -} - -/** Builds a ZIP whose EOCD cd-size field is corrupt so minizip rejects it and - * unzip's raw central-directory fallback parser is exercised. The nonzero - * version fields on each central directory record also make minizip reject - * the archive under the VM's stream semantics, where its reopen-based seek - * callback reads EOCD fields from offset 0 instead of the EOCD record. - * `prefix` bytes (e.g. a real local file header) are placed at offset 0. */ -function buildFallbackArchive(prefix: Uint8Array, entries: HostileEntry[]): Uint8Array { - const enc = new TextEncoder(); - const cdParts: Uint8Array[] = []; - for (const e of entries) { - const nameBytes = enc.encode(e.name); - const cd = new Uint8Array(46 + nameBytes.length); - const dv = new DataView(cd.buffer); - dv.setUint32(0, 0x02014b50, true); // central directory signature - dv.setUint16(4, 20, true); // version made by - dv.setUint16(6, 20, true); // version needed to extract - dv.setUint16(10, e.method, true); - dv.setUint32(20, e.compressedSize, true); - dv.setUint32(24, e.uncompressedSize, true); - dv.setUint16(28, nameBytes.length, true); - dv.setUint32(42, e.localOffset, true); - cd.set(nameBytes, 46); - cdParts.push(cd); - } - const cdOffset = prefix.length; - const cdLen = cdParts.reduce((n, p) => n + p.length, 0); - const eocd = new Uint8Array(22); - const dv = new DataView(eocd.buffer); - dv.setUint32(0, 0x06054b50, true); // EOCD signature - dv.setUint16(8, entries.length, true); // entries on this disk - dv.setUint16(10, entries.length, true);// total entries - dv.setUint32(12, 0xffffffff, true); // corrupt cd size: forces the fallback parser - dv.setUint32(16, cdOffset, true); - const out = new Uint8Array(prefix.length + cdLen + 22); - out.set(prefix, 0); - let off = cdOffset; - for (const p of cdParts) { out.set(p, off); off += p.length; } - out.set(eocd, off); - return out; -} - -describe('zip/unzip commands', () => { - let kernel: Kernel; - - afterEach(async () => { - await kernel?.dispose(); - }); - - it('zip creates valid archive, unzip extracts it, contents match', async () => { - const vfs = createInMemoryFileSystem(); - await vfs.writeFile('/hello.txt', 'Hello, World!\n'); - - kernel = createKernel({ filesystem: vfs }); - await kernel.mount( - createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] }), - ); - - // Create zip archive - const zipResult = await kernel.exec('zip /archive.zip /hello.txt'); - expect(zipResult.exitCode, zipResult.stderr).toBe(0); - - // Verify archive was created - expect(await vfs.exists('/archive.zip')).toBe(true); - - // Extract to a different directory - const unzipResult = await kernel.exec('unzip -d /extracted /archive.zip'); - expect(unzipResult.exitCode, unzipResult.stderr).toBe(0); - - // Verify extracted content matches original - const extracted = await vfs.readTextFile('/extracted/hello.txt'); - expect(extracted).toBe('Hello, World!\n'); - }); - - it('zip -r compresses directory recursively', async () => { - const vfs = createInMemoryFileSystem(); - await vfs.mkdir('/mydir'); - await vfs.writeFile('/mydir/a.txt', 'file a\n'); - await vfs.writeFile('/mydir/b.txt', 'file b\n'); - - kernel = createKernel({ filesystem: vfs }); - await kernel.mount( - createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] }), - ); - - const zipResult = await kernel.exec('zip -r /dir.zip /mydir'); - expect(zipResult.exitCode, zipResult.stderr).toBe(0); - expect(await vfs.exists('/dir.zip')).toBe(true); - - // Extract and verify - const unzipResult = await kernel.exec('unzip -d /out /dir.zip'); - expect(unzipResult.exitCode, unzipResult.stderr).toBe(0); - - const a = await vfs.readTextFile('/out/mydir/a.txt'); - const b = await vfs.readTextFile('/out/mydir/b.txt'); - expect(a).toBe('file a\n'); - expect(b).toBe('file b\n'); - }); - - it('unzip -l lists archive contents with sizes', async () => { - const vfs = createInMemoryFileSystem(); - await vfs.writeFile('/data.txt', 'some data content\n'); - - kernel = createKernel({ filesystem: vfs }); - await kernel.mount( - createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] }), - ); - - // Create archive first - const zipResult = await kernel.exec('zip /list-test.zip /data.txt'); - expect(zipResult.exitCode, zipResult.stderr).toBe(0); - - // List contents - const listResult = await kernel.exec('unzip -l /list-test.zip'); - expect(listResult.exitCode, listResult.stderr).toBe(0); - expect(listResult.stdout).toContain('data.txt'); - // Should show the file size (18 bytes) - expect(listResult.stdout).toContain('18'); - // Should show summary line with file count - expect(listResult.stdout).toMatch(/1 file/); - }); - - it('zip/unzip roundtrip preserves file contents exactly', async () => { - const vfs = createInMemoryFileSystem(); - // Binary-like content with various byte values - const content = new Uint8Array(256); - for (let i = 0; i < 256; i++) content[i] = i; - await vfs.writeFile('/binary.bin', content); - - kernel = createKernel({ filesystem: vfs }); - await kernel.mount( - createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] }), - ); - - const zipResult = await kernel.exec('zip /roundtrip.zip /binary.bin'); - expect(zipResult.exitCode, zipResult.stderr).toBe(0); - - const unzipResult = await kernel.exec('unzip -d /rt-out /roundtrip.zip'); - expect(unzipResult.exitCode, unzipResult.stderr).toBe(0); - - const extracted = await vfs.readFile('/rt-out/binary.bin'); - expect(extracted.length).toBe(256); - for (let i = 0; i < 256; i++) { - expect(extracted[i]).toBe(i); - } - }); - - it('unzip -d extracts to specified directory', async () => { - const vfs = createInMemoryFileSystem(); - await vfs.writeFile('/src.txt', 'target content\n'); - - kernel = createKernel({ filesystem: vfs }); - await kernel.mount( - createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] }), - ); - - const zipResult = await kernel.exec('zip /dest-test.zip /src.txt'); - expect(zipResult.exitCode, zipResult.stderr).toBe(0); - - // Extract to a new directory - const unzipResult = await kernel.exec('unzip -d /custom-dir /dest-test.zip'); - expect(unzipResult.exitCode, unzipResult.stderr).toBe(0); - - expect(await vfs.exists('/custom-dir/src.txt')).toBe(true); - const extracted = await vfs.readTextFile('/custom-dir/src.txt'); - expect(extracted).toBe('target content\n'); - }); - - it('fallback parser rejects an entry with a wrapping local offset', async () => { - const vfs = createInMemoryFileSystem(); - const bytes = buildFallbackArchive(new Uint8Array(0), [ - { name: 'evil.txt', method: 0, compressedSize: 4, uncompressedSize: 4, localOffset: 0xfffffff0 }, - ]); - await vfs.writeFile('/evil.zip', bytes); - - kernel = createKernel({ filesystem: vfs }); - await kernel.mount( - createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] }), - ); - - const result = await kernel.exec('unzip -d /out /evil.zip'); - expect(result.exitCode, result.stderr).toBe(1); - expect(result.stderr).toMatch(/error/); - expect(await vfs.exists('/out/evil.txt')).toBe(false); - }); - - it('fallback parser skips an entry whose normalized name is empty', async () => { - const vfs = createInMemoryFileSystem(); - const bytes = buildFallbackArchive(new Uint8Array(0), [ - { name: '/', method: 0, compressedSize: 0, uncompressedSize: 0, localOffset: 0 }, - ]); - await vfs.writeFile('/empty-name.zip', bytes); - - kernel = createKernel({ filesystem: vfs }); - await kernel.mount( - createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] }), - ); - - const result = await kernel.exec('unzip /empty-name.zip'); - expect(result.exitCode, result.stderr).toBe(0); - expect(result.stdout).not.toMatch(/error/); - expect(result.stderr).not.toMatch(/error/); - }); - - it('fallback parser caps hostile uncompressed sizes before allocating', async () => { - const vfs = createInMemoryFileSystem(); - // A real 31-byte local header for a 1-byte stored payload. - const prefix = new Uint8Array(31); - const pdv = new DataView(prefix.buffer); - pdv.setUint32(0, 0x04034b50, true); // local file header signature - pdv.setUint16(4, 20, true); // version needed to extract - pdv.setUint16(26, 0, true); // name length - pdv.setUint16(28, 0, true); // extra length - prefix[30] = 0x41; // one payload byte - const bytes = buildFallbackArchive(prefix, [ - { name: 'big.bin', method: 0, compressedSize: 1, uncompressedSize: 0xffffffff, localOffset: 0 }, - ]); - await vfs.writeFile('/big.zip', bytes); - - kernel = createKernel({ filesystem: vfs }); - await kernel.mount( - createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] }), - ); - - const result = await kernel.exec('unzip -d /cap-out /big.zip'); - expect(result.exitCode, result.stderr).toBe(1); - expect(result.stderr).toMatch(/too large/); - expect(await vfs.exists('/cap-out/big.bin')).toBe(false); - }); -}); diff --git a/scripts/benchmarks/session.bench.ts b/scripts/benchmarks/session.bench.ts index e93a600d16..13b80b8562 100644 --- a/scripts/benchmarks/session.bench.ts +++ b/scripts/benchmarks/session.bench.ts @@ -299,7 +299,7 @@ async function loadPiSoftware(): Promise { // non-snapshot fallback path and hides the optimization entirely. const local = join( import.meta.dirname, - "../../../secure-exec/registry/agent/pi/dist/index.js", + "../../../secure-exec/software/pi/dist/index.js", ); if (existsSync(local)) return (await import(local)).default; // Fallback: the published/installed software package. Variable specifier so @@ -309,7 +309,7 @@ async function loadPiSoftware(): Promise { return (await import(piPkg)).default; } catch { throw new Error( - "Could not resolve the pi software package (../secure-exec/registry/agent/pi/dist or @agentos-software/pi). Build it first.", + "Could not resolve the pi software package (../secure-exec/software/pi/dist or @agentos-software/pi). Build it first.", ); } } @@ -323,7 +323,7 @@ function findPiSdkRoot(): string | null { createRequire( join( import.meta.dirname, - "../../../secure-exec/registry/agent/pi/package.json", + "../../../secure-exec/software/pi/package.json", ), ), ]; @@ -373,7 +373,7 @@ function resolvePiSdkRootOrThrow(): string { * The bare-node session-creation script, run in a FRESH node process per sample * so each pays the full cold SDK load (the VM lane reloads the SDK in a fresh V8 * isolate every session, so this is the apples-to-apples "Node.js equivalent"). - * Mirrors secure-exec registry/agent/pi/src/adapter.ts `newSession`, with no VM. It times the + * Mirrors secure-exec software/pi/src/adapter.ts `newSession`, with no VM. It times the * SDK load + session construction internally and prints `__MS__=`. */ function bareNodeScript(root: string): string { diff --git a/scripts/check-layout.mjs b/scripts/check-layout.mjs new file mode 100644 index 0000000000..971600ce27 --- /dev/null +++ b/scripts/check-layout.mjs @@ -0,0 +1,126 @@ +import { execFileSync } from "node:child_process"; +import { + existsSync, + readdirSync, + readFileSync, + statSync, +} from "node:fs"; +import { join, relative, sep } from "node:path"; + +const root = process.cwd(); +const failures = []; + +const fail = (message) => failures.push(message); +const rel = (path) => relative(root, path).split(sep).join("/"); +const readJson = (path) => JSON.parse(readFileSync(path, "utf8")); + +if (existsSync(join(root, "registry"))) { + fail("registry/ must not exist; use software/ and toolchain/ instead"); +} + +const ignoredDirs = new Set([ + ".git", + "node_modules", + "target", + "dist", + "build", + "vendor", + ".turbo", +]); + +const walk = (dir, visit) => { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (entry.isDirectory() && ignoredDirs.has(entry.name)) continue; + const path = join(dir, entry.name); + if (entry.isDirectory()) { + walk(path, visit); + } else { + visit(path); + } + } +}; + +const allowedTestHomes = [ + /^software\/[^/]+\/test\/.+\.test\.ts$/, + /^toolchain\/conformance\/.+\.test\.ts$/, + /^packages\/[^/]+\/tests\/.+\.test\.ts$/, + /^scripts\/.+\.test\.ts$/, +]; + +walk(root, (path) => { + const normalized = rel(path); + if (!normalized.endsWith(".test.ts")) return; + if (!allowedTestHomes.some((pattern) => pattern.test(normalized))) { + fail(`${normalized} is not in an allowed test home`); + } +}); + +const softwareRoot = join(root, "software"); +if (existsSync(softwareRoot)) { + for (const dir of readdirSync(softwareRoot, { withFileTypes: true })) { + if (!dir.isDirectory()) continue; + const packageDir = join(softwareRoot, dir.name); + const manifestPath = join(packageDir, "agentos-package.json"); + if (!existsSync(manifestPath)) continue; + const manifest = readJson(manifestPath); + const commands = manifest.commands ?? []; + if (commands.length > 0 && !existsSync(join(packageDir, "test"))) { + fail(`software/${dir.name} declares commands but has no test/ directory`); + } + if (manifest.registry && !manifest.registry.category) { + fail(`software/${dir.name} registry block is missing category`); + } + } +} + +const colocatedCargoTomls = []; +const collectCargoTomls = (dir) => { + if (!existsSync(dir)) return; + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const path = join(dir, entry.name); + if (entry.isDirectory()) { + collectCargoTomls(path); + } else if (entry.name === "Cargo.toml") { + colocatedCargoTomls.push(path); + } + } +}; +collectCargoTomls(join(root, "software")); + +for (const cargoToml of colocatedCargoTomls) { + const normalized = rel(cargoToml); + if (!/\/native\/crates\/[^/]+\/Cargo\.toml$/.test(normalized)) continue; + const text = readFileSync(cargoToml, "utf8"); + if (!/^\[package\]\nworkspace = "\.\.\/\.\.\/\.\.\/\.\.\/\.\.\/toolchain"/m.test(text)) { + fail(`${normalized} must explicitly belong to the toolchain workspace`); + } +} + +try { + const metadata = JSON.parse( + execFileSync("cargo", ["metadata", "--format-version", "1", "--no-deps"], { + cwd: root, + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }), + ); + for (const pkg of metadata.packages ?? []) { + const manifestPath = pkg.manifest_path.split(sep).join("/"); + if (manifestPath.includes("/software/")) { + fail(`main Cargo workspace claims colocated crate ${manifestPath}`); + } + } +} catch (error) { + fail(`cargo metadata failed for root workspace: ${error.message}`); +} + +if (failures.length > 0) { + for (const failure of failures) { + console.error(`check-layout: ${failure}`); + } + process.exit(1); +} + +console.log( + `check-layout: OK (${colocatedCargoTomls.length} colocated Cargo.toml files checked)`, +); diff --git a/scripts/ci.sh b/scripts/ci.sh index ffa6b8fbf0..e65118086e 100755 --- a/scripts/ci.sh +++ b/scripts/ci.sh @@ -33,6 +33,7 @@ run_step node --test scripts/check-rust-package-metadata.test.mjs run_step node scripts/check-rust-package-metadata.mjs run_step node --test scripts/check-agentos-client-protocol-compat.test.mjs run_step node scripts/check-agentos-client-protocol-compat.mjs +run_step pnpm check-layout if [[ -f scripts/check-registry-test-runtime-boundary.test.mjs ]]; then run_step node --test scripts/check-registry-test-runtime-boundary.test.mjs run_step node scripts/check-registry-test-runtime-boundary.mjs diff --git a/scripts/publish/src/lib/packages.ts b/scripts/publish/src/lib/packages.ts index 5969746863..6860e23f96 100644 --- a/scripts/publish/src/lib/packages.ts +++ b/scripts/publish/src/lib/packages.ts @@ -161,7 +161,7 @@ export function discoverPackages( } } - // 2. pnpm workspace packages. Skip the registry/software/* WASM command + // 2. pnpm workspace packages. Skip the software/* WASM command // packages. They are built and shipped separately, never published to npm // from this flow. const pnpmList = execSync("pnpm -r list --json --depth -1", { @@ -183,11 +183,11 @@ export function discoverPackages( ) { continue; } - // registry/software packages version independently and publish from + // software packages version independently and publish from // local via the toolchain — EXCEPT `common`, which core hard-depends on // and therefore ships on the main release track in lockstep. if ( - p.path.includes("/registry/software/") && + p.path.includes("/software/") && p.name !== "@agentos-software/common" ) { continue; diff --git a/scripts/secure-exec-agentos-map.json b/scripts/secure-exec-agentos-map.json index 5af919bb85..f257f17672 100644 --- a/scripts/secure-exec-agentos-map.json +++ b/scripts/secure-exec-agentos-map.json @@ -247,7 +247,7 @@ { "packageGlob": "@agentos-software/*", "sourcePathGlob": "secure-exec/registry/{agent,software}/*", - "targetPathGlob": "agentos/registry/{agent,software}/*" + "targetPathGlob": "agentos/software/*" } ], "scriptedReplacementOrder": [ diff --git a/scripts/verify-check-types.mjs b/scripts/verify-check-types.mjs index a2b7b4fdf6..69b85d4cab 100644 --- a/scripts/verify-check-types.mjs +++ b/scripts/verify-check-types.mjs @@ -27,7 +27,7 @@ const found = execSync( '-not -path "*/.cache/*"', '-not -path "*/.turbo/*"', '-not -path "*/vendor/*"', - '-not -path "./registry/tests/projects/*"', + '-not -path "./packages/runtime-core/tests/integration/projects/*"', '-not -path "./crates/execution/assets/undici-shims/*"', ].join(" "), { encoding: "utf8", cwd: root }, diff --git a/scripts/verify-fixed-versions.mjs b/scripts/verify-fixed-versions.mjs index c6bc103fff..0e0933777e 100644 --- a/scripts/verify-fixed-versions.mjs +++ b/scripts/verify-fixed-versions.mjs @@ -35,7 +35,6 @@ function isExcluded(relPath) { if (relPath === "node_modules" || relPath.startsWith("node_modules/")) return true; if (relPath === ".claude" || relPath.startsWith(".claude/")) return true; if (relPath === "scripts/publish" || relPath.startsWith("scripts/publish/")) return true; - if (relPath === "registry/tests" || relPath.startsWith("registry/tests/")) return true; return relPath .split("/") .some((part) => part === "fixtures" || part === "vendor" || part === "tests"); diff --git a/software/CONTRIBUTING.md b/software/CONTRIBUTING.md new file mode 100644 index 0000000000..ac86bfa6c5 --- /dev/null +++ b/software/CONTRIBUTING.md @@ -0,0 +1,94 @@ +# Contributing a Registry Package + +Software and agent packages for agentOS VMs, published under the +`@agentos-software/*` npm scope. This is the quick path to adding one; the +full documentation lives on the website: + +- [Software Definition](https://agentos-sdk.dev/docs/custom-software/definition) — package anatomy and manifest fields +- [Building Binaries](https://agentos-sdk.dev/docs/custom-software/building-wasm) — compiling commands to WASM +- [Publishing Packages](https://agentos-sdk.dev/docs/custom-software/publishing) — shipping to npm with the toolchain + +## File structure + +``` +software// WASM command packages and JavaScript agent adapters +toolchain/ shared build infrastructure for command binaries +``` + +Each package contains: + +``` +software// +├── package.json name, per-package semver, build script +├── agentos-package.json runtime manifest (commands/aliases/provides) + +│ `registry` block (title/description/priority/image) +│ that lists the package on agentos-sdk.dev/registry +├── src/index.ts descriptor export consumed by `software: []` +├── bin/ staged binaries (gitignored, built) +└── dist/package/ assembled runtime dir shipped in the npm tarball +``` + +## Building + +From the repo root: + +```bash +just toolchain-build # compile the fast native wasm command gate +just toolchain-cmd # build one command (required for git, duckdb, vim, codex) +just software-build # stage bin/ + assemble dist/package/ +pnpm --filter './software/*' test +``` + +See [Building Binaries](https://agentos-sdk.dev/docs/custom-software/building-wasm) +for toolchain details (Rust vs C builds, the patched WASI sysroot). + +## Adding a package + +1. Copy an existing package of the same kind (`software/jq` is a minimal + example) to `software//`. +2. Add the command source under `software//native/` (Rust: + `native/crates/cmd-`; C: `native/c/.c`). +3. Fill in `agentos-package.json`: `commands` (and `aliases`/`provides` if + needed) plus a `registry` block with `title` and `description` — without + that block the package is not listed on the website registry page. +4. Register the directory in `pnpm-workspace.yaml` (it is covered by the + `software/*` glob) and run `pnpm install`. +5. `just software-build `. + +See [Software Definition](https://agentos-sdk.dev/docs/custom-software/definition) +for every manifest field. + +## Testing in an external project + +Inside this repo, tests and examples resolve packages via `workspace:*` — no +publishing needed. To try a package in an external project, pack the built +tarball and install it by path: + +```bash +cd software/ +npm pack # produces agentos-software--.tgz +cd /path/to/your-project +npm install /path/to/agentos-software--.tgz +``` + +Then register it in your VM and run a command: + +```typescript +import myPkg from "@agentos-software/"; +const vm = agentOS({ software: [myPkg] }); +``` + +Real publishes go through `agentos-toolchain publish` (dist-tag `dev` by +default) — see [Publishing Packages](https://agentos-sdk.dev/docs/custom-software/publishing). + +## Opening a PR + +- Branch, commit with a plain conventional-commit title + (`feat(software): add package`), no agent attribution. +- Include: the package directory, the native build wiring, and the + `registry` block so the website picks it up. +- Keep the package version at its own semver (packages version + independently); never touch other packages' versions or the `latest` + dist-tag. +- Cheap gates before pushing: `cargo check --workspace`, `pnpm build`, + `pnpm check-types`, and `just software-build `. diff --git a/software/README.md b/software/README.md new file mode 100644 index 0000000000..d104021d1e --- /dev/null +++ b/software/README.md @@ -0,0 +1,129 @@ +# agentOS Software Catalog + +Software packages for secure-exec VMs: WASM command binaries +and JavaScript agent adapters live together under `software/*`. Everything in +this catalog publishes under the `@agentos-software/*` npm scope. + +## Consuming packages + +```bash +npm install @agentos-software/coreutils @agentos-software/grep +# or a meta-package for a complete set: +npm install @agentos-software/common +``` + +Each package default-exports a descriptor whose `packageDir` points at the +self-contained runtime dir the sidecar projects under +`/opt/agentos//` (meta-packages export an array of descriptors): + +```typescript +import coreutils from "@agentos-software/coreutils"; +import grep from "@agentos-software/grep"; + +export const software = [coreutils, grep]; +``` + +## Package anatomy + +``` +software// +├── package.json name, per-package semver version, build script +├── agentos-package.json manifest: runtime fields (name/agent/provides) + +│ staging fields (commands/aliases/stubs) +├── src/index.ts descriptor: packageDir -> ./package/ (dist/package) +├── bin/ staged command binaries (gitignored, built) +└── dist/package/ the assembled runtime dir (shipped in the npm tarball): + ├── package.json { name, version, bin: { : "bin/" } } + ├── agentos-package.json + └── bin/ the binaries, copied verbatim +``` + +The whole lifecycle is owned by **`@rivet-dev/agentos-toolchain`** +(`packages/agentos-toolchain`) — the same CLI 3rd-party repos use to build and +publish their own agentOS packages (`npx @rivet-dev/agentos-toolchain`): + +- `stage --commands-dir ` — populate `bin/` from a compiled commands + directory, per the `commands` / `aliases` / `stubs` lists in + `agentos-package.json`. +- `build` — assemble the clean `dist/package/` runtime dir from `bin/`. +- `pack` — build a self-contained node-closure package (JS agents). +- `publish` — publish to npm; dist-tag `dev` by default, `latest` only with an + explicit `--latest`. + +## Building + +All recipes run from the repo root (see `justfile`): + +```bash +just toolchain-build # compile the fast native wasm command gate +just toolchain-cmd # build ONE command binary, whatever its toolchain +just software-build # stage + assemble every software package +just software-build coreutils # ... or just one +pnpm --filter './software/*' test +``` + +`toolchain-cmd` (= `make -C toolchain cmd/`) is the uniform +per-binary entry point; it dispatches to whichever toolchain owns the command: + +| kind | commands | what it runs | +|---|---|---| +| Rust | any `software//native/crates/cmd-` command crate (sh, ls, rg, …) | `cargo build -p cmd-` (build-std) + `wasm-opt` | +| C | `zip unzip envsubst sqlite3 curl wget duckdb` | `make -C c sysroot build/` + per-command install | +| codex | `codex`, `codex-exec` | the codex fork build (needs the fork checkout) | +| C | `vim` (pinned upstream clone + bridge in `c/vim/`) | `make -C c sysroot build/vim` + install | + +The default native build (`toolchain`) compiles the fast command gate to +`wasm32-wasip1` with a patched std (`-Z build-std`, `toolchain/std-patches/`), runs +`wasm-opt -O3`, and drops the binaries in +`toolchain/target/wasm32-wasip1/release/commands/`. The bulk gate +intentionally excludes slow/heavy or non-default commands: `git`, `duckdb`, +`vim` and the external `codex`/`codex-exec` fork build. Build those explicitly with +`just toolchain-cmd ` when working on them. Package builds then run +`agentos-toolchain stage` (with `--if-missing skip`, so a checkout without the +native build still assembles valid empty placeholders) followed by `tsc` and +`agentos-toolchain build`. + +Within this repo, everything consumes the LOCAL builds by default: the software +packages are pnpm workspace members, so tests and examples resolve them via +`workspace:*` — no publish needed for local development. + +Exceptions: +- `software/codex/wasm/` is the install target of the codex fork's build + (`make -C toolchain codex`); `software/codex-cli` stages from it. +- C-built commands (sqlite3, zip, unzip, curl, wget, duckdb) need the patched + sysroot; `just toolchain-cmd ` builds it on demand. Without it + those packages stay empty placeholders. +- `vim` builds from source: `just toolchain-cmd vim` clones the pinned + vim tag and compiles it against the patched sysroot + the termios/termcap + bridge in `software/vim/native/c/vim-bridge/` (its runtime tree is staged by the + package `scripts/stage-runtime.mjs` and applied via manifest `provides`). + +## Publishing + +Packages **version independently** (per-package semver in each +`package.json`). Publishing NEVER moves the `latest` dist-tag unless asked: + +```bash +just registry-publish coreutils # publish @agentos-software/coreutils under dist-tag `dev` +just registry-publish coreutils my-branch # ... under a custom tag +just registry-publish coreutils latest # DELIBERATE release: moves `latest` +just registry-publish-all # every built software package, dist-tag `dev` +``` + +Bump the package's `version` in its `package.json` (commit it) before +publishing. CI does not publish these packages (the publish workflow's package +discovery skips `@agentos-software/*` except the manifest); the agent packages +under `software/*` preview-publish via `.github/workflows/publish.yaml` +under a branch dist-tag. + +agent-os consumes the published packages pinned per-package in its catalog +(`just agentos-pkgs-status` there), and flips to these local checkouts with +`just agentos-pkgs-local`. + +## Contributing + +See [CONTRIBUTING.md](CONTRIBUTING.md) for how to add new packages. + +## License + +Apache-2.0 diff --git a/software/browserbase/agentos-package.json b/software/browserbase/agentos-package.json new file mode 100644 index 0000000000..3f53841f14 --- /dev/null +++ b/software/browserbase/agentos-package.json @@ -0,0 +1,13 @@ +{ + "name": "browserbase", + "registry": { + "title": "Browserbase", + "description": "The Browserbase `browse` CLI: let agents browse the web with a cloud browser, no sandbox required.", + "beta": true, + "types": [ + "browser" + ], + "image": "/images/registry/browserbase.svg", + "category": "browser" + } +} diff --git a/software/browserbase/package.json b/software/browserbase/package.json new file mode 100644 index 0000000000..18f1b8a84b --- /dev/null +++ b/software/browserbase/package.json @@ -0,0 +1,40 @@ +{ + "name": "@agentos-software/browserbase", + "version": "0.3.3", + "type": "module", + "license": "Apache-2.0", + "description": "Browserbase browse CLI for secure-exec VMs (cloud browser automation)", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "bin": { + "browse": "./node_modules/browse/bin/run.js" + }, + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js", + "default": "./dist/index.js" + } + }, + "files": [ + "dist", + "!dist/package", + "!dist/package.tar" + ], + "scripts": { + "build": "tsc && rm -rf dist/package dist/package.tar && agentos-toolchain pack . --out dist/package --prune-native", + "check-types": "tsc --noEmit", + "test": "vitest run test/ --passWithNoTests" + }, + "dependencies": { + "browse": "^0.9.3" + }, + "devDependencies": { + "@agentos-software/manifest": "workspace:*", + "@rivet-dev/agentos-toolchain": "workspace:*", + "@types/node": "^22.10.2", + "typescript": "^5.7.2", + "@agentos/test-harness": "workspace:*", + "vitest": "^2.1.9" + } +} diff --git a/registry/agent/claude/src/index.ts b/software/browserbase/src/index.ts similarity index 100% rename from registry/agent/claude/src/index.ts rename to software/browserbase/src/index.ts diff --git a/software/browserbase/tsconfig.json b/software/browserbase/tsconfig.json new file mode 100644 index 0000000000..03ce790ab7 --- /dev/null +++ b/software/browserbase/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*"] +} diff --git a/software/build-essential/agentos-package.json b/software/build-essential/agentos-package.json new file mode 100644 index 0000000000..e2c0382a37 --- /dev/null +++ b/software/build-essential/agentos-package.json @@ -0,0 +1,8 @@ +{ + "registry": { + "title": "Build Essential", + "description": "Meta-package: common + git + curl.", + "priority": 100, + "category": "meta" + } +} diff --git a/software/build-essential/package.json b/software/build-essential/package.json new file mode 100644 index 0000000000..2002836894 --- /dev/null +++ b/software/build-essential/package.json @@ -0,0 +1,37 @@ +{ + "name": "@agentos-software/build-essential", + "version": "0.3.0-rc.2", + "type": "module", + "license": "Apache-2.0", + "description": "Build-essential WASM command set for secure-exec VMs (common + git + curl)", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "files": [ + "dist", + "!dist/package", + "!dist/package.tar" + ], + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "scripts": { + "build": "tsc", + "check-types": "tsc --noEmit", + "test": "vitest run test/ --passWithNoTests" + }, + "dependencies": { + "@agentos-software/common": "workspace:*", + "@agentos-software/curl": "workspace:*", + "@agentos-software/git": "workspace:*" + }, + "devDependencies": { + "@agentos-software/manifest": "workspace:*", + "@types/node": "^22.10.2", + "typescript": "^5.9.2", + "@agentos/test-harness": "workspace:*", + "vitest": "^2.1.9" + } +} diff --git a/registry/software/build-essential/src/index.ts b/software/build-essential/src/index.ts similarity index 100% rename from registry/software/build-essential/src/index.ts rename to software/build-essential/src/index.ts diff --git a/software/build-essential/tsconfig.json b/software/build-essential/tsconfig.json new file mode 100644 index 0000000000..03ce790ab7 --- /dev/null +++ b/software/build-essential/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*"] +} diff --git a/software/claude/agentos-package.json b/software/claude/agentos-package.json new file mode 100644 index 0000000000..27f8b1c5ea --- /dev/null +++ b/software/claude/agentos-package.json @@ -0,0 +1,36 @@ +{ + "name": "claude", + "agent": { + "acpEntrypoint": "claude-sdk-acp", + "env": { + "CLAUDE_AGENT_SDK_CLIENT_APP": "@rivet-dev/agentos", + "CLAUDE_CODE_SIMPLE": "1", + "CLAUDE_CODE_FORCE_AGENT_OS_RIPGREP": "1", + "CLAUDE_CODE_DEFER_GROWTHBOOK_INIT": "1", + "CLAUDE_CODE_DISABLE_CWD_PERSIST": "1", + "CLAUDE_CODE_DISABLE_DEV_NULL_REDIRECT": "1", + "CLAUDE_CODE_NODE_SHELL_WRAPPER": "1", + "CLAUDE_CODE_DISABLE_STREAM_JSON_HOOK_EVENTS": "1", + "CLAUDE_CODE_SHELL": "/bin/sh", + "CLAUDE_CODE_SKIP_INITIAL_MESSAGES": "1", + "CLAUDE_CODE_SKIP_SANDBOX_INIT": "1", + "CLAUDE_CODE_SIMPLE_SHELL_EXEC": "1", + "CLAUDE_CODE_SWAP_STDIO": "0", + "CLAUDE_CODE_USE_PIPE_OUTPUT": "1", + "DISABLE_TELEMETRY": "1", + "SHELL": "/bin/sh", + "USE_BUILTIN_RIPGREP": "0" + } + }, + "registry": { + "slug": "claude-code", + "title": "Claude Code", + "description": "Run Claude Code as an agentOS agent with full tool access, file editing, and shell execution.", + "beta": true, + "docsHref": "/docs/agents/claude", + "image": "/images/registry/claude-code.svg", + "priority": 90, + "category": "agents" + }, + "kind": "agent" +} diff --git a/software/claude/package.json b/software/claude/package.json new file mode 100644 index 0000000000..de72c01931 --- /dev/null +++ b/software/claude/package.json @@ -0,0 +1,42 @@ +{ + "name": "@agentos-software/claude-code", + "version": "0.2.1", + "type": "module", + "license": "Apache-2.0", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "bin": { + "claude": "./dist/claude-cli.mjs", + "claude-sdk-acp": "./dist/adapter.js" + }, + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js", + "default": "./dist/index.js" + } + }, + "files": [ + "dist", + "!dist/package", + "!dist/package.tar", + "agentos-package.json" + ], + "scripts": { + "build": "tsc && node ./scripts/build-patched-cli.mjs && rm -rf dist/package && agentos-toolchain pack . --out dist/package --agent claude-sdk-acp --prune-native", + "check-types": "tsc --noEmit", + "test": "pnpm build && node --test --test-force-exit tests/*.test.mjs" + }, + "dependencies": { + "@agentclientprotocol/sdk": "^0.16.1", + "@anthropic-ai/claude-agent-sdk": "0.2.87", + "zod": "^4.1.11" + }, + "devDependencies": { + "@agentos-software/manifest": "workspace:*", + "@rivet-dev/agentos-toolchain": "workspace:*", + "@types/node": "^22.10.2", + "typescript": "^5.7.2", + "@agentos/test-harness": "workspace:*" + } +} diff --git a/registry/agent/claude/scripts/build-patched-cli.mjs b/software/claude/scripts/build-patched-cli.mjs similarity index 100% rename from registry/agent/claude/scripts/build-patched-cli.mjs rename to software/claude/scripts/build-patched-cli.mjs diff --git a/registry/agent/claude/src/adapter.ts b/software/claude/src/adapter.ts similarity index 100% rename from registry/agent/claude/src/adapter.ts rename to software/claude/src/adapter.ts diff --git a/registry/agent/opencode/src/index.ts b/software/claude/src/index.ts similarity index 100% rename from registry/agent/opencode/src/index.ts rename to software/claude/src/index.ts diff --git a/registry/agent/claude/src/patched-cli.ts b/software/claude/src/patched-cli.ts similarity index 100% rename from registry/agent/claude/src/patched-cli.ts rename to software/claude/src/patched-cli.ts diff --git a/registry/agent/claude/tests/adapter.test.mjs b/software/claude/tests/adapter.test.mjs similarity index 100% rename from registry/agent/claude/tests/adapter.test.mjs rename to software/claude/tests/adapter.test.mjs diff --git a/registry/agent/claude/tests/patched-cli.test.mjs b/software/claude/tests/patched-cli.test.mjs similarity index 100% rename from registry/agent/claude/tests/patched-cli.test.mjs rename to software/claude/tests/patched-cli.test.mjs diff --git a/software/claude/tsconfig.json b/software/claude/tsconfig.json new file mode 100644 index 0000000000..73a06ddbdb --- /dev/null +++ b/software/claude/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "declaration": true, + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist"] +} diff --git a/software/codex-cli/agentos-package.json b/software/codex-cli/agentos-package.json new file mode 100644 index 0000000000..ad23a92b67 --- /dev/null +++ b/software/codex-cli/agentos-package.json @@ -0,0 +1,12 @@ +{ + "commands": [ + "codex", + "codex-exec" + ], + "registry": { + "title": "Codex CLI", + "description": "OpenAI Codex CLI integration.", + "image": "/images/registry/codex.svg", + "category": "agents" + } +} diff --git a/software/codex-cli/native/crates/cmd-codex-exec/Cargo.toml b/software/codex-cli/native/crates/cmd-codex-exec/Cargo.toml new file mode 100644 index 0000000000..eeca177f14 --- /dev/null +++ b/software/codex-cli/native/crates/cmd-codex-exec/Cargo.toml @@ -0,0 +1,18 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-codex-exec" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "codex-exec command binary for secure-exec VM" + +[[bin]] +name = "codex-exec" +path = "src/main.rs" + +[dependencies] +wasi-http = { package = "secureexec-wasi-http", path = "../../../../../toolchain/crates/libs/wasi-http" } + +# WASI stub crates for future codex-core dependencies that don't support wasm32-wasip1. +codex-network-proxy = "0.0.0" +codex-otel = "0.0.0" diff --git a/registry/native/crates/commands/codex-exec/src/main.rs b/software/codex-cli/native/crates/cmd-codex-exec/src/main.rs similarity index 100% rename from registry/native/crates/commands/codex-exec/src/main.rs rename to software/codex-cli/native/crates/cmd-codex-exec/src/main.rs diff --git a/software/codex-cli/native/crates/cmd-codex/Cargo.toml b/software/codex-cli/native/crates/cmd-codex/Cargo.toml new file mode 100644 index 0000000000..cc73f06a16 --- /dev/null +++ b/software/codex-cli/native/crates/cmd-codex/Cargo.toml @@ -0,0 +1,34 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-codex" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "codex standalone binary for secure-exec VM" + +[[bin]] +name = "codex" +path = "src/main.rs" + +# The full codex-exec dependency from rivet-dev/codex (wasi-support branch) +# will be wired in when the vendoring blocker is resolved. +# +# The fork at github.com/rivet-dev/codex already has cfg(target_os = "wasi") +# gates on codex-core (portable-pty, network-proxy, sandbox deps) and +# codex-exec (all platform deps gated behind non-WASI). +# +# wasi-spawn provides the WasiChild abstraction that will replace +# tokio::process::Command in codex-core when compiled for WASI. + +[dependencies] +wasi-spawn = { package = "secureexec-wasi-spawn", path = "../../../../../toolchain/crates/libs/wasi-spawn" } +wasi-http = { package = "secureexec-wasi-http", path = "../../../../../toolchain/crates/libs/wasi-http" } + +# TUI dependencies — ratatui for rendering, crossterm for terminal backend +# crossterm is patched for WASI support (see std-patches/crates/crossterm/) +ratatui = { version = "0.29", default-features = false, features = ["crossterm"] } + +# WASI stub crates for codex-core dependencies that don't support wasm32-wasip1. +# These provide zero-size/no-op implementations so codex-core compiles for WASI. +codex-network-proxy = "0.0.0" +codex-otel = "0.0.0" diff --git a/software/codex-cli/native/crates/cmd-codex/src/main.rs b/software/codex-cli/native/crates/cmd-codex/src/main.rs new file mode 100644 index 0000000000..88f5d80c09 --- /dev/null +++ b/software/codex-cli/native/crates/cmd-codex/src/main.rs @@ -0,0 +1,337 @@ +/// Codex TUI for secure-exec VM. +/// +/// Full terminal UI using ratatui + crossterm backend, rendering through +/// the WasmVM PTY. This is the interactive entry point — for headless +/// (scriptable) usage, see codex-exec. +/// +/// Uses wasi-spawn for process spawning via host_process FFI and +/// wasi-http for HTTP/HTTPS requests via host_net TCP/TLS imports. +/// +/// crossterm is patched for WASI support (see std-patches/crates/crossterm/): +/// - Terminal raw mode tracked in-process (host PTY handles discipline) +/// - Terminal size from COLUMNS/LINES env vars +/// - Event source reads stdin directly and parses ANSI escape sequences +/// - IsTty returns true (PTY slave FDs are terminals) +use std::io; + +use ratatui::{ + backend::CrosstermBackend, + crossterm::{ + event::{self, Event, KeyCode, KeyEvent, KeyModifiers}, + execute, + terminal::{self, EnterAlternateScreen, LeaveAlternateScreen}, + }, + layout::{Constraint, Layout}, + style::{Color, Modifier, Style}, + text::{Line, Span, Text}, + widgets::{Block, Borders, Paragraph, Wrap}, + Frame, Terminal, +}; + +// Validate WASI stub crates compile by referencing key types +use codex_network_proxy::NetworkProxy; +use codex_otel::SessionTelemetry; + +const VERSION: &str = env!("CARGO_PKG_VERSION"); +const MAX_INPUT_CHARS: usize = 8192; +const MAX_MESSAGES: usize = 200; + +fn main() { + let args: Vec = std::env::args().collect(); + + // Handle --help + if args.iter().any(|a| a == "--help" || a == "-h") { + print_help(); + return; + } + + // Handle --version + if args.iter().any(|a| a == "--version" || a == "-V") { + println!("codex {}", VERSION); + return; + } + + // Handle --model flag (accept but note it's stored for future use) + let model = args + .windows(2) + .find(|w| w[0] == "--model") + .map(|w| w[1].clone()); + + // Built-in HTTP test subcommand (validates wasi-http integration) + if args.get(1).map(|s| s.as_str()) == Some("--http-test") { + return http_test(&args[2..]); + } + + // Stub validation subcommand (validates WASI stub crates) + if args.get(1).map(|s| s.as_str()) == Some("--stub-test") { + return stub_test(); + } + + // Run the TUI + if let Err(e) = run_tui(model.as_deref()) { + eprintln!("codex: TUI error: {}", e); + std::process::exit(1); + } +} + +/// Main TUI event loop using ratatui + crossterm. +fn run_tui(model: Option<&str>) -> io::Result<()> { + let _terminal_guard = TerminalGuard::enter()?; + + let stdout = io::stdout(); + let backend = CrosstermBackend::new(stdout); + let mut terminal = Terminal::new(backend)?; + + // App state + let mut input = String::new(); + let mut messages: Vec = Vec::new(); + let mut should_quit = false; + + // Draw initial frame + terminal.draw(|f| draw_ui(f, &input, &messages, model))?; + + // Event loop + while !should_quit { + match event::read()? { + Event::Key(key) => { + handle_key_event(key, &mut input, &mut messages, &mut should_quit); + terminal.draw(|f| draw_ui(f, &input, &messages, model))?; + } + Event::Resize(_, _) => { + terminal.draw(|f| draw_ui(f, &input, &messages, model))?; + } + _ => {} + } + } + + Ok(()) +} + +/// Handle a key event, updating app state. +fn handle_key_event( + key: KeyEvent, + input: &mut String, + messages: &mut Vec, + should_quit: &mut bool, +) { + match (key.code, key.modifiers) { + // Ctrl+C quits + (KeyCode::Char('c'), m) if m.contains(KeyModifiers::CONTROL) => { + *should_quit = true; + } + // 'q' on empty input quits + (KeyCode::Char('q'), KeyModifiers::NONE) if input.is_empty() => { + *should_quit = true; + } + // Enter submits the input + (KeyCode::Enter, _) => { + if !input.is_empty() { + let prompt = input.clone(); + push_message(messages, format!("> {}", prompt)); + push_message( + messages, + "codex: agent loop is under development".to_string(), + ); + push_message( + messages, + format!("codex: prompt received ({} chars)", prompt.len()), + ); + input.clear(); + } + } + // Backspace deletes last char + (KeyCode::Backspace, _) => { + input.pop(); + } + // Esc clears input + (KeyCode::Esc, _) => { + input.clear(); + } + // Regular character input + (KeyCode::Char(c), _) => { + if input.chars().count() < MAX_INPUT_CHARS { + input.push(c); + } + } + _ => {} + } +} + +fn push_message(messages: &mut Vec, message: String) { + messages.push(message); + if messages.len() > MAX_MESSAGES { + messages.drain(..messages.len() - MAX_MESSAGES); + } +} + +struct TerminalGuard { + raw_mode_enabled: bool, + alternate_screen_enabled: bool, +} + +impl TerminalGuard { + fn enter() -> io::Result { + terminal::enable_raw_mode()?; + + if let Err(error) = execute!(io::stdout(), EnterAlternateScreen) { + let _ = terminal::disable_raw_mode(); + return Err(error); + } + + Ok(Self { + raw_mode_enabled: true, + alternate_screen_enabled: true, + }) + } +} + +impl Drop for TerminalGuard { + fn drop(&mut self) { + if self.alternate_screen_enabled { + let _ = execute!(io::stdout(), LeaveAlternateScreen); + self.alternate_screen_enabled = false; + } + + if self.raw_mode_enabled { + let _ = terminal::disable_raw_mode(); + self.raw_mode_enabled = false; + } + } +} + +/// Draw the TUI layout. +fn draw_ui(f: &mut Frame, input: &str, messages: &[String], model: Option<&str>) { + let area = f.area(); + + let chunks = Layout::vertical([ + Constraint::Length(3), // Header + Constraint::Min(5), // Messages + Constraint::Length(3), // Input + ]) + .split(area); + + // Header + let model_text = model.unwrap_or("default"); + let header = Paragraph::new(Line::from(vec![ + Span::styled( + "Codex ", + Style::default() + .fg(Color::Cyan) + .add_modifier(Modifier::BOLD), + ), + Span::styled(VERSION, Style::default().fg(Color::DarkGray)), + Span::raw(" "), + Span::styled( + format!("model: {}", model_text), + Style::default().fg(Color::Yellow), + ), + Span::raw(" "), + Span::styled("WasmVM", Style::default().fg(Color::Green)), + ])) + .block(Block::default().borders(Borders::ALL).title(" codex ")); + f.render_widget(header, chunks[0]); + + // Messages area + let msg_lines: Vec = messages + .iter() + .map(|m| { + if m.starts_with("> ") { + Line::from(Span::styled(m.as_str(), Style::default().fg(Color::Cyan))) + } else { + Line::from(Span::raw(m.as_str())) + } + }) + .collect(); + + let welcome = if messages.is_empty() { + vec![ + Line::from(""), + Line::from(Span::styled( + "Welcome to Codex on WasmVM!", + Style::default() + .fg(Color::Cyan) + .add_modifier(Modifier::BOLD), + )), + Line::from(""), + Line::from("Type a prompt and press Enter to submit."), + Line::from("Press 'q' (on empty input) or Ctrl+C to exit."), + ] + } else { + msg_lines + }; + + let messages_widget = Paragraph::new(Text::from(welcome)) + .block(Block::default().borders(Borders::ALL).title(" messages ")) + .wrap(Wrap { trim: false }); + f.render_widget(messages_widget, chunks[1]); + + // Input area + let input_widget = Paragraph::new(Line::from(vec![ + Span::styled("❯ ", Style::default().fg(Color::Green)), + Span::raw(input), + ])) + .block(Block::default().borders(Borders::ALL).title(" input ")); + f.render_widget(input_widget, chunks[2]); +} + +fn print_help() { + println!( + "codex {} — interactive Codex TUI for secure-exec VM", + VERSION + ); + println!(); + println!("USAGE:"); + println!(" codex [OPTIONS]"); + println!(); + println!("OPTIONS:"); + println!(" -h, --help Print this help message"); + println!(" -V, --version Print version information"); + println!(" --model MODEL Select model for completions"); + println!(" --http-test URL Test HTTP client via host_net"); + println!(" --stub-test Validate WASI stub crates"); + println!(); + println!("DESCRIPTION:"); + println!(" Interactive TUI for the Codex agent, rendered through"); + println!(" the WasmVM PTY using ratatui + crossterm backend."); + println!(" For headless mode, use codex-exec instead."); +} + +fn stub_test() { + let proxy = NetworkProxy; + let mut env = std::collections::HashMap::new(); + proxy.apply_to_env(&mut env); + println!("network-proxy: NetworkProxy is zero-size, apply_to_env is no-op"); + + let telemetry = SessionTelemetry::new(); + telemetry.counter("test.counter", 1, &[]); + telemetry.histogram("test.histogram", 42, &[]); + println!("otel: SessionTelemetry metrics are no-ops"); + + let global = codex_otel::metrics::global(); + assert!(global.is_none(), "global metrics should be None on WASI"); + println!("otel: global() returns None (no exporter on WASI)"); + + println!("stub-test: all stubs validated successfully"); +} + +fn http_test(args: &[String]) { + if args.is_empty() { + eprintln!("usage: codex --http-test "); + std::process::exit(1); + } + + let url = &args[0]; + match wasi_http::get(url) { + Ok(resp) => { + println!("status: {}", resp.status); + match resp.text() { + Ok(body) => println!("body: {}", body), + Err(e) => eprintln!("body decode error: {}", e), + } + } + Err(e) => { + eprintln!("http error: {}", e); + std::process::exit(1); + } + } +} diff --git a/software/codex-cli/package.json b/software/codex-cli/package.json new file mode 100644 index 0000000000..717d9087f8 --- /dev/null +++ b/software/codex-cli/package.json @@ -0,0 +1,33 @@ +{ + "name": "@agentos-software/codex-cli", + "version": "0.3.3", + "type": "module", + "license": "Apache-2.0", + "description": "OpenAI Codex command package for secure-exec VMs", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "files": [ + "dist", + "!dist/package", + "!dist/package.tar" + ], + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "scripts": { + "build": "agentos-toolchain stage --commands-dir ../codex/wasm --if-missing skip && tsc && agentos-toolchain build", + "check-types": "tsc --noEmit", + "test": "vitest run test/ --passWithNoTests" + }, + "devDependencies": { + "@agentos-software/manifest": "workspace:*", + "@rivet-dev/agentos-toolchain": "workspace:*", + "@types/node": "^22.10.2", + "typescript": "^5.9.2", + "@agentos/test-harness": "workspace:*", + "vitest": "^2.1.9" + } +} diff --git a/registry/agent/pi-cli/src/index.ts b/software/codex-cli/src/index.ts similarity index 100% rename from registry/agent/pi-cli/src/index.ts rename to software/codex-cli/src/index.ts diff --git a/registry/tests/wasmvm/codex-exec.test.ts b/software/codex-cli/test/codex-exec.test.ts similarity index 94% rename from registry/tests/wasmvm/codex-exec.test.ts rename to software/codex-cli/test/codex-exec.test.ts index 1d3da03c42..2581c85fc2 100644 --- a/registry/tests/wasmvm/codex-exec.test.ts +++ b/software/codex-cli/test/codex-exec.test.ts @@ -14,9 +14,16 @@ */ import { describe, it, expect, afterEach } from 'vitest'; -import { createWasmVmRuntime } from '../helpers.js'; -import { COMMANDS_DIR, createKernel, describeIf, hasWasmBinaries } from '../helpers.js'; -import type { Kernel } from '../helpers.js'; +import { existsSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { createWasmVmRuntime } from '@agentos/test-harness'; +import { COMMANDS_DIR, createKernel, describeIf, hasWasmBinaries } from '@agentos/test-harness'; +import type { Kernel } from '@agentos/test-harness'; + +const hasCodexExec = + process.env.AGENTOS_CODEX_WASM_E2E === '1' && + hasWasmBinaries && + existsSync(resolve(COMMANDS_DIR, 'codex-exec')); // Minimal in-memory VFS for kernel tests class SimpleVFS { @@ -114,8 +121,8 @@ async function createTestKernel(): Promise<{ kernel: Kernel; vfs: SimpleVFS }> { return { kernel, vfs }; } -// TODO(P6): requires codex-exec WASM artifact from an external fork build, intentionally excluded from the fast registry-build gate. -describe.skip('codex-exec command (WasmVM)', { timeout: 30_000 }, () => { +// TODO(P6): requires codex-exec WASM artifact from an external fork build, intentionally excluded from the fast software-build gate. +describeIf(hasCodexExec, 'codex-exec command (WasmVM)', { timeout: 30_000 }, () => { let kernel: Kernel; afterEach(async () => { diff --git a/registry/tests/wasmvm/codex-tui.test.ts b/software/codex-cli/test/codex-tui.test.ts similarity index 91% rename from registry/tests/wasmvm/codex-tui.test.ts rename to software/codex-cli/test/codex-tui.test.ts index ea004eb986..1f1afef2d1 100644 --- a/registry/tests/wasmvm/codex-tui.test.ts +++ b/software/codex-cli/test/codex-tui.test.ts @@ -14,11 +14,17 @@ */ import { describe, it, expect, afterEach } from 'vitest'; -import { TerminalHarness } from './terminal-harness.js'; -import { createWasmVmRuntime } from '../helpers.js'; -import { COMMANDS_DIR, createKernel, describeIf, hasWasmBinaries } from '../helpers.js'; -import type { Kernel } from '../helpers.js'; - +import { existsSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { TerminalHarness } from '@agentos/test-harness'; +import { createWasmVmRuntime } from '@agentos/test-harness'; +import { COMMANDS_DIR, createKernel, describeIf, hasWasmBinaries } from '@agentos/test-harness'; +import type { Kernel } from '@agentos/test-harness'; + +const hasCodex = + process.env.AGENTOS_CODEX_WASM_E2E === '1' && + hasWasmBinaries && + existsSync(resolve(COMMANDS_DIR, 'codex')); const hasApiKey = !!process.env.OPENAI_API_KEY; /** brush-shell interactive prompt. */ @@ -124,8 +130,8 @@ async function createTestKernel(): Promise<{ kernel: Kernel; vfs: SimpleVFS }> { // Non-interactive tests (kernel.exec — --help bypasses TUI) // --------------------------------------------------------------------------- -// TODO(P6): requires codex WASM artifact from an external fork build, intentionally excluded from the fast registry-build gate. -describe.skip('codex TUI (WasmVM) - non-interactive', { timeout: 30_000 }, () => { +// TODO(P6): requires codex WASM artifact from an external fork build, intentionally excluded from the fast software-build gate. +describeIf(hasCodex, 'codex TUI (WasmVM) - non-interactive', { timeout: 30_000 }, () => { let kernel: Kernel; afterEach(async () => { @@ -155,8 +161,8 @@ describe.skip('codex TUI (WasmVM) - non-interactive', { timeout: 30_000 }, () => // Interactive TUI tests (PTY via TerminalHarness) // --------------------------------------------------------------------------- -// TODO(P6): requires codex WASM artifact from an external fork build, intentionally excluded from the fast registry-build gate. -describe.skip('codex TUI (WasmVM) - interactive', { timeout: 30_000 }, () => { +// TODO(P6): requires codex WASM artifact from an external fork build, intentionally excluded from the fast software-build gate. +describeIf(hasCodex, 'codex TUI (WasmVM) - interactive', { timeout: 30_000 }, () => { let kernel: Kernel; let harness: TerminalHarness; @@ -261,8 +267,8 @@ describe.skip('codex TUI (WasmVM) - interactive', { timeout: 30_000 }, () => { // API integration tests (gated behind OPENAI_API_KEY) // --------------------------------------------------------------------------- -// TODO(P6): requires codex WASM artifact from an external fork build, intentionally excluded from the fast registry-build gate. -describe.skip('codex TUI API integration (requires OPENAI_API_KEY)', { timeout: 60_000 }, () => { +// TODO(P6): requires codex WASM artifact from an external fork build, intentionally excluded from the fast software-build gate. +describeIf(hasCodex && hasApiKey, 'codex TUI API integration (requires OPENAI_API_KEY)', { timeout: 60_000 }, () => { let kernel: Kernel; let harness: TerminalHarness; diff --git a/software/codex-cli/tsconfig.json b/software/codex-cli/tsconfig.json new file mode 100644 index 0000000000..03ce790ab7 --- /dev/null +++ b/software/codex-cli/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*"] +} diff --git a/software/codex/agentos-package.json b/software/codex/agentos-package.json new file mode 100644 index 0000000000..0a05c9b778 --- /dev/null +++ b/software/codex/agentos-package.json @@ -0,0 +1,12 @@ +{ + "registry": { + "title": "Codex", + "description": "Run OpenAI's Codex coding agent inside agentOS with programmatic API access.", + "beta": true, + "docsHref": "/docs/agents/codex", + "image": "/images/registry/codex.svg", + "priority": 80, + "category": "agents" + }, + "kind": "agent" +} diff --git a/software/codex/package.json b/software/codex/package.json new file mode 100644 index 0000000000..ae83c6e51b --- /dev/null +++ b/software/codex/package.json @@ -0,0 +1,30 @@ +{ + "name": "@agentos-software/codex", + "version": "0.2.0-rc.3", + "type": "module", + "license": "Apache-2.0", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js", + "default": "./dist/index.js" + } + }, + "scripts": { + "prepare-deps": "pnpm --filter @agentos-software/codex-cli exec tsc", + "build": "pnpm prepare-deps && tsc", + "check-types": "tsc --noEmit", + "test": "pnpm build && node --test tests/*.test.mjs" + }, + "dependencies": { + "@agentos-software/codex-cli": "workspace:*" + }, + "devDependencies": { + "@agentos-software/manifest": "workspace:*", + "@types/node": "^22.10.2", + "typescript": "^5.7.2", + "@agentos/test-harness": "workspace:*" + } +} diff --git a/registry/agent/codex/src/index.ts b/software/codex/src/index.ts similarity index 100% rename from registry/agent/codex/src/index.ts rename to software/codex/src/index.ts diff --git a/registry/agent/codex/tests/package.test.mjs b/software/codex/tests/package.test.mjs similarity index 100% rename from registry/agent/codex/tests/package.test.mjs rename to software/codex/tests/package.test.mjs diff --git a/software/codex/tsconfig.json b/software/codex/tsconfig.json new file mode 100644 index 0000000000..73a06ddbdb --- /dev/null +++ b/software/codex/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "declaration": true, + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist"] +} diff --git a/software/common/agentos-package.json b/software/common/agentos-package.json new file mode 100644 index 0000000000..b8dc86c870 --- /dev/null +++ b/software/common/agentos-package.json @@ -0,0 +1,8 @@ +{ + "registry": { + "title": "Common", + "description": "Meta-package: coreutils + sed + grep + gawk + findutils + diffutils + tar + gzip.", + "priority": 95, + "category": "meta" + } +} diff --git a/software/common/package.json b/software/common/package.json new file mode 100644 index 0000000000..4e2113cb3b --- /dev/null +++ b/software/common/package.json @@ -0,0 +1,42 @@ +{ + "name": "@agentos-software/common", + "version": "0.3.0-rc.2", + "type": "module", + "license": "Apache-2.0", + "description": "Common WASM command set for secure-exec VMs (coreutils + sed + grep + gawk + findutils + diffutils + tar + gzip)", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "files": [ + "dist", + "!dist/package", + "!dist/package.tar" + ], + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "scripts": { + "build": "tsc", + "check-types": "tsc --noEmit", + "test": "vitest run test/ --passWithNoTests" + }, + "dependencies": { + "@agentos-software/coreutils": "workspace:*", + "@agentos-software/sed": "workspace:*", + "@agentos-software/grep": "workspace:*", + "@agentos-software/gawk": "workspace:*", + "@agentos-software/findutils": "workspace:*", + "@agentos-software/diffutils": "workspace:*", + "@agentos-software/tar": "workspace:*", + "@agentos-software/gzip": "workspace:*" + }, + "devDependencies": { + "@agentos-software/manifest": "workspace:*", + "@types/node": "^22.10.2", + "typescript": "^5.9.2", + "@agentos/test-harness": "workspace:*", + "vitest": "^2.1.9" + } +} diff --git a/registry/software/common/src/index.ts b/software/common/src/index.ts similarity index 100% rename from registry/software/common/src/index.ts rename to software/common/src/index.ts diff --git a/software/common/tsconfig.json b/software/common/tsconfig.json new file mode 100644 index 0000000000..03ce790ab7 --- /dev/null +++ b/software/common/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*"] +} diff --git a/software/coreutils/agentos-package.json b/software/coreutils/agentos-package.json new file mode 100644 index 0000000000..30fdb36450 --- /dev/null +++ b/software/coreutils/agentos-package.json @@ -0,0 +1,128 @@ +{ + "commands": [ + "sh", + "arch", + "b2sum", + "base32", + "base64", + "basename", + "basenc", + "cat", + "chmod", + "cksum", + "column", + "comm", + "cp", + "cut", + "date", + "dd", + "dircolors", + "dirname", + "du", + "echo", + "env", + "expand", + "expr", + "factor", + "false", + "fmt", + "fold", + "head", + "join", + "link", + "ln", + "logname", + "ls", + "md5sum", + "mkdir", + "mktemp", + "mv", + "nice", + "nl", + "nohup", + "nproc", + "numfmt", + "od", + "paste", + "pathchk", + "printenv", + "printf", + "ptx", + "pwd", + "readlink", + "realpath", + "rev", + "rm", + "rmdir", + "seq", + "sha1sum", + "sha224sum", + "sha256sum", + "sha384sum", + "sha512sum", + "shred", + "shuf", + "sleep", + "sort", + "split", + "stat", + "stdbuf", + "strings", + "sum", + "tac", + "tail", + "tee", + "test", + "timeout", + "touch", + "tr", + "true", + "truncate", + "tsort", + "uname", + "unexpand", + "uniq", + "unlink", + "wc", + "which", + "whoami", + "yes" + ], + "aliases": { + "bash": "sh", + "more": "cat", + "dir": "ls", + "vdir": "ls", + "[": "test" + }, + "stubs": [ + "chcon", + "runcon", + "chgrp", + "chown", + "chroot", + "df", + "groups", + "id", + "hostname", + "hostid", + "install", + "kill", + "mkfifo", + "mknod", + "pinky", + "who", + "users", + "uptime", + "stty", + "sync", + "tty" + ], + "registry": { + "title": "Coreutils", + "description": "sh, cat, ls, cp, mv, rm, sort, and 80+ essential POSIX commands.", + "priority": 45, + "image": "/images/registry/coreutils.svg", + "category": "core" + } +} diff --git a/software/coreutils/native/crates/cmd-arch/Cargo.toml b/software/coreutils/native/crates/cmd-arch/Cargo.toml new file mode 100644 index 0000000000..cb72652e83 --- /dev/null +++ b/software/coreutils/native/crates/cmd-arch/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-arch" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "arch standalone binary for secure-exec VM" + +[[bin]] +name = "arch" +path = "src/main.rs" + +[dependencies] +uu_arch = "0.7.0" diff --git a/registry/native/crates/commands/arch/src/main.rs b/software/coreutils/native/crates/cmd-arch/src/main.rs similarity index 100% rename from registry/native/crates/commands/arch/src/main.rs rename to software/coreutils/native/crates/cmd-arch/src/main.rs diff --git a/software/coreutils/native/crates/cmd-b2sum/Cargo.toml b/software/coreutils/native/crates/cmd-b2sum/Cargo.toml new file mode 100644 index 0000000000..3f5da391a6 --- /dev/null +++ b/software/coreutils/native/crates/cmd-b2sum/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-b2sum" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "b2sum standalone binary for secure-exec VM" + +[[bin]] +name = "b2sum" +path = "src/main.rs" + +[dependencies] +uu_b2sum = "0.7.0" diff --git a/registry/native/crates/commands/b2sum/src/main.rs b/software/coreutils/native/crates/cmd-b2sum/src/main.rs similarity index 100% rename from registry/native/crates/commands/b2sum/src/main.rs rename to software/coreutils/native/crates/cmd-b2sum/src/main.rs diff --git a/software/coreutils/native/crates/cmd-base32/Cargo.toml b/software/coreutils/native/crates/cmd-base32/Cargo.toml new file mode 100644 index 0000000000..29ea0a98a1 --- /dev/null +++ b/software/coreutils/native/crates/cmd-base32/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-base32" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "base32 standalone binary for secure-exec VM" + +[[bin]] +name = "base32" +path = "src/main.rs" + +[dependencies] +uu_base32 = "0.7.0" diff --git a/registry/native/crates/commands/base32/src/main.rs b/software/coreutils/native/crates/cmd-base32/src/main.rs similarity index 100% rename from registry/native/crates/commands/base32/src/main.rs rename to software/coreutils/native/crates/cmd-base32/src/main.rs diff --git a/software/coreutils/native/crates/cmd-base64/Cargo.toml b/software/coreutils/native/crates/cmd-base64/Cargo.toml new file mode 100644 index 0000000000..105af17592 --- /dev/null +++ b/software/coreutils/native/crates/cmd-base64/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-base64" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "base64 standalone binary for secure-exec VM" + +[[bin]] +name = "base64" +path = "src/main.rs" + +[dependencies] +uu_base64 = "0.7.0" diff --git a/registry/native/crates/commands/base64/src/main.rs b/software/coreutils/native/crates/cmd-base64/src/main.rs similarity index 100% rename from registry/native/crates/commands/base64/src/main.rs rename to software/coreutils/native/crates/cmd-base64/src/main.rs diff --git a/software/coreutils/native/crates/cmd-basename/Cargo.toml b/software/coreutils/native/crates/cmd-basename/Cargo.toml new file mode 100644 index 0000000000..aac009cc30 --- /dev/null +++ b/software/coreutils/native/crates/cmd-basename/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-basename" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "basename standalone binary for secure-exec VM" + +[[bin]] +name = "basename" +path = "src/main.rs" + +[dependencies] +uu_basename = "0.7.0" diff --git a/registry/native/crates/commands/basename/src/main.rs b/software/coreutils/native/crates/cmd-basename/src/main.rs similarity index 100% rename from registry/native/crates/commands/basename/src/main.rs rename to software/coreutils/native/crates/cmd-basename/src/main.rs diff --git a/software/coreutils/native/crates/cmd-basenc/Cargo.toml b/software/coreutils/native/crates/cmd-basenc/Cargo.toml new file mode 100644 index 0000000000..9a297f21c0 --- /dev/null +++ b/software/coreutils/native/crates/cmd-basenc/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-basenc" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "basenc standalone binary for secure-exec VM" + +[[bin]] +name = "basenc" +path = "src/main.rs" + +[dependencies] +uu_basenc = "0.7.0" diff --git a/registry/native/crates/commands/basenc/src/main.rs b/software/coreutils/native/crates/cmd-basenc/src/main.rs similarity index 100% rename from registry/native/crates/commands/basenc/src/main.rs rename to software/coreutils/native/crates/cmd-basenc/src/main.rs diff --git a/software/coreutils/native/crates/cmd-cat/Cargo.toml b/software/coreutils/native/crates/cmd-cat/Cargo.toml new file mode 100644 index 0000000000..747109c7cd --- /dev/null +++ b/software/coreutils/native/crates/cmd-cat/Cargo.toml @@ -0,0 +1,17 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-cat" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "cat standalone binary for secure-exec VM" + +[[bin]] +name = "cat" +path = "src/main.rs" + +[dependencies] +uu_cat = "0.7.0" + +[target.'cfg(any(target_os = "linux", target_os = "android"))'.dev-dependencies] +uucore = { version = "0.7.0", features = ["pipes"] } diff --git a/registry/native/crates/commands/cat/src/main.rs b/software/coreutils/native/crates/cmd-cat/src/main.rs similarity index 100% rename from registry/native/crates/commands/cat/src/main.rs rename to software/coreutils/native/crates/cmd-cat/src/main.rs diff --git a/registry/native/crates/commands/cat/tests/uucore_pipes.rs b/software/coreutils/native/crates/cmd-cat/tests/uucore_pipes.rs similarity index 100% rename from registry/native/crates/commands/cat/tests/uucore_pipes.rs rename to software/coreutils/native/crates/cmd-cat/tests/uucore_pipes.rs diff --git a/software/coreutils/native/crates/cmd-chmod/Cargo.toml b/software/coreutils/native/crates/cmd-chmod/Cargo.toml new file mode 100644 index 0000000000..9e56965b70 --- /dev/null +++ b/software/coreutils/native/crates/cmd-chmod/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-chmod" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "chmod standalone binary for secure-exec VM" + +[[bin]] +name = "chmod" +path = "src/main.rs" + +[dependencies] +uu_chmod = "0.7.0" diff --git a/registry/native/crates/commands/chmod/src/main.rs b/software/coreutils/native/crates/cmd-chmod/src/main.rs similarity index 100% rename from registry/native/crates/commands/chmod/src/main.rs rename to software/coreutils/native/crates/cmd-chmod/src/main.rs diff --git a/software/coreutils/native/crates/cmd-cksum/Cargo.toml b/software/coreutils/native/crates/cmd-cksum/Cargo.toml new file mode 100644 index 0000000000..25df33cb56 --- /dev/null +++ b/software/coreutils/native/crates/cmd-cksum/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-cksum" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "cksum standalone binary for secure-exec VM" + +[[bin]] +name = "cksum" +path = "src/main.rs" + +[dependencies] +uu_cksum = "0.7.0" diff --git a/registry/native/crates/commands/cksum/src/main.rs b/software/coreutils/native/crates/cmd-cksum/src/main.rs similarity index 100% rename from registry/native/crates/commands/cksum/src/main.rs rename to software/coreutils/native/crates/cmd-cksum/src/main.rs diff --git a/software/coreutils/native/crates/cmd-column/Cargo.toml b/software/coreutils/native/crates/cmd-column/Cargo.toml new file mode 100644 index 0000000000..8b16759a99 --- /dev/null +++ b/software/coreutils/native/crates/cmd-column/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-column" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "column standalone binary for secure-exec VM" + +[[bin]] +name = "column" +path = "src/main.rs" + +[dependencies] +secureexec-column = { path = "../column" } diff --git a/registry/native/crates/commands/column/src/main.rs b/software/coreutils/native/crates/cmd-column/src/main.rs similarity index 100% rename from registry/native/crates/commands/column/src/main.rs rename to software/coreutils/native/crates/cmd-column/src/main.rs diff --git a/software/coreutils/native/crates/cmd-comm/Cargo.toml b/software/coreutils/native/crates/cmd-comm/Cargo.toml new file mode 100644 index 0000000000..91e507f0f8 --- /dev/null +++ b/software/coreutils/native/crates/cmd-comm/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-comm" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "comm standalone binary for secure-exec VM" + +[[bin]] +name = "comm" +path = "src/main.rs" + +[dependencies] +uu_comm = "0.7.0" diff --git a/registry/native/crates/commands/comm/src/main.rs b/software/coreutils/native/crates/cmd-comm/src/main.rs similarity index 100% rename from registry/native/crates/commands/comm/src/main.rs rename to software/coreutils/native/crates/cmd-comm/src/main.rs diff --git a/software/coreutils/native/crates/cmd-cp/Cargo.toml b/software/coreutils/native/crates/cmd-cp/Cargo.toml new file mode 100644 index 0000000000..56a958c1fe --- /dev/null +++ b/software/coreutils/native/crates/cmd-cp/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-cp" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "cp standalone binary for secure-exec VM" + +[[bin]] +name = "cp" +path = "src/main.rs" + +[dependencies] +uu_cp = "0.7.0" diff --git a/registry/native/crates/commands/cp/src/main.rs b/software/coreutils/native/crates/cmd-cp/src/main.rs similarity index 100% rename from registry/native/crates/commands/cp/src/main.rs rename to software/coreutils/native/crates/cmd-cp/src/main.rs diff --git a/software/coreutils/native/crates/cmd-cut/Cargo.toml b/software/coreutils/native/crates/cmd-cut/Cargo.toml new file mode 100644 index 0000000000..72e5a5f814 --- /dev/null +++ b/software/coreutils/native/crates/cmd-cut/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-cut" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "cut standalone binary for secure-exec VM" + +[[bin]] +name = "cut" +path = "src/main.rs" + +[dependencies] +uu_cut = "0.7.0" diff --git a/registry/native/crates/commands/cut/src/main.rs b/software/coreutils/native/crates/cmd-cut/src/main.rs similarity index 100% rename from registry/native/crates/commands/cut/src/main.rs rename to software/coreutils/native/crates/cmd-cut/src/main.rs diff --git a/software/coreutils/native/crates/cmd-date/Cargo.toml b/software/coreutils/native/crates/cmd-date/Cargo.toml new file mode 100644 index 0000000000..7b0d440e2f --- /dev/null +++ b/software/coreutils/native/crates/cmd-date/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-date" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "date standalone binary for secure-exec VM" + +[[bin]] +name = "date" +path = "src/main.rs" + +[dependencies] +uu_date = "0.7.0" diff --git a/registry/native/crates/commands/date/src/main.rs b/software/coreutils/native/crates/cmd-date/src/main.rs similarity index 100% rename from registry/native/crates/commands/date/src/main.rs rename to software/coreutils/native/crates/cmd-date/src/main.rs diff --git a/software/coreutils/native/crates/cmd-dd/Cargo.toml b/software/coreutils/native/crates/cmd-dd/Cargo.toml new file mode 100644 index 0000000000..744ee8059f --- /dev/null +++ b/software/coreutils/native/crates/cmd-dd/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-dd" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "dd standalone binary for secure-exec VM" + +[[bin]] +name = "dd" +path = "src/main.rs" + +[dependencies] +uu_dd = "0.7.0" diff --git a/registry/native/crates/commands/dd/src/main.rs b/software/coreutils/native/crates/cmd-dd/src/main.rs similarity index 100% rename from registry/native/crates/commands/dd/src/main.rs rename to software/coreutils/native/crates/cmd-dd/src/main.rs diff --git a/software/coreutils/native/crates/cmd-dircolors/Cargo.toml b/software/coreutils/native/crates/cmd-dircolors/Cargo.toml new file mode 100644 index 0000000000..86c849c392 --- /dev/null +++ b/software/coreutils/native/crates/cmd-dircolors/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-dircolors" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "dircolors standalone binary for secure-exec VM" + +[[bin]] +name = "dircolors" +path = "src/main.rs" + +[dependencies] +uu_dircolors = "0.7.0" diff --git a/registry/native/crates/commands/dircolors/src/main.rs b/software/coreutils/native/crates/cmd-dircolors/src/main.rs similarity index 100% rename from registry/native/crates/commands/dircolors/src/main.rs rename to software/coreutils/native/crates/cmd-dircolors/src/main.rs diff --git a/software/coreutils/native/crates/cmd-dirname/Cargo.toml b/software/coreutils/native/crates/cmd-dirname/Cargo.toml new file mode 100644 index 0000000000..bd1615de6b --- /dev/null +++ b/software/coreutils/native/crates/cmd-dirname/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-dirname" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "dirname standalone binary for secure-exec VM" + +[[bin]] +name = "dirname" +path = "src/main.rs" + +[dependencies] +uu_dirname = "0.7.0" diff --git a/registry/native/crates/commands/dirname/src/main.rs b/software/coreutils/native/crates/cmd-dirname/src/main.rs similarity index 100% rename from registry/native/crates/commands/dirname/src/main.rs rename to software/coreutils/native/crates/cmd-dirname/src/main.rs diff --git a/software/coreutils/native/crates/cmd-du/Cargo.toml b/software/coreutils/native/crates/cmd-du/Cargo.toml new file mode 100644 index 0000000000..8a2cc1e9d3 --- /dev/null +++ b/software/coreutils/native/crates/cmd-du/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-du" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "du standalone binary for secure-exec VM" + +[[bin]] +name = "du" +path = "src/main.rs" + +[dependencies] +secureexec-du = { path = "../du" } diff --git a/registry/native/crates/commands/du/src/main.rs b/software/coreutils/native/crates/cmd-du/src/main.rs similarity index 100% rename from registry/native/crates/commands/du/src/main.rs rename to software/coreutils/native/crates/cmd-du/src/main.rs diff --git a/software/coreutils/native/crates/cmd-echo/Cargo.toml b/software/coreutils/native/crates/cmd-echo/Cargo.toml new file mode 100644 index 0000000000..2f882f37be --- /dev/null +++ b/software/coreutils/native/crates/cmd-echo/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-echo" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "echo standalone binary for secure-exec VM" + +[[bin]] +name = "echo" +path = "src/main.rs" + +[dependencies] +uu_echo = "0.7.0" diff --git a/registry/native/crates/commands/echo/src/main.rs b/software/coreutils/native/crates/cmd-echo/src/main.rs similarity index 100% rename from registry/native/crates/commands/echo/src/main.rs rename to software/coreutils/native/crates/cmd-echo/src/main.rs diff --git a/software/coreutils/native/crates/cmd-env/Cargo.toml b/software/coreutils/native/crates/cmd-env/Cargo.toml new file mode 100644 index 0000000000..6fe19a8d7d --- /dev/null +++ b/software/coreutils/native/crates/cmd-env/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-env" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "env standalone binary for secure-exec VM" + +[[bin]] +name = "env" +path = "src/main.rs" + +[dependencies] +shims = { package = "secureexec-shims", path = "../../../../../toolchain/crates/libs/shims" } diff --git a/registry/native/crates/commands/env/src/main.rs b/software/coreutils/native/crates/cmd-env/src/main.rs similarity index 100% rename from registry/native/crates/commands/env/src/main.rs rename to software/coreutils/native/crates/cmd-env/src/main.rs diff --git a/software/coreutils/native/crates/cmd-expand/Cargo.toml b/software/coreutils/native/crates/cmd-expand/Cargo.toml new file mode 100644 index 0000000000..08807dc938 --- /dev/null +++ b/software/coreutils/native/crates/cmd-expand/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-expand" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "expand standalone binary for secure-exec VM" + +[[bin]] +name = "expand" +path = "src/main.rs" + +[dependencies] +uu_expand = "0.7.0" diff --git a/registry/native/crates/commands/expand/src/main.rs b/software/coreutils/native/crates/cmd-expand/src/main.rs similarity index 100% rename from registry/native/crates/commands/expand/src/main.rs rename to software/coreutils/native/crates/cmd-expand/src/main.rs diff --git a/software/coreutils/native/crates/cmd-expr/Cargo.toml b/software/coreutils/native/crates/cmd-expr/Cargo.toml new file mode 100644 index 0000000000..10bd466ba0 --- /dev/null +++ b/software/coreutils/native/crates/cmd-expr/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-expr" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "expr standalone binary for secure-exec VM" + +[[bin]] +name = "expr" +path = "src/main.rs" + +[dependencies] +secureexec-expr = { path = "../expr" } diff --git a/registry/native/crates/commands/expr/src/main.rs b/software/coreutils/native/crates/cmd-expr/src/main.rs similarity index 100% rename from registry/native/crates/commands/expr/src/main.rs rename to software/coreutils/native/crates/cmd-expr/src/main.rs diff --git a/software/coreutils/native/crates/cmd-factor/Cargo.toml b/software/coreutils/native/crates/cmd-factor/Cargo.toml new file mode 100644 index 0000000000..cd4f1f233b --- /dev/null +++ b/software/coreutils/native/crates/cmd-factor/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-factor" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "factor standalone binary for secure-exec VM" + +[[bin]] +name = "factor" +path = "src/main.rs" + +[dependencies] +uu_factor = "0.7.0" diff --git a/registry/native/crates/commands/factor/src/main.rs b/software/coreutils/native/crates/cmd-factor/src/main.rs similarity index 100% rename from registry/native/crates/commands/factor/src/main.rs rename to software/coreutils/native/crates/cmd-factor/src/main.rs diff --git a/software/coreutils/native/crates/cmd-false/Cargo.toml b/software/coreutils/native/crates/cmd-false/Cargo.toml new file mode 100644 index 0000000000..a3660c6acf --- /dev/null +++ b/software/coreutils/native/crates/cmd-false/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-false" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "false standalone binary for secure-exec VM" + +[[bin]] +name = "false" +path = "src/main.rs" + +[dependencies] +uu_false = "0.7.0" diff --git a/registry/native/crates/commands/false/src/main.rs b/software/coreutils/native/crates/cmd-false/src/main.rs similarity index 100% rename from registry/native/crates/commands/false/src/main.rs rename to software/coreutils/native/crates/cmd-false/src/main.rs diff --git a/software/coreutils/native/crates/cmd-fmt/Cargo.toml b/software/coreutils/native/crates/cmd-fmt/Cargo.toml new file mode 100644 index 0000000000..2ebb0f6f6a --- /dev/null +++ b/software/coreutils/native/crates/cmd-fmt/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-fmt" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "fmt standalone binary for secure-exec VM" + +[[bin]] +name = "fmt" +path = "src/main.rs" + +[dependencies] +uu_fmt = "0.7.0" diff --git a/registry/native/crates/commands/fmt/src/main.rs b/software/coreutils/native/crates/cmd-fmt/src/main.rs similarity index 100% rename from registry/native/crates/commands/fmt/src/main.rs rename to software/coreutils/native/crates/cmd-fmt/src/main.rs diff --git a/software/coreutils/native/crates/cmd-fold/Cargo.toml b/software/coreutils/native/crates/cmd-fold/Cargo.toml new file mode 100644 index 0000000000..7d79e95796 --- /dev/null +++ b/software/coreutils/native/crates/cmd-fold/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-fold" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "fold standalone binary for secure-exec VM" + +[[bin]] +name = "fold" +path = "src/main.rs" + +[dependencies] +uu_fold = "0.7.0" diff --git a/registry/native/crates/commands/fold/src/main.rs b/software/coreutils/native/crates/cmd-fold/src/main.rs similarity index 100% rename from registry/native/crates/commands/fold/src/main.rs rename to software/coreutils/native/crates/cmd-fold/src/main.rs diff --git a/software/coreutils/native/crates/cmd-head/Cargo.toml b/software/coreutils/native/crates/cmd-head/Cargo.toml new file mode 100644 index 0000000000..287d999055 --- /dev/null +++ b/software/coreutils/native/crates/cmd-head/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-head" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "head standalone binary for secure-exec VM" + +[[bin]] +name = "head" +path = "src/main.rs" + +[dependencies] +uu_head = "0.7.0" diff --git a/registry/native/crates/commands/head/src/main.rs b/software/coreutils/native/crates/cmd-head/src/main.rs similarity index 100% rename from registry/native/crates/commands/head/src/main.rs rename to software/coreutils/native/crates/cmd-head/src/main.rs diff --git a/software/coreutils/native/crates/cmd-join/Cargo.toml b/software/coreutils/native/crates/cmd-join/Cargo.toml new file mode 100644 index 0000000000..c81c31cb5f --- /dev/null +++ b/software/coreutils/native/crates/cmd-join/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-join" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "join standalone binary for secure-exec VM" + +[[bin]] +name = "join" +path = "src/main.rs" + +[dependencies] +uu_join = "0.7.0" diff --git a/registry/native/crates/commands/join/src/main.rs b/software/coreutils/native/crates/cmd-join/src/main.rs similarity index 100% rename from registry/native/crates/commands/join/src/main.rs rename to software/coreutils/native/crates/cmd-join/src/main.rs diff --git a/software/coreutils/native/crates/cmd-link/Cargo.toml b/software/coreutils/native/crates/cmd-link/Cargo.toml new file mode 100644 index 0000000000..043488ee6c --- /dev/null +++ b/software/coreutils/native/crates/cmd-link/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-link" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "link standalone binary for secure-exec VM" + +[[bin]] +name = "link" +path = "src/main.rs" + +[dependencies] +uu_link = "0.7.0" diff --git a/registry/native/crates/commands/link/src/main.rs b/software/coreutils/native/crates/cmd-link/src/main.rs similarity index 100% rename from registry/native/crates/commands/link/src/main.rs rename to software/coreutils/native/crates/cmd-link/src/main.rs diff --git a/software/coreutils/native/crates/cmd-ln/Cargo.toml b/software/coreutils/native/crates/cmd-ln/Cargo.toml new file mode 100644 index 0000000000..92950c1f35 --- /dev/null +++ b/software/coreutils/native/crates/cmd-ln/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-ln" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "ln standalone binary for secure-exec VM" + +[[bin]] +name = "ln" +path = "src/main.rs" + +[dependencies] +uu_ln = "0.7.0" diff --git a/registry/native/crates/commands/ln/src/main.rs b/software/coreutils/native/crates/cmd-ln/src/main.rs similarity index 100% rename from registry/native/crates/commands/ln/src/main.rs rename to software/coreutils/native/crates/cmd-ln/src/main.rs diff --git a/software/coreutils/native/crates/cmd-logname/Cargo.toml b/software/coreutils/native/crates/cmd-logname/Cargo.toml new file mode 100644 index 0000000000..33b6d0d678 --- /dev/null +++ b/software/coreutils/native/crates/cmd-logname/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-logname" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "logname standalone binary for secure-exec VM" + +[[bin]] +name = "logname" +path = "src/main.rs" + +[dependencies] +uu_logname = "0.7.0" diff --git a/registry/native/crates/commands/logname/src/main.rs b/software/coreutils/native/crates/cmd-logname/src/main.rs similarity index 100% rename from registry/native/crates/commands/logname/src/main.rs rename to software/coreutils/native/crates/cmd-logname/src/main.rs diff --git a/software/coreutils/native/crates/cmd-ls/Cargo.toml b/software/coreutils/native/crates/cmd-ls/Cargo.toml new file mode 100644 index 0000000000..71ba7a86e8 --- /dev/null +++ b/software/coreutils/native/crates/cmd-ls/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-ls" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "ls standalone binary for secure-exec VM" + +[[bin]] +name = "ls" +path = "src/main.rs" + +[dependencies] +uu_ls = "0.7.0" diff --git a/registry/native/crates/commands/ls/src/main.rs b/software/coreutils/native/crates/cmd-ls/src/main.rs similarity index 100% rename from registry/native/crates/commands/ls/src/main.rs rename to software/coreutils/native/crates/cmd-ls/src/main.rs diff --git a/software/coreutils/native/crates/cmd-md5sum/Cargo.toml b/software/coreutils/native/crates/cmd-md5sum/Cargo.toml new file mode 100644 index 0000000000..ea660b5d0f --- /dev/null +++ b/software/coreutils/native/crates/cmd-md5sum/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-md5sum" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "md5sum standalone binary for secure-exec VM" + +[[bin]] +name = "md5sum" +path = "src/main.rs" + +[dependencies] +uu_md5sum = "0.7.0" diff --git a/registry/native/crates/commands/md5sum/src/main.rs b/software/coreutils/native/crates/cmd-md5sum/src/main.rs similarity index 100% rename from registry/native/crates/commands/md5sum/src/main.rs rename to software/coreutils/native/crates/cmd-md5sum/src/main.rs diff --git a/software/coreutils/native/crates/cmd-mkdir/Cargo.toml b/software/coreutils/native/crates/cmd-mkdir/Cargo.toml new file mode 100644 index 0000000000..7a3e9f4ccf --- /dev/null +++ b/software/coreutils/native/crates/cmd-mkdir/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-mkdir" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "mkdir standalone binary for secure-exec VM" + +[[bin]] +name = "mkdir" +path = "src/main.rs" + +[dependencies] +uu_mkdir = "0.7.0" diff --git a/registry/native/crates/commands/mkdir/src/main.rs b/software/coreutils/native/crates/cmd-mkdir/src/main.rs similarity index 100% rename from registry/native/crates/commands/mkdir/src/main.rs rename to software/coreutils/native/crates/cmd-mkdir/src/main.rs diff --git a/software/coreutils/native/crates/cmd-mktemp/Cargo.toml b/software/coreutils/native/crates/cmd-mktemp/Cargo.toml new file mode 100644 index 0000000000..57935df632 --- /dev/null +++ b/software/coreutils/native/crates/cmd-mktemp/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-mktemp" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "mktemp standalone binary for secure-exec VM" + +[[bin]] +name = "mktemp" +path = "src/main.rs" + +[dependencies] +uu_mktemp = "0.7.0" diff --git a/registry/native/crates/commands/mktemp/src/main.rs b/software/coreutils/native/crates/cmd-mktemp/src/main.rs similarity index 100% rename from registry/native/crates/commands/mktemp/src/main.rs rename to software/coreutils/native/crates/cmd-mktemp/src/main.rs diff --git a/software/coreutils/native/crates/cmd-mv/Cargo.toml b/software/coreutils/native/crates/cmd-mv/Cargo.toml new file mode 100644 index 0000000000..c4999d907c --- /dev/null +++ b/software/coreutils/native/crates/cmd-mv/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-mv" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "mv standalone binary for secure-exec VM" + +[[bin]] +name = "mv" +path = "src/main.rs" + +[dependencies] +uu_mv = "0.7.0" diff --git a/registry/native/crates/commands/mv/src/main.rs b/software/coreutils/native/crates/cmd-mv/src/main.rs similarity index 100% rename from registry/native/crates/commands/mv/src/main.rs rename to software/coreutils/native/crates/cmd-mv/src/main.rs diff --git a/registry/native/crates/commands/mv/tests/simple_mv.rs b/software/coreutils/native/crates/cmd-mv/tests/simple_mv.rs similarity index 100% rename from registry/native/crates/commands/mv/tests/simple_mv.rs rename to software/coreutils/native/crates/cmd-mv/tests/simple_mv.rs diff --git a/software/coreutils/native/crates/cmd-nice/Cargo.toml b/software/coreutils/native/crates/cmd-nice/Cargo.toml new file mode 100644 index 0000000000..2f77be052e --- /dev/null +++ b/software/coreutils/native/crates/cmd-nice/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-nice" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "nice standalone binary for secure-exec VM" + +[[bin]] +name = "nice" +path = "src/main.rs" + +[dependencies] +shims = { package = "secureexec-shims", path = "../../../../../toolchain/crates/libs/shims" } diff --git a/registry/native/crates/commands/nice/src/main.rs b/software/coreutils/native/crates/cmd-nice/src/main.rs similarity index 100% rename from registry/native/crates/commands/nice/src/main.rs rename to software/coreutils/native/crates/cmd-nice/src/main.rs diff --git a/software/coreutils/native/crates/cmd-nl/Cargo.toml b/software/coreutils/native/crates/cmd-nl/Cargo.toml new file mode 100644 index 0000000000..067baf0805 --- /dev/null +++ b/software/coreutils/native/crates/cmd-nl/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-nl" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "nl standalone binary for secure-exec VM" + +[[bin]] +name = "nl" +path = "src/main.rs" + +[dependencies] +uu_nl = "0.7.0" diff --git a/registry/native/crates/commands/nl/src/main.rs b/software/coreutils/native/crates/cmd-nl/src/main.rs similarity index 100% rename from registry/native/crates/commands/nl/src/main.rs rename to software/coreutils/native/crates/cmd-nl/src/main.rs diff --git a/software/coreutils/native/crates/cmd-nohup/Cargo.toml b/software/coreutils/native/crates/cmd-nohup/Cargo.toml new file mode 100644 index 0000000000..94edf64e00 --- /dev/null +++ b/software/coreutils/native/crates/cmd-nohup/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-nohup" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "nohup standalone binary for secure-exec VM" + +[[bin]] +name = "nohup" +path = "src/main.rs" + +[dependencies] +shims = { package = "secureexec-shims", path = "../../../../../toolchain/crates/libs/shims" } diff --git a/registry/native/crates/commands/nohup/src/main.rs b/software/coreutils/native/crates/cmd-nohup/src/main.rs similarity index 100% rename from registry/native/crates/commands/nohup/src/main.rs rename to software/coreutils/native/crates/cmd-nohup/src/main.rs diff --git a/registry/native/crates/commands/nohup/tests/streaming.rs b/software/coreutils/native/crates/cmd-nohup/tests/streaming.rs similarity index 100% rename from registry/native/crates/commands/nohup/tests/streaming.rs rename to software/coreutils/native/crates/cmd-nohup/tests/streaming.rs diff --git a/software/coreutils/native/crates/cmd-nproc/Cargo.toml b/software/coreutils/native/crates/cmd-nproc/Cargo.toml new file mode 100644 index 0000000000..9c6fc0fbbb --- /dev/null +++ b/software/coreutils/native/crates/cmd-nproc/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-nproc" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "nproc standalone binary for secure-exec VM" + +[[bin]] +name = "nproc" +path = "src/main.rs" + +[dependencies] +uu_nproc = "0.7.0" diff --git a/registry/native/crates/commands/nproc/src/main.rs b/software/coreutils/native/crates/cmd-nproc/src/main.rs similarity index 100% rename from registry/native/crates/commands/nproc/src/main.rs rename to software/coreutils/native/crates/cmd-nproc/src/main.rs diff --git a/software/coreutils/native/crates/cmd-numfmt/Cargo.toml b/software/coreutils/native/crates/cmd-numfmt/Cargo.toml new file mode 100644 index 0000000000..6ae2fedb83 --- /dev/null +++ b/software/coreutils/native/crates/cmd-numfmt/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-numfmt" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "numfmt standalone binary for secure-exec VM" + +[[bin]] +name = "numfmt" +path = "src/main.rs" + +[dependencies] +uu_numfmt = "0.7.0" diff --git a/registry/native/crates/commands/numfmt/src/main.rs b/software/coreutils/native/crates/cmd-numfmt/src/main.rs similarity index 100% rename from registry/native/crates/commands/numfmt/src/main.rs rename to software/coreutils/native/crates/cmd-numfmt/src/main.rs diff --git a/software/coreutils/native/crates/cmd-od/Cargo.toml b/software/coreutils/native/crates/cmd-od/Cargo.toml new file mode 100644 index 0000000000..102a12cdab --- /dev/null +++ b/software/coreutils/native/crates/cmd-od/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-od" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "od standalone binary for secure-exec VM" + +[[bin]] +name = "od" +path = "src/main.rs" + +[dependencies] +uu_od = "0.7.0" diff --git a/registry/native/crates/commands/od/src/main.rs b/software/coreutils/native/crates/cmd-od/src/main.rs similarity index 100% rename from registry/native/crates/commands/od/src/main.rs rename to software/coreutils/native/crates/cmd-od/src/main.rs diff --git a/software/coreutils/native/crates/cmd-paste/Cargo.toml b/software/coreutils/native/crates/cmd-paste/Cargo.toml new file mode 100644 index 0000000000..8e45b33c09 --- /dev/null +++ b/software/coreutils/native/crates/cmd-paste/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-paste" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "paste standalone binary for secure-exec VM" + +[[bin]] +name = "paste" +path = "src/main.rs" + +[dependencies] +uu_paste = "0.7.0" diff --git a/registry/native/crates/commands/paste/src/main.rs b/software/coreutils/native/crates/cmd-paste/src/main.rs similarity index 100% rename from registry/native/crates/commands/paste/src/main.rs rename to software/coreutils/native/crates/cmd-paste/src/main.rs diff --git a/software/coreutils/native/crates/cmd-pathchk/Cargo.toml b/software/coreutils/native/crates/cmd-pathchk/Cargo.toml new file mode 100644 index 0000000000..b675dc27f1 --- /dev/null +++ b/software/coreutils/native/crates/cmd-pathchk/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-pathchk" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "pathchk standalone binary for secure-exec VM" + +[[bin]] +name = "pathchk" +path = "src/main.rs" + +[dependencies] +uu_pathchk = "0.7.0" diff --git a/registry/native/crates/commands/pathchk/src/main.rs b/software/coreutils/native/crates/cmd-pathchk/src/main.rs similarity index 100% rename from registry/native/crates/commands/pathchk/src/main.rs rename to software/coreutils/native/crates/cmd-pathchk/src/main.rs diff --git a/software/coreutils/native/crates/cmd-printenv/Cargo.toml b/software/coreutils/native/crates/cmd-printenv/Cargo.toml new file mode 100644 index 0000000000..918b962ea6 --- /dev/null +++ b/software/coreutils/native/crates/cmd-printenv/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-printenv" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "printenv standalone binary for secure-exec VM" + +[[bin]] +name = "printenv" +path = "src/main.rs" + +[dependencies] +uu_printenv = "0.7.0" diff --git a/registry/native/crates/commands/printenv/src/main.rs b/software/coreutils/native/crates/cmd-printenv/src/main.rs similarity index 100% rename from registry/native/crates/commands/printenv/src/main.rs rename to software/coreutils/native/crates/cmd-printenv/src/main.rs diff --git a/software/coreutils/native/crates/cmd-printf/Cargo.toml b/software/coreutils/native/crates/cmd-printf/Cargo.toml new file mode 100644 index 0000000000..c0512cfd17 --- /dev/null +++ b/software/coreutils/native/crates/cmd-printf/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-printf" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "printf standalone binary for secure-exec VM" + +[[bin]] +name = "printf" +path = "src/main.rs" + +[dependencies] +uu_printf = "0.7.0" diff --git a/registry/native/crates/commands/printf/src/main.rs b/software/coreutils/native/crates/cmd-printf/src/main.rs similarity index 100% rename from registry/native/crates/commands/printf/src/main.rs rename to software/coreutils/native/crates/cmd-printf/src/main.rs diff --git a/software/coreutils/native/crates/cmd-ptx/Cargo.toml b/software/coreutils/native/crates/cmd-ptx/Cargo.toml new file mode 100644 index 0000000000..9eefecd190 --- /dev/null +++ b/software/coreutils/native/crates/cmd-ptx/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-ptx" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "ptx standalone binary for secure-exec VM" + +[[bin]] +name = "ptx" +path = "src/main.rs" + +[dependencies] +uu_ptx = "0.7.0" diff --git a/registry/native/crates/commands/ptx/src/main.rs b/software/coreutils/native/crates/cmd-ptx/src/main.rs similarity index 100% rename from registry/native/crates/commands/ptx/src/main.rs rename to software/coreutils/native/crates/cmd-ptx/src/main.rs diff --git a/software/coreutils/native/crates/cmd-pwd/Cargo.toml b/software/coreutils/native/crates/cmd-pwd/Cargo.toml new file mode 100644 index 0000000000..b3e7ff6a74 --- /dev/null +++ b/software/coreutils/native/crates/cmd-pwd/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-pwd" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "pwd standalone binary for secure-exec VM" + +[[bin]] +name = "pwd" +path = "src/main.rs" + +[dependencies] +uu_pwd = "0.7.0" diff --git a/registry/native/crates/commands/pwd/src/main.rs b/software/coreutils/native/crates/cmd-pwd/src/main.rs similarity index 100% rename from registry/native/crates/commands/pwd/src/main.rs rename to software/coreutils/native/crates/cmd-pwd/src/main.rs diff --git a/software/coreutils/native/crates/cmd-readlink/Cargo.toml b/software/coreutils/native/crates/cmd-readlink/Cargo.toml new file mode 100644 index 0000000000..7fa48bbe6f --- /dev/null +++ b/software/coreutils/native/crates/cmd-readlink/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-readlink" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "readlink standalone binary for secure-exec VM" + +[[bin]] +name = "readlink" +path = "src/main.rs" + +[dependencies] +uu_readlink = "0.7.0" diff --git a/registry/native/crates/commands/readlink/src/main.rs b/software/coreutils/native/crates/cmd-readlink/src/main.rs similarity index 100% rename from registry/native/crates/commands/readlink/src/main.rs rename to software/coreutils/native/crates/cmd-readlink/src/main.rs diff --git a/software/coreutils/native/crates/cmd-realpath/Cargo.toml b/software/coreutils/native/crates/cmd-realpath/Cargo.toml new file mode 100644 index 0000000000..dbbbc9da3a --- /dev/null +++ b/software/coreutils/native/crates/cmd-realpath/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-realpath" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "realpath standalone binary for secure-exec VM" + +[[bin]] +name = "realpath" +path = "src/main.rs" + +[dependencies] +uu_realpath = "0.7.0" diff --git a/registry/native/crates/commands/realpath/src/main.rs b/software/coreutils/native/crates/cmd-realpath/src/main.rs similarity index 100% rename from registry/native/crates/commands/realpath/src/main.rs rename to software/coreutils/native/crates/cmd-realpath/src/main.rs diff --git a/software/coreutils/native/crates/cmd-rev/Cargo.toml b/software/coreutils/native/crates/cmd-rev/Cargo.toml new file mode 100644 index 0000000000..66a1ffd19d --- /dev/null +++ b/software/coreutils/native/crates/cmd-rev/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-rev" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "rev standalone binary for secure-exec VM" + +[[bin]] +name = "rev" +path = "src/main.rs" + +[dependencies] +secureexec-rev = { path = "../rev" } diff --git a/registry/native/crates/commands/rev/src/main.rs b/software/coreutils/native/crates/cmd-rev/src/main.rs similarity index 100% rename from registry/native/crates/commands/rev/src/main.rs rename to software/coreutils/native/crates/cmd-rev/src/main.rs diff --git a/software/coreutils/native/crates/cmd-rm/Cargo.toml b/software/coreutils/native/crates/cmd-rm/Cargo.toml new file mode 100644 index 0000000000..4a9dc744e7 --- /dev/null +++ b/software/coreutils/native/crates/cmd-rm/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-rm" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "rm standalone binary for secure-exec VM" + +[[bin]] +name = "rm" +path = "src/main.rs" + +[dependencies] +uu_rm = "0.7.0" diff --git a/registry/native/crates/commands/rm/src/main.rs b/software/coreutils/native/crates/cmd-rm/src/main.rs similarity index 100% rename from registry/native/crates/commands/rm/src/main.rs rename to software/coreutils/native/crates/cmd-rm/src/main.rs diff --git a/software/coreutils/native/crates/cmd-rmdir/Cargo.toml b/software/coreutils/native/crates/cmd-rmdir/Cargo.toml new file mode 100644 index 0000000000..f9729df6f7 --- /dev/null +++ b/software/coreutils/native/crates/cmd-rmdir/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-rmdir" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "rmdir standalone binary for secure-exec VM" + +[[bin]] +name = "rmdir" +path = "src/main.rs" + +[dependencies] +uu_rmdir = "0.7.0" diff --git a/registry/native/crates/commands/rmdir/src/main.rs b/software/coreutils/native/crates/cmd-rmdir/src/main.rs similarity index 100% rename from registry/native/crates/commands/rmdir/src/main.rs rename to software/coreutils/native/crates/cmd-rmdir/src/main.rs diff --git a/software/coreutils/native/crates/cmd-seq/Cargo.toml b/software/coreutils/native/crates/cmd-seq/Cargo.toml new file mode 100644 index 0000000000..bb035b54fc --- /dev/null +++ b/software/coreutils/native/crates/cmd-seq/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-seq" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "seq standalone binary for secure-exec VM" + +[[bin]] +name = "seq" +path = "src/main.rs" + +[dependencies] +uu_seq = "0.7.0" diff --git a/registry/native/crates/commands/seq/src/main.rs b/software/coreutils/native/crates/cmd-seq/src/main.rs similarity index 100% rename from registry/native/crates/commands/seq/src/main.rs rename to software/coreutils/native/crates/cmd-seq/src/main.rs diff --git a/software/coreutils/native/crates/cmd-sh/Cargo.toml b/software/coreutils/native/crates/cmd-sh/Cargo.toml new file mode 100644 index 0000000000..51976f0cc4 --- /dev/null +++ b/software/coreutils/native/crates/cmd-sh/Cargo.toml @@ -0,0 +1,19 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-sh" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "sh standalone binary for secure-exec VM" + +[[bin]] +name = "sh" +path = "src/main.rs" + +# `reedline` is enabled on ALL targets, including wasm: the vendored wasi +# patches supply what it needs (crossterm wasi backend, fd-lock, and +# brush-interactive's cfg(target_os = "wasi") alternatives to the +# tokio block_in_place paths), per the "behave like native Linux" rule. +# `minimal` stays available as an explicit opt-out backend. +[dependencies] +brush-shell = { version = "0.3.0", default-features = false, features = ["reedline", "minimal"] } diff --git a/registry/native/crates/commands/sh/src/main.rs b/software/coreutils/native/crates/cmd-sh/src/main.rs similarity index 100% rename from registry/native/crates/commands/sh/src/main.rs rename to software/coreutils/native/crates/cmd-sh/src/main.rs diff --git a/software/coreutils/native/crates/cmd-sha1sum/Cargo.toml b/software/coreutils/native/crates/cmd-sha1sum/Cargo.toml new file mode 100644 index 0000000000..1e4b91ec01 --- /dev/null +++ b/software/coreutils/native/crates/cmd-sha1sum/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-sha1sum" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "sha1sum standalone binary for secure-exec VM" + +[[bin]] +name = "sha1sum" +path = "src/main.rs" + +[dependencies] +uu_sha1sum = "0.7.0" diff --git a/registry/native/crates/commands/sha1sum/src/main.rs b/software/coreutils/native/crates/cmd-sha1sum/src/main.rs similarity index 100% rename from registry/native/crates/commands/sha1sum/src/main.rs rename to software/coreutils/native/crates/cmd-sha1sum/src/main.rs diff --git a/software/coreutils/native/crates/cmd-sha224sum/Cargo.toml b/software/coreutils/native/crates/cmd-sha224sum/Cargo.toml new file mode 100644 index 0000000000..bdd6f4cb70 --- /dev/null +++ b/software/coreutils/native/crates/cmd-sha224sum/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-sha224sum" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "sha224sum standalone binary for secure-exec VM" + +[[bin]] +name = "sha224sum" +path = "src/main.rs" + +[dependencies] +uu_sha224sum = "0.7.0" diff --git a/registry/native/crates/commands/sha224sum/src/main.rs b/software/coreutils/native/crates/cmd-sha224sum/src/main.rs similarity index 100% rename from registry/native/crates/commands/sha224sum/src/main.rs rename to software/coreutils/native/crates/cmd-sha224sum/src/main.rs diff --git a/software/coreutils/native/crates/cmd-sha256sum/Cargo.toml b/software/coreutils/native/crates/cmd-sha256sum/Cargo.toml new file mode 100644 index 0000000000..a43ebd9ff9 --- /dev/null +++ b/software/coreutils/native/crates/cmd-sha256sum/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-sha256sum" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "sha256sum standalone binary for secure-exec VM" + +[[bin]] +name = "sha256sum" +path = "src/main.rs" + +[dependencies] +uu_sha256sum = "0.7.0" diff --git a/registry/native/crates/commands/sha256sum/src/main.rs b/software/coreutils/native/crates/cmd-sha256sum/src/main.rs similarity index 100% rename from registry/native/crates/commands/sha256sum/src/main.rs rename to software/coreutils/native/crates/cmd-sha256sum/src/main.rs diff --git a/software/coreutils/native/crates/cmd-sha384sum/Cargo.toml b/software/coreutils/native/crates/cmd-sha384sum/Cargo.toml new file mode 100644 index 0000000000..3cec44da9d --- /dev/null +++ b/software/coreutils/native/crates/cmd-sha384sum/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-sha384sum" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "sha384sum standalone binary for secure-exec VM" + +[[bin]] +name = "sha384sum" +path = "src/main.rs" + +[dependencies] +uu_sha384sum = "0.7.0" diff --git a/registry/native/crates/commands/sha384sum/src/main.rs b/software/coreutils/native/crates/cmd-sha384sum/src/main.rs similarity index 100% rename from registry/native/crates/commands/sha384sum/src/main.rs rename to software/coreutils/native/crates/cmd-sha384sum/src/main.rs diff --git a/software/coreutils/native/crates/cmd-sha512sum/Cargo.toml b/software/coreutils/native/crates/cmd-sha512sum/Cargo.toml new file mode 100644 index 0000000000..0d15e9ac32 --- /dev/null +++ b/software/coreutils/native/crates/cmd-sha512sum/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-sha512sum" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "sha512sum standalone binary for secure-exec VM" + +[[bin]] +name = "sha512sum" +path = "src/main.rs" + +[dependencies] +uu_sha512sum = "0.7.0" diff --git a/registry/native/crates/commands/sha512sum/src/main.rs b/software/coreutils/native/crates/cmd-sha512sum/src/main.rs similarity index 100% rename from registry/native/crates/commands/sha512sum/src/main.rs rename to software/coreutils/native/crates/cmd-sha512sum/src/main.rs diff --git a/software/coreutils/native/crates/cmd-shred/Cargo.toml b/software/coreutils/native/crates/cmd-shred/Cargo.toml new file mode 100644 index 0000000000..15cb98f87c --- /dev/null +++ b/software/coreutils/native/crates/cmd-shred/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-shred" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "shred standalone binary for secure-exec VM" + +[[bin]] +name = "shred" +path = "src/main.rs" + +[dependencies] +uu_shred = "0.7.0" diff --git a/registry/native/crates/commands/shred/src/main.rs b/software/coreutils/native/crates/cmd-shred/src/main.rs similarity index 100% rename from registry/native/crates/commands/shred/src/main.rs rename to software/coreutils/native/crates/cmd-shred/src/main.rs diff --git a/software/coreutils/native/crates/cmd-shuf/Cargo.toml b/software/coreutils/native/crates/cmd-shuf/Cargo.toml new file mode 100644 index 0000000000..06dc44cf52 --- /dev/null +++ b/software/coreutils/native/crates/cmd-shuf/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-shuf" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "shuf standalone binary for secure-exec VM" + +[[bin]] +name = "shuf" +path = "src/main.rs" + +[dependencies] +uu_shuf = "0.7.0" diff --git a/registry/native/crates/commands/shuf/src/main.rs b/software/coreutils/native/crates/cmd-shuf/src/main.rs similarity index 100% rename from registry/native/crates/commands/shuf/src/main.rs rename to software/coreutils/native/crates/cmd-shuf/src/main.rs diff --git a/software/coreutils/native/crates/cmd-sleep/Cargo.toml b/software/coreutils/native/crates/cmd-sleep/Cargo.toml new file mode 100644 index 0000000000..816b051b22 --- /dev/null +++ b/software/coreutils/native/crates/cmd-sleep/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-sleep" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "sleep standalone binary for secure-exec VM" + +[[bin]] +name = "sleep" +path = "src/main.rs" + +[dependencies] +secureexec-builtins = { path = "../../../../../toolchain/crates/libs/builtins" } diff --git a/registry/native/crates/commands/sleep/src/main.rs b/software/coreutils/native/crates/cmd-sleep/src/main.rs similarity index 100% rename from registry/native/crates/commands/sleep/src/main.rs rename to software/coreutils/native/crates/cmd-sleep/src/main.rs diff --git a/software/coreutils/native/crates/cmd-sort/Cargo.toml b/software/coreutils/native/crates/cmd-sort/Cargo.toml new file mode 100644 index 0000000000..0507575a3b --- /dev/null +++ b/software/coreutils/native/crates/cmd-sort/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-sort" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "sort standalone binary for secure-exec VM" + +[[bin]] +name = "sort" +path = "src/main.rs" + +[dependencies] +uu_sort = "0.7.0" diff --git a/registry/native/crates/commands/sort/src/main.rs b/software/coreutils/native/crates/cmd-sort/src/main.rs similarity index 100% rename from registry/native/crates/commands/sort/src/main.rs rename to software/coreutils/native/crates/cmd-sort/src/main.rs diff --git a/registry/native/crates/commands/sort/tests/external_sort.rs b/software/coreutils/native/crates/cmd-sort/tests/external_sort.rs similarity index 100% rename from registry/native/crates/commands/sort/tests/external_sort.rs rename to software/coreutils/native/crates/cmd-sort/tests/external_sort.rs diff --git a/software/coreutils/native/crates/cmd-split/Cargo.toml b/software/coreutils/native/crates/cmd-split/Cargo.toml new file mode 100644 index 0000000000..c0aa7cc026 --- /dev/null +++ b/software/coreutils/native/crates/cmd-split/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-split" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "split standalone binary for secure-exec VM" + +[[bin]] +name = "split" +path = "src/main.rs" + +[dependencies] +uu_split = "0.7.0" diff --git a/registry/native/crates/commands/split/src/main.rs b/software/coreutils/native/crates/cmd-split/src/main.rs similarity index 100% rename from registry/native/crates/commands/split/src/main.rs rename to software/coreutils/native/crates/cmd-split/src/main.rs diff --git a/software/coreutils/native/crates/cmd-stat/Cargo.toml b/software/coreutils/native/crates/cmd-stat/Cargo.toml new file mode 100644 index 0000000000..ee741b2614 --- /dev/null +++ b/software/coreutils/native/crates/cmd-stat/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-stat" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "stat standalone binary for secure-exec VM" + +[[bin]] +name = "stat" +path = "src/main.rs" + +[dependencies] +uu_stat = "0.7.0" diff --git a/registry/native/crates/commands/stat/src/main.rs b/software/coreutils/native/crates/cmd-stat/src/main.rs similarity index 100% rename from registry/native/crates/commands/stat/src/main.rs rename to software/coreutils/native/crates/cmd-stat/src/main.rs diff --git a/software/coreutils/native/crates/cmd-stdbuf/Cargo.toml b/software/coreutils/native/crates/cmd-stdbuf/Cargo.toml new file mode 100644 index 0000000000..499e2ef4d7 --- /dev/null +++ b/software/coreutils/native/crates/cmd-stdbuf/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-stdbuf" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "stdbuf standalone binary for secure-exec VM" + +[[bin]] +name = "stdbuf" +path = "src/main.rs" + +[dependencies] +shims = { package = "secureexec-shims", path = "../../../../../toolchain/crates/libs/shims" } diff --git a/registry/native/crates/commands/stdbuf/src/main.rs b/software/coreutils/native/crates/cmd-stdbuf/src/main.rs similarity index 100% rename from registry/native/crates/commands/stdbuf/src/main.rs rename to software/coreutils/native/crates/cmd-stdbuf/src/main.rs diff --git a/registry/native/crates/commands/stdbuf/tests/streaming.rs b/software/coreutils/native/crates/cmd-stdbuf/tests/streaming.rs similarity index 100% rename from registry/native/crates/commands/stdbuf/tests/streaming.rs rename to software/coreutils/native/crates/cmd-stdbuf/tests/streaming.rs diff --git a/software/coreutils/native/crates/cmd-strings/Cargo.toml b/software/coreutils/native/crates/cmd-strings/Cargo.toml new file mode 100644 index 0000000000..4ad7da7c02 --- /dev/null +++ b/software/coreutils/native/crates/cmd-strings/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-strings" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "strings standalone binary for secure-exec VM" + +[[bin]] +name = "strings" +path = "src/main.rs" + +[dependencies] +secureexec-strings-cmd = { path = "../strings-cmd" } diff --git a/registry/native/crates/commands/strings/src/main.rs b/software/coreutils/native/crates/cmd-strings/src/main.rs similarity index 100% rename from registry/native/crates/commands/strings/src/main.rs rename to software/coreutils/native/crates/cmd-strings/src/main.rs diff --git a/software/coreutils/native/crates/cmd-sum/Cargo.toml b/software/coreutils/native/crates/cmd-sum/Cargo.toml new file mode 100644 index 0000000000..5b07f6452b --- /dev/null +++ b/software/coreutils/native/crates/cmd-sum/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-sum" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "sum standalone binary for secure-exec VM" + +[[bin]] +name = "sum" +path = "src/main.rs" + +[dependencies] +uu_sum = "0.7.0" diff --git a/registry/native/crates/commands/sum/src/main.rs b/software/coreutils/native/crates/cmd-sum/src/main.rs similarity index 100% rename from registry/native/crates/commands/sum/src/main.rs rename to software/coreutils/native/crates/cmd-sum/src/main.rs diff --git a/software/coreutils/native/crates/cmd-tac/Cargo.toml b/software/coreutils/native/crates/cmd-tac/Cargo.toml new file mode 100644 index 0000000000..f0069010db --- /dev/null +++ b/software/coreutils/native/crates/cmd-tac/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-tac" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "tac standalone binary for secure-exec VM" + +[[bin]] +name = "tac" +path = "src/main.rs" + +[dependencies] +uu_tac = "0.7.0" diff --git a/registry/native/crates/commands/tac/src/main.rs b/software/coreutils/native/crates/cmd-tac/src/main.rs similarity index 100% rename from registry/native/crates/commands/tac/src/main.rs rename to software/coreutils/native/crates/cmd-tac/src/main.rs diff --git a/software/coreutils/native/crates/cmd-tail/Cargo.toml b/software/coreutils/native/crates/cmd-tail/Cargo.toml new file mode 100644 index 0000000000..7a14ea0dba --- /dev/null +++ b/software/coreutils/native/crates/cmd-tail/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-tail" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "tail standalone binary for secure-exec VM" + +[[bin]] +name = "tail" +path = "src/main.rs" + +[dependencies] +uu_tail = "0.7.0" diff --git a/registry/native/crates/commands/tail/src/main.rs b/software/coreutils/native/crates/cmd-tail/src/main.rs similarity index 100% rename from registry/native/crates/commands/tail/src/main.rs rename to software/coreutils/native/crates/cmd-tail/src/main.rs diff --git a/software/coreutils/native/crates/cmd-tee/Cargo.toml b/software/coreutils/native/crates/cmd-tee/Cargo.toml new file mode 100644 index 0000000000..c94c0664e7 --- /dev/null +++ b/software/coreutils/native/crates/cmd-tee/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-tee" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "tee standalone binary for secure-exec VM" + +[[bin]] +name = "tee" +path = "src/main.rs" + +[dependencies] +uu_tee = "0.7.0" diff --git a/registry/native/crates/commands/tee/src/main.rs b/software/coreutils/native/crates/cmd-tee/src/main.rs similarity index 100% rename from registry/native/crates/commands/tee/src/main.rs rename to software/coreutils/native/crates/cmd-tee/src/main.rs diff --git a/software/coreutils/native/crates/cmd-test/Cargo.toml b/software/coreutils/native/crates/cmd-test/Cargo.toml new file mode 100644 index 0000000000..1a1581e18b --- /dev/null +++ b/software/coreutils/native/crates/cmd-test/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-test" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "test/[ standalone binary for secure-exec VM" + +[[bin]] +name = "test" +path = "src/main.rs" + +[dependencies] +secureexec-builtins = { path = "../../../../../toolchain/crates/libs/builtins" } diff --git a/registry/native/crates/commands/test/src/main.rs b/software/coreutils/native/crates/cmd-test/src/main.rs similarity index 100% rename from registry/native/crates/commands/test/src/main.rs rename to software/coreutils/native/crates/cmd-test/src/main.rs diff --git a/software/coreutils/native/crates/cmd-timeout/Cargo.toml b/software/coreutils/native/crates/cmd-timeout/Cargo.toml new file mode 100644 index 0000000000..af6e289f59 --- /dev/null +++ b/software/coreutils/native/crates/cmd-timeout/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-timeout" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "timeout standalone binary for secure-exec VM" + +[[bin]] +name = "timeout" +path = "src/main.rs" + +[dependencies] +shims = { package = "secureexec-shims", path = "../../../../../toolchain/crates/libs/shims" } diff --git a/registry/native/crates/commands/timeout/src/main.rs b/software/coreutils/native/crates/cmd-timeout/src/main.rs similarity index 100% rename from registry/native/crates/commands/timeout/src/main.rs rename to software/coreutils/native/crates/cmd-timeout/src/main.rs diff --git a/software/coreutils/native/crates/cmd-touch/Cargo.toml b/software/coreutils/native/crates/cmd-touch/Cargo.toml new file mode 100644 index 0000000000..6353e89c77 --- /dev/null +++ b/software/coreutils/native/crates/cmd-touch/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-touch" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "touch standalone binary for secure-exec VM" + +[[bin]] +name = "touch" +path = "src/main.rs" + +[dependencies] +uu_touch = "0.7.0" diff --git a/registry/native/crates/commands/touch/src/main.rs b/software/coreutils/native/crates/cmd-touch/src/main.rs similarity index 100% rename from registry/native/crates/commands/touch/src/main.rs rename to software/coreutils/native/crates/cmd-touch/src/main.rs diff --git a/software/coreutils/native/crates/cmd-tr/Cargo.toml b/software/coreutils/native/crates/cmd-tr/Cargo.toml new file mode 100644 index 0000000000..c11682cac7 --- /dev/null +++ b/software/coreutils/native/crates/cmd-tr/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-tr" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "tr standalone binary for secure-exec VM" + +[[bin]] +name = "tr" +path = "src/main.rs" + +[dependencies] +uu_tr = "0.7.0" diff --git a/registry/native/crates/commands/tr/src/main.rs b/software/coreutils/native/crates/cmd-tr/src/main.rs similarity index 100% rename from registry/native/crates/commands/tr/src/main.rs rename to software/coreutils/native/crates/cmd-tr/src/main.rs diff --git a/software/coreutils/native/crates/cmd-true/Cargo.toml b/software/coreutils/native/crates/cmd-true/Cargo.toml new file mode 100644 index 0000000000..bed79e45c4 --- /dev/null +++ b/software/coreutils/native/crates/cmd-true/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-true" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "true standalone binary for secure-exec VM" + +[[bin]] +name = "true" +path = "src/main.rs" + +[dependencies] +uu_true = "0.7.0" diff --git a/registry/native/crates/commands/true/src/main.rs b/software/coreutils/native/crates/cmd-true/src/main.rs similarity index 100% rename from registry/native/crates/commands/true/src/main.rs rename to software/coreutils/native/crates/cmd-true/src/main.rs diff --git a/software/coreutils/native/crates/cmd-truncate/Cargo.toml b/software/coreutils/native/crates/cmd-truncate/Cargo.toml new file mode 100644 index 0000000000..65fa7e80fd --- /dev/null +++ b/software/coreutils/native/crates/cmd-truncate/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-truncate" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "truncate standalone binary for secure-exec VM" + +[[bin]] +name = "truncate" +path = "src/main.rs" + +[dependencies] +uu_truncate = "0.7.0" diff --git a/registry/native/crates/commands/truncate/src/main.rs b/software/coreutils/native/crates/cmd-truncate/src/main.rs similarity index 100% rename from registry/native/crates/commands/truncate/src/main.rs rename to software/coreutils/native/crates/cmd-truncate/src/main.rs diff --git a/software/coreutils/native/crates/cmd-tsort/Cargo.toml b/software/coreutils/native/crates/cmd-tsort/Cargo.toml new file mode 100644 index 0000000000..b0e057b0c8 --- /dev/null +++ b/software/coreutils/native/crates/cmd-tsort/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-tsort" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "tsort standalone binary for secure-exec VM" + +[[bin]] +name = "tsort" +path = "src/main.rs" + +[dependencies] +uu_tsort = "0.7.0" diff --git a/registry/native/crates/commands/tsort/src/main.rs b/software/coreutils/native/crates/cmd-tsort/src/main.rs similarity index 100% rename from registry/native/crates/commands/tsort/src/main.rs rename to software/coreutils/native/crates/cmd-tsort/src/main.rs diff --git a/software/coreutils/native/crates/cmd-uname/Cargo.toml b/software/coreutils/native/crates/cmd-uname/Cargo.toml new file mode 100644 index 0000000000..b0f0a8c71a --- /dev/null +++ b/software/coreutils/native/crates/cmd-uname/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-uname" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "uname standalone binary for secure-exec VM" + +[[bin]] +name = "uname" +path = "src/main.rs" + +[dependencies] +uu_uname = "0.7.0" diff --git a/registry/native/crates/commands/uname/src/main.rs b/software/coreutils/native/crates/cmd-uname/src/main.rs similarity index 100% rename from registry/native/crates/commands/uname/src/main.rs rename to software/coreutils/native/crates/cmd-uname/src/main.rs diff --git a/software/coreutils/native/crates/cmd-unexpand/Cargo.toml b/software/coreutils/native/crates/cmd-unexpand/Cargo.toml new file mode 100644 index 0000000000..d8a9dc9e78 --- /dev/null +++ b/software/coreutils/native/crates/cmd-unexpand/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-unexpand" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "unexpand standalone binary for secure-exec VM" + +[[bin]] +name = "unexpand" +path = "src/main.rs" + +[dependencies] +uu_unexpand = "0.7.0" diff --git a/registry/native/crates/commands/unexpand/src/main.rs b/software/coreutils/native/crates/cmd-unexpand/src/main.rs similarity index 100% rename from registry/native/crates/commands/unexpand/src/main.rs rename to software/coreutils/native/crates/cmd-unexpand/src/main.rs diff --git a/software/coreutils/native/crates/cmd-uniq/Cargo.toml b/software/coreutils/native/crates/cmd-uniq/Cargo.toml new file mode 100644 index 0000000000..c3ead17984 --- /dev/null +++ b/software/coreutils/native/crates/cmd-uniq/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-uniq" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "uniq standalone binary for secure-exec VM" + +[[bin]] +name = "uniq" +path = "src/main.rs" + +[dependencies] +uu_uniq = "0.7.0" diff --git a/registry/native/crates/commands/uniq/src/main.rs b/software/coreutils/native/crates/cmd-uniq/src/main.rs similarity index 100% rename from registry/native/crates/commands/uniq/src/main.rs rename to software/coreutils/native/crates/cmd-uniq/src/main.rs diff --git a/software/coreutils/native/crates/cmd-unlink/Cargo.toml b/software/coreutils/native/crates/cmd-unlink/Cargo.toml new file mode 100644 index 0000000000..eb669d0add --- /dev/null +++ b/software/coreutils/native/crates/cmd-unlink/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-unlink" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "unlink standalone binary for secure-exec VM" + +[[bin]] +name = "unlink" +path = "src/main.rs" + +[dependencies] +uu_unlink = "0.7.0" diff --git a/registry/native/crates/commands/unlink/src/main.rs b/software/coreutils/native/crates/cmd-unlink/src/main.rs similarity index 100% rename from registry/native/crates/commands/unlink/src/main.rs rename to software/coreutils/native/crates/cmd-unlink/src/main.rs diff --git a/software/coreutils/native/crates/cmd-wc/Cargo.toml b/software/coreutils/native/crates/cmd-wc/Cargo.toml new file mode 100644 index 0000000000..728b9fdad7 --- /dev/null +++ b/software/coreutils/native/crates/cmd-wc/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-wc" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "wc standalone binary for secure-exec VM" + +[[bin]] +name = "wc" +path = "src/main.rs" + +[dependencies] +uu_wc = "0.7.0" diff --git a/registry/native/crates/commands/wc/src/main.rs b/software/coreutils/native/crates/cmd-wc/src/main.rs similarity index 100% rename from registry/native/crates/commands/wc/src/main.rs rename to software/coreutils/native/crates/cmd-wc/src/main.rs diff --git a/software/coreutils/native/crates/cmd-which/Cargo.toml b/software/coreutils/native/crates/cmd-which/Cargo.toml new file mode 100644 index 0000000000..0ea0f38b94 --- /dev/null +++ b/software/coreutils/native/crates/cmd-which/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-which" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "which standalone binary for secure-exec VM" + +[[bin]] +name = "which" +path = "src/main.rs" + +[dependencies] +shims = { package = "secureexec-shims", path = "../../../../../toolchain/crates/libs/shims" } diff --git a/registry/native/crates/commands/which/src/main.rs b/software/coreutils/native/crates/cmd-which/src/main.rs similarity index 100% rename from registry/native/crates/commands/which/src/main.rs rename to software/coreutils/native/crates/cmd-which/src/main.rs diff --git a/registry/native/crates/commands/which/tests/executable_mode.rs b/software/coreutils/native/crates/cmd-which/tests/executable_mode.rs similarity index 100% rename from registry/native/crates/commands/which/tests/executable_mode.rs rename to software/coreutils/native/crates/cmd-which/tests/executable_mode.rs diff --git a/software/coreutils/native/crates/cmd-whoami/Cargo.toml b/software/coreutils/native/crates/cmd-whoami/Cargo.toml new file mode 100644 index 0000000000..988ca47dae --- /dev/null +++ b/software/coreutils/native/crates/cmd-whoami/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-whoami" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "whoami standalone binary for secure-exec VM" + +[[bin]] +name = "whoami" +path = "src/main.rs" + +[dependencies] +secureexec-builtins = { path = "../../../../../toolchain/crates/libs/builtins" } diff --git a/registry/native/crates/commands/whoami/src/main.rs b/software/coreutils/native/crates/cmd-whoami/src/main.rs similarity index 100% rename from registry/native/crates/commands/whoami/src/main.rs rename to software/coreutils/native/crates/cmd-whoami/src/main.rs diff --git a/software/coreutils/native/crates/cmd-yes/Cargo.toml b/software/coreutils/native/crates/cmd-yes/Cargo.toml new file mode 100644 index 0000000000..d32f9d582d --- /dev/null +++ b/software/coreutils/native/crates/cmd-yes/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-yes" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "yes standalone binary for secure-exec VM" + +[[bin]] +name = "yes" +path = "src/main.rs" + +[dependencies] +uu_yes = "0.7.0" diff --git a/registry/native/crates/commands/yes/src/main.rs b/software/coreutils/native/crates/cmd-yes/src/main.rs similarity index 100% rename from registry/native/crates/commands/yes/src/main.rs rename to software/coreutils/native/crates/cmd-yes/src/main.rs diff --git a/software/coreutils/native/crates/column/Cargo.toml b/software/coreutils/native/crates/column/Cargo.toml new file mode 100644 index 0000000000..3134b792fd --- /dev/null +++ b/software/coreutils/native/crates/column/Cargo.toml @@ -0,0 +1,9 @@ +[package] +workspace = "../../../../../toolchain" +name = "secureexec-column" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "column implementation for secure-exec standalone binaries" + +[dependencies] diff --git a/registry/native/crates/libs/column/src/lib.rs b/software/coreutils/native/crates/column/src/lib.rs similarity index 100% rename from registry/native/crates/libs/column/src/lib.rs rename to software/coreutils/native/crates/column/src/lib.rs diff --git a/software/coreutils/native/crates/du/Cargo.toml b/software/coreutils/native/crates/du/Cargo.toml new file mode 100644 index 0000000000..f167b14657 --- /dev/null +++ b/software/coreutils/native/crates/du/Cargo.toml @@ -0,0 +1,9 @@ +[package] +workspace = "../../../../../toolchain" +name = "secureexec-du" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "du implementation for secure-exec standalone binaries" + +[dependencies] diff --git a/registry/native/crates/libs/du/src/lib.rs b/software/coreutils/native/crates/du/src/lib.rs similarity index 100% rename from registry/native/crates/libs/du/src/lib.rs rename to software/coreutils/native/crates/du/src/lib.rs diff --git a/software/coreutils/native/crates/expr/Cargo.toml b/software/coreutils/native/crates/expr/Cargo.toml new file mode 100644 index 0000000000..e8f413d5c6 --- /dev/null +++ b/software/coreutils/native/crates/expr/Cargo.toml @@ -0,0 +1,10 @@ +[package] +workspace = "../../../../../toolchain" +name = "secureexec-expr" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "expr implementation for secure-exec standalone binaries" + +[dependencies] +regex = "1" diff --git a/registry/native/crates/libs/expr/src/lib.rs b/software/coreutils/native/crates/expr/src/lib.rs similarity index 100% rename from registry/native/crates/libs/expr/src/lib.rs rename to software/coreutils/native/crates/expr/src/lib.rs diff --git a/software/coreutils/native/crates/rev/Cargo.toml b/software/coreutils/native/crates/rev/Cargo.toml new file mode 100644 index 0000000000..3c8a175e92 --- /dev/null +++ b/software/coreutils/native/crates/rev/Cargo.toml @@ -0,0 +1,9 @@ +[package] +workspace = "../../../../../toolchain" +name = "secureexec-rev" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "rev implementation for secure-exec standalone binaries" + +[dependencies] diff --git a/registry/native/crates/libs/rev/src/lib.rs b/software/coreutils/native/crates/rev/src/lib.rs similarity index 100% rename from registry/native/crates/libs/rev/src/lib.rs rename to software/coreutils/native/crates/rev/src/lib.rs diff --git a/software/coreutils/native/crates/strings-cmd/Cargo.toml b/software/coreutils/native/crates/strings-cmd/Cargo.toml new file mode 100644 index 0000000000..7c3c023a54 --- /dev/null +++ b/software/coreutils/native/crates/strings-cmd/Cargo.toml @@ -0,0 +1,9 @@ +[package] +workspace = "../../../../../toolchain" +name = "secureexec-strings-cmd" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "strings command implementation for secure-exec standalone binaries" + +[dependencies] diff --git a/registry/native/crates/libs/strings-cmd/src/lib.rs b/software/coreutils/native/crates/strings-cmd/src/lib.rs similarity index 100% rename from registry/native/crates/libs/strings-cmd/src/lib.rs rename to software/coreutils/native/crates/strings-cmd/src/lib.rs diff --git a/software/coreutils/package.json b/software/coreutils/package.json new file mode 100644 index 0000000000..d17440e7a4 --- /dev/null +++ b/software/coreutils/package.json @@ -0,0 +1,33 @@ +{ + "name": "@agentos-software/coreutils", + "version": "0.3.3", + "type": "module", + "license": "Apache-2.0", + "description": "GNU coreutils for secure-exec VMs (sh, cat, ls, cp, sort, and 80+ commands)", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "files": [ + "dist", + "!dist/package", + "!dist/package.tar" + ], + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "scripts": { + "build": "agentos-toolchain stage --commands-dir ../../toolchain/target/wasm32-wasip1/release/commands --if-missing skip && tsc && agentos-toolchain build", + "check-types": "tsc --noEmit", + "test": "vitest run test/ --passWithNoTests" + }, + "devDependencies": { + "@agentos-software/manifest": "workspace:*", + "@rivet-dev/agentos-toolchain": "workspace:*", + "@types/node": "^22.10.2", + "typescript": "^5.9.2", + "@agentos/test-harness": "workspace:*", + "vitest": "^2.1.9" + } +} diff --git a/registry/agent/pi/src/index.ts b/software/coreutils/src/index.ts similarity index 100% rename from registry/agent/pi/src/index.ts rename to software/coreutils/src/index.ts diff --git a/registry/tests/wasmvm/shell-redirect.test.ts b/software/coreutils/test/shell-redirect.test.ts similarity index 97% rename from registry/tests/wasmvm/shell-redirect.test.ts rename to software/coreutils/test/shell-redirect.test.ts index ccee94a39b..203fec8844 100644 --- a/registry/tests/wasmvm/shell-redirect.test.ts +++ b/software/coreutils/test/shell-redirect.test.ts @@ -7,7 +7,7 @@ import { describeIf, hasWasmBinaries, type Kernel, -} from "../helpers.js"; +} from '@agentos/test-harness'; describeIf(hasWasmBinaries, "wasmvm shell redirects", () => { let kernel: Kernel | undefined; diff --git a/registry/tests/wasmvm/shell-terminal.test.ts b/software/coreutils/test/shell-terminal.test.ts similarity index 98% rename from registry/tests/wasmvm/shell-terminal.test.ts rename to software/coreutils/test/shell-terminal.test.ts index 7efaffbb69..ca99bb54eb 100644 --- a/registry/tests/wasmvm/shell-terminal.test.ts +++ b/software/coreutils/test/shell-terminal.test.ts @@ -7,10 +7,10 @@ */ import { describe, it, expect, afterEach } from "vitest"; -import { TerminalHarness } from './terminal-harness.js'; -import { createWasmVmRuntime } from '../helpers.js'; -import { COMMANDS_DIR, createKernel, describeIf, hasWasmBinaries } from '../helpers.js'; -import type { Kernel } from "../helpers.js"; +import { TerminalHarness } from '@agentos/test-harness'; +import { createWasmVmRuntime } from '@agentos/test-harness'; +import { COMMANDS_DIR, createKernel, describeIf, hasWasmBinaries } from '@agentos/test-harness'; +import type { Kernel } from '@agentos/test-harness'; /** brush-shell interactive prompt (captured empirically). */ const PROMPT = "sh-0.4$ "; @@ -169,7 +169,7 @@ describeIf(hasWasmBinaries, "wasmvm-shell-terminal", () => { expect(harness.screenshotTrimmed()).toBe( [`${PROMPT}echo hello`, "hello", PROMPT].join("\n"), ); - }); + }, 15_000); it("ls / shows listing — directory entries include /bin from command registration", async () => { const { kernel } = await createShellKernel(); diff --git a/software/coreutils/tsconfig.json b/software/coreutils/tsconfig.json new file mode 100644 index 0000000000..03ce790ab7 --- /dev/null +++ b/software/coreutils/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*"] +} diff --git a/software/curl/agentos-package.json b/software/curl/agentos-package.json new file mode 100644 index 0000000000..98a4c83f33 --- /dev/null +++ b/software/curl/agentos-package.json @@ -0,0 +1,12 @@ +{ + "commands": [ + "curl" + ], + "registry": { + "title": "curl", + "description": "HTTP(S) client for fetching URLs and APIs.", + "priority": 50, + "image": "/images/registry/curl.svg", + "category": "networking" + } +} diff --git a/registry/native/c/curl-upstream-overlay/lib/curl_setup.h b/software/curl/native/c/overlay/lib/curl_setup.h similarity index 99% rename from registry/native/c/curl-upstream-overlay/lib/curl_setup.h rename to software/curl/native/c/overlay/lib/curl_setup.h index 8edcf809eb..50b4479f23 100644 --- a/registry/native/c/curl-upstream-overlay/lib/curl_setup.h +++ b/software/curl/native/c/overlay/lib/curl_setup.h @@ -742,7 +742,7 @@ #if defined(USE_GNUTLS) || defined(USE_OPENSSL) || defined(USE_MBEDTLS) || \ defined(USE_WOLFSSL) || defined(USE_SCHANNEL) || defined(USE_SECTRANSP) || \ - defined(USE_BEARSSL) || defined(USE_RUSTLS) || defined(USE_WASI_TLS) + defined(USE_BEARSSL) || defined(USE_RUSTLS) #define USE_SSL /* SSL support has been enabled */ #endif diff --git a/registry/native/c/curl-upstream-overlay/lib/vtls/vtls.c b/software/curl/native/c/overlay/lib/vtls/vtls.c similarity index 99% rename from registry/native/c/curl-upstream-overlay/lib/vtls/vtls.c rename to software/curl/native/c/overlay/lib/vtls/vtls.c index 21b758f2e8..61b407ab22 100644 --- a/registry/native/c/curl-upstream-overlay/lib/vtls/vtls.c +++ b/software/curl/native/c/overlay/lib/vtls/vtls.c @@ -64,7 +64,6 @@ #include "mbedtls.h" /* mbedTLS versions */ #include "bearssl.h" /* BearSSL versions */ #include "rustls.h" /* Rustls versions */ -#include "wasi_tls.h" /* WASI host TLS */ #include "slist.h" #include "sendf.h" @@ -1380,8 +1379,6 @@ static const struct Curl_ssl Curl_ssl_multi = { const struct Curl_ssl *Curl_ssl = #if defined(CURL_WITH_MULTI_SSL) &Curl_ssl_multi; -#elif defined(USE_WASI_TLS) - &Curl_ssl_wasi_tls; #elif defined(USE_WOLFSSL) &Curl_ssl_wolfssl; #elif defined(USE_GNUTLS) @@ -1426,9 +1423,6 @@ static const struct Curl_ssl *available_backends[] = { #endif #if defined(USE_RUSTLS) &Curl_ssl_rustls, -#endif -#if defined(USE_WASI_TLS) - &Curl_ssl_wasi_tls, #endif NULL }; diff --git a/software/curl/native/crates/cmd-curl/Cargo.toml b/software/curl/native/crates/cmd-curl/Cargo.toml new file mode 100644 index 0000000000..5a65154842 --- /dev/null +++ b/software/curl/native/crates/cmd-curl/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-curl" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "Minimal curl-compatible HTTP client for secure-exec" + +[[bin]] +name = "curl" +path = "src/main.rs" + +[dependencies] +wasi-http = { package = "secureexec-wasi-http", path = "../../../../../toolchain/crates/libs/wasi-http" } diff --git a/registry/native/crates/commands/curl/src/main.rs b/software/curl/native/crates/cmd-curl/src/main.rs similarity index 100% rename from registry/native/crates/commands/curl/src/main.rs rename to software/curl/native/crates/cmd-curl/src/main.rs diff --git a/software/curl/package.json b/software/curl/package.json new file mode 100644 index 0000000000..9a0ce7d95e --- /dev/null +++ b/software/curl/package.json @@ -0,0 +1,33 @@ +{ + "name": "@agentos-software/curl", + "version": "0.3.3", + "type": "module", + "license": "Apache-2.0", + "description": "curl HTTP client for secure-exec VMs", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "files": [ + "dist", + "!dist/package", + "!dist/package.tar" + ], + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "scripts": { + "build": "agentos-toolchain stage --commands-dir ../../toolchain/target/wasm32-wasip1/release/commands --if-missing skip && tsc && agentos-toolchain build", + "check-types": "tsc --noEmit", + "test": "vitest run test/ --passWithNoTests" + }, + "devDependencies": { + "@agentos-software/manifest": "workspace:*", + "@rivet-dev/agentos-toolchain": "workspace:*", + "@types/node": "^22.10.2", + "typescript": "^5.9.2", + "@agentos/test-harness": "workspace:*", + "vitest": "^2.1.9" + } +} diff --git a/registry/software/browserbase/src/index.ts b/software/curl/src/index.ts similarity index 100% rename from registry/software/browserbase/src/index.ts rename to software/curl/src/index.ts diff --git a/software/curl/test/curl.test.ts b/software/curl/test/curl.test.ts new file mode 100644 index 0000000000..d79395baf1 --- /dev/null +++ b/software/curl/test/curl.test.ts @@ -0,0 +1,918 @@ +/** + * Integration tests for curl and the C socket layer (host_socket.c). + * + * Tests the WASM socket implementation that powers curl: + * - DNS resolution (getaddrinfo) + * - TCP socket creation and connection + * - Non-blocking socket mode (fcntl O_NONBLOCK) + * - Socket options (getsockopt SO_ERROR, setsockopt TCP_NODELAY) + * - Poll for readability/writability + * - HTTP send/recv over raw sockets + * - Remote endpoint connectivity + */ + +import { describe, it, expect, afterEach, beforeAll, afterAll } from 'vitest'; +import { createWasmVmRuntime } from '@agentos/test-harness'; +import { + allowAll, + C_BUILD_DIR, + COMMANDS_DIR, + createInMemoryFileSystem, + createKernel, + describeIf, + hasCWasmBinaries, + hasWasmBinaries, + itIf, +} from '@agentos/test-harness'; +import type { Kernel } from '@agentos/test-harness'; +import { + createServer as createHttpServer, + type IncomingMessage, + type Server as HttpServer, + type ServerResponse, +} from 'node:http'; +import { createServer as createHttpsServer, type Server as HttpsServer } from 'node:https'; +import { + createConnection, + createServer as createTcpServer, + type Server as TcpServer, +} from 'node:net'; +import { execSync } from 'node:child_process'; +import { + existsSync, + mkdtempSync, + readFileSync, + rmSync, + unlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { brotliCompressSync, gzipSync, zstdCompressSync } from 'node:zlib'; + +// The upstream curl parity assertions below only hold for the C-built curl +// artifact; the Rust fallback in COMMANDS_DIR intentionally supports a smaller +// flag surface and should not be used for these cases. +const hasHttpGetTest = hasWasmBinaries && existsSync(resolve(COMMANDS_DIR, 'http_get_test')); +const hasCurl = hasCWasmBinaries('curl'); +const runExternalNetwork = process.env.AGENTOS_E2E_NETWORK === '1'; +const EXTERNAL_HOST = 'example.com'; +const EXTERNAL_TCP_PORT = 80; +const EXTERNAL_HTTP_URL = `http://${EXTERNAL_HOST}/`; +const EXTERNAL_HTTPS_URL = `https://${EXTERNAL_HOST}/`; +const EXTERNAL_EXPECTED_BODY = 'Example Domain'; +const EXTERNAL_RETRY_ATTEMPTS = 3; +const EXTERNAL_RETRY_DELAY_MS = 1_000; +const EXTERNAL_PROBE_TIMEOUT_MS = 8_000; +let hasOpenssl = false; + +try { + execSync('openssl version', { stdio: 'pipe' }); + hasOpenssl = true; +} catch { + hasOpenssl = false; +} + +function sleep(ms: number): Promise { + return new Promise((resolveSleep) => setTimeout(resolveSleep, ms)); +} + +function formatError(error: unknown): string { + if (error instanceof Error) return error.message; + return String(error); +} + +async function retryExternal(run: () => Promise, attempts = EXTERNAL_RETRY_ATTEMPTS): Promise { + let lastError: unknown; + for (let attempt = 1; attempt <= attempts; attempt += 1) { + try { + return await run(); + } catch (error) { + lastError = error; + if (attempt < attempts) { + await sleep(EXTERNAL_RETRY_DELAY_MS); + } + } + } + + throw lastError ?? new Error('external network probe failed'); +} + +async function probeExternalTcp(): Promise { + await new Promise((resolveConnect, rejectConnect) => { + const socket = createConnection({ + host: EXTERNAL_HOST, + port: EXTERNAL_TCP_PORT, + }); + let settled = false; + + const finish = (callback: () => void) => { + if (settled) return; + settled = true; + callback(); + }; + + socket.setTimeout(EXTERNAL_PROBE_TIMEOUT_MS); + socket.once('connect', () => { + finish(() => { + socket.end(); + resolveConnect(); + }); + }); + socket.once('timeout', () => { + finish(() => { + socket.destroy(); + rejectConnect(new Error(`timed out connecting to ${EXTERNAL_HOST}:${EXTERNAL_TCP_PORT}`)); + }); + }); + socket.once('error', (error) => { + finish(() => { + socket.destroy(); + rejectConnect(error); + }); + }); + }); +} + +async function probeExternalHttps(): Promise { + const response = await fetch(EXTERNAL_HTTPS_URL, { + signal: AbortSignal.timeout(EXTERNAL_PROBE_TIMEOUT_MS), + }); + if (!response.ok) { + throw new Error(`host probe failed with HTTP ${response.status}`); + } + await response.arrayBuffer(); +} + +const externalNetworkSkipReason = runExternalNetwork + ? await (async () => { + try { + await retryExternal(async () => { + await probeExternalTcp(); + await probeExternalHttps(); + }); + return false as const; + } catch (error) { + return `external network unavailable: ${formatError(error)}`; + } + })() + : 'set AGENTOS_E2E_NETWORK=1 to enable external-network coverage'; + +// A known, highly compressible payload used by the --compressed round-trip +// tests. Long + repetitive so every codec produces a body distinct from the +// plaintext (proving curl actually inflated it, not just passed it through). +const COMPRESSION_PAYLOAD = + 'agentos-compression-parity ' + 'the quick brown fox jumps over the lazy dog. '.repeat(64); + +// Build a real CA and a leaf server certificate signed by it, with a SAN that +// covers the 127.0.0.1 loopback endpoint the tests connect to. This lets curl's +// mbedTLS backend perform genuine chain + hostname verification, exactly like +// Linux curl against a private CA. +function makeCaSignedCert(caCommonName: string): { + caPem: string; + serverKey: string; + serverCert: string; +} { + const dir = mkdtempSync(join(tmpdir(), 'curl-ca-')); + try { + execSync(`openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out "${dir}/ca.key" 2>/dev/null`); + execSync( + `openssl req -x509 -new -key "${dir}/ca.key" -days 3650 -subj "/CN=${caCommonName}" -out "${dir}/ca.crt" 2>/dev/null`, + ); + execSync(`openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out "${dir}/srv.key" 2>/dev/null`); + execSync(`openssl req -new -key "${dir}/srv.key" -subj "/CN=localhost" -out "${dir}/srv.csr" 2>/dev/null`); + writeFileSync(`${dir}/ext.cnf`, 'subjectAltName=DNS:localhost,IP:127.0.0.1\n'); + execSync( + `openssl x509 -req -in "${dir}/srv.csr" -CA "${dir}/ca.crt" -CAkey "${dir}/ca.key" ` + + `-CAcreateserial -days 3650 -extfile "${dir}/ext.cnf" -out "${dir}/srv.crt" 2>/dev/null`, + ); + return { + caPem: readFileSync(`${dir}/ca.crt`, 'utf8'), + serverKey: readFileSync(`${dir}/srv.key`, 'utf8'), + serverCert: readFileSync(`${dir}/srv.crt`, 'utf8'), + }; + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +function generateSelfSignedCert(): { key: string; cert: string } { + const keyPath = join(tmpdir(), `curl-test-key-${process.pid}-${Date.now()}.pem`); + try { + const key = execSync( + 'openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 2>/dev/null', + { encoding: 'utf8' }, + ); + writeFileSync(keyPath, key); + const cert = execSync( + `openssl req -new -x509 -key "${keyPath}" -days 1 -subj "/CN=localhost" -addext "subjectAltName=DNS:localhost,IP:127.0.0.1" 2>/dev/null`, + { encoding: 'utf8' }, + ); + return { key, cert }; + } finally { + try { + unlinkSync(keyPath); + } catch { + // Best effort cleanup for test temp files. + } + } +} + +describeIf(hasCurl || hasHttpGetTest, 'curl and socket layer', () => { + let kernel: Kernel; + let httpServer: HttpServer; + let httpsServer: HttpsServer; + let validHttpsServer: HttpsServer; + let caHttpsServer: HttpsServer; + let keepAliveServer: TcpServer; + let httpPort: number; + let httpsPort: number; + let validHttpsPort: number; + let caHttpsPort: number; + let keepAlivePort: number; + // CA (PEM) trusted by the seeded /etc/ssl/certs/ca-certificates.crt bundle — + // it signs validHttpsServer's leaf. caOnlyPem signs caHttpsServer's leaf and + // is deliberately NOT in the bundle, so it verifies only via --cacert. + let seededCaPem = ''; + let caOnlyPem = ''; + let flakyRequestCount = 0; + + beforeAll(async () => { + httpServer = createHttpServer((req: IncomingMessage, res: ServerResponse) => { + const url = req.url ?? '/'; + + if (url === '/json') { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ ok: true, path: url })); + return; + } + + if (url === '/redirect') { + res.writeHead(302, { Location: `http://127.0.0.1:${httpPort}/final` }); + res.end(); + return; + } + + if (url === '/final') { + res.writeHead(200, { 'Content-Type': 'text/plain' }); + res.end('followed redirect'); + return; + } + + if (url === '/one') { + res.writeHead(200, { 'Content-Type': 'text/plain' }); + res.end('first-response\n'); + return; + } + + if (url === '/two') { + res.writeHead(200, { 'Content-Type': 'text/plain' }); + res.end('second-response\n'); + return; + } + + if (url === '/echo' && req.method === 'POST') { + let body = ''; + req.on('data', (chunk) => { + body += chunk; + }); + req.on('end', () => { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + method: req.method, + body, + header: req.headers['x-test'] ?? null, + })); + }); + return; + } + + if (url === '/json-post' && req.method === 'POST') { + let body = ''; + req.on('data', (chunk) => { + body += chunk; + }); + req.on('end', () => { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + method: req.method, + contentType: req.headers['content-type'] ?? null, + accept: req.headers.accept ?? null, + body, + })); + }); + return; + } + + if (url === '/head-test') { + res.writeHead(200, { + 'Content-Type': 'text/plain', + 'X-Test-Header': 'present', + }); + if (req.method === 'HEAD') { + res.end(); + } else { + res.end('body should not appear in HEAD output'); + } + return; + } + + if (url === '/auth-required') { + const auth = req.headers.authorization; + if (!auth || !auth.startsWith('Basic ')) { + res.writeHead(401, { 'Content-Type': 'text/plain' }); + res.end('unauthorized'); + return; + } + + const decoded = Buffer.from(auth.slice(6), 'base64').toString(); + res.writeHead(200, { 'Content-Type': 'text/plain' }); + res.end(`authenticated: ${decoded}`); + return; + } + + if (url === '/upload' && req.method === 'POST') { + const contentType = req.headers['content-type'] ?? ''; + const chunks: Buffer[] = []; + req.on('data', (chunk) => { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + }); + req.on('end', () => { + const body = Buffer.concat(chunks).toString(); + res.writeHead(200, { 'Content-Type': 'text/plain' }); + res.end( + `multipart: ${contentType.startsWith('multipart/form-data')}\n` + + `body-contains-file: ${body.includes('upload.txt')}`, + ); + }); + return; + } + + if (url === '/binary') { + const payload = Buffer.alloc(256); + for (let i = 0; i < payload.length; i++) payload[i] = i & 0xff; + res.writeHead(200, { + 'Content-Type': 'application/octet-stream', + 'Content-Length': String(payload.length), + }); + res.end(payload); + return; + } + + if (url === '/named.txt') { + const body = 'downloaded-by-remote-name\n'; + res.writeHead(200, { + 'Content-Type': 'text/plain', + 'Content-Length': String(Buffer.byteLength(body)), + }); + res.end(body); + return; + } + + if (url === '/flaky') { + flakyRequestCount += 1; + if (flakyRequestCount === 1) { + res.writeHead(503, { 'Content-Type': 'text/plain' }); + res.end('retry please'); + return; + } + + res.writeHead(200, { 'Content-Type': 'text/plain' }); + res.end('retry succeeded'); + return; + } + + if (url === '/status') { + res.writeHead(201, { 'Content-Type': 'text/plain' }); + res.end('created'); + return; + } + + if (url === '/gzip' || url === '/brotli' || url === '/zstd') { + const raw = Buffer.from(COMPRESSION_PAYLOAD); + let encoding: string; + let body: Buffer; + if (url === '/gzip') { + encoding = 'gzip'; + body = gzipSync(raw); + } else if (url === '/brotli') { + encoding = 'br'; + body = brotliCompressSync(raw); + } else { + encoding = 'zstd'; + body = zstdCompressSync(raw); + } + res.writeHead(200, { + 'Content-Type': 'text/plain', + 'Content-Encoding': encoding, + 'Content-Length': String(body.length), + }); + res.end(body); + return; + } + + res.writeHead(404, { 'Content-Type': 'text/plain' }); + res.end('not found'); + }); + + await new Promise((resolveListen) => { + httpServer.listen(0, '127.0.0.1', resolveListen); + }); + httpPort = (httpServer.address() as import('node:net').AddressInfo).port; + + if (hasOpenssl) { + const tlsCert = generateSelfSignedCert(); + httpsServer = createHttpsServer({ key: tlsCert.key, cert: tlsCert.cert }, (req, res) => { + if (req.url === '/json') { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ secure: true, path: req.url })); + return; + } + + if (req.url === '/keepalive') { + const body = 'hello from tls keepalive'; + res.writeHead(200, { + 'Content-Type': 'text/plain', + 'Content-Length': String(Buffer.byteLength(body)), + Connection: 'keep-alive', + 'Keep-Alive': 'timeout=60', + }); + res.end(body); + return; + } + + res.writeHead(404, { 'Content-Type': 'text/plain' }); + res.end('not found'); + }); + httpsServer.keepAliveTimeout = 60000; + + await new Promise((resolveListen) => { + httpsServer.listen(0, '127.0.0.1', resolveListen); + }); + httpsPort = (httpsServer.address() as import('node:net').AddressInfo).port; + + // HTTPS server whose leaf chains to a CA seeded into the guest's + // /etc/ssl/certs/ca-certificates.crt — verified with NO -k / --cacert. + const trusted = makeCaSignedCert('AgentOS Test Root CA'); + seededCaPem = trusted.caPem; + validHttpsServer = createHttpsServer( + { key: trusted.serverKey, cert: trusted.serverCert }, + (req, res) => { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ verified: true, path: req.url })); + }, + ); + await new Promise((resolveListen) => { + validHttpsServer.listen(0, '127.0.0.1', resolveListen); + }); + validHttpsPort = (validHttpsServer.address() as import('node:net').AddressInfo).port; + + // HTTPS server whose CA is provided ONLY via --cacert (not in the bundle). + const caOnly = makeCaSignedCert('AgentOS Cacert-Only CA'); + caOnlyPem = caOnly.caPem; + caHttpsServer = createHttpsServer( + { key: caOnly.serverKey, cert: caOnly.serverCert }, + (req, res) => { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ cacert: true, path: req.url })); + }, + ); + await new Promise((resolveListen) => { + caHttpsServer.listen(0, '127.0.0.1', resolveListen); + }); + caHttpsPort = (caHttpsServer.address() as import('node:net').AddressInfo).port; + } + + keepAliveServer = createTcpServer((socket) => { + socket.once('data', () => { + const body = 'hello from keepalive'; + socket.write( + 'HTTP/1.1 200 OK\r\n' + + 'Content-Type: text/plain\r\n' + + `Content-Length: ${Buffer.byteLength(body)}\r\n` + + 'Connection: keep-alive\r\n' + + 'Keep-Alive: timeout=60\r\n' + + '\r\n' + + body, + ); + // Intentionally keep the socket open to exercise curl shutdown logic. + }); + }); + + await new Promise((resolveListen) => { + keepAliveServer.listen(0, '127.0.0.1', resolveListen); + }); + keepAlivePort = (keepAliveServer.address() as import('node:net').AddressInfo).port; + }); + + afterAll(async () => { + if (httpServer) { + await new Promise((resolveClose) => httpServer.close(() => resolveClose())); + } + if (httpsServer) { + await new Promise((resolveClose) => httpsServer.close(() => resolveClose())); + } + if (validHttpsServer) { + await new Promise((resolveClose) => validHttpsServer.close(() => resolveClose())); + } + if (caHttpsServer) { + await new Promise((resolveClose) => caHttpsServer.close(() => resolveClose())); + } + if (keepAliveServer) { + await new Promise((resolveClose) => keepAliveServer.close(() => resolveClose())); + } + }); + + async function createKernelWithNet() { + flakyRequestCount = 0; + const filesystem = createInMemoryFileSystem(); + await (filesystem as any).chmod('/', 0o1777); + await filesystem.mkdir('/tmp', { recursive: true }); + await (filesystem as any).chmod('/tmp', 0o1777); + + kernel = createKernel({ + filesystem, + permissions: allowAll, + loopbackExemptPorts: [ + httpPort, + httpsPort, + validHttpsPort, + caHttpsPort, + keepAlivePort, + ], + }); + await kernel.mount(createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] })); + + // Seed the Debian-shaped trust store the way the native VM bootstrap does, + // so curl's compile-time default CA bundle resolves in-guest. Only the + // "trusted" CA is placed here; the cacert-only CA is intentionally absent. + if (seededCaPem) { + await filesystem.mkdir('/etc/ssl/certs', { recursive: true }); + await kernel.writeFile('/etc/ssl/certs/ca-certificates.crt', seededCaPem); + } + return kernel; + } + + async function execWithRetry(command: string) { + let lastResult: Awaited> | undefined; + for (let attempt = 1; attempt <= EXTERNAL_RETRY_ATTEMPTS; attempt += 1) { + lastResult = await kernel.exec(command); + if (lastResult.exitCode === 0) return lastResult; + if (attempt < EXTERNAL_RETRY_ATTEMPTS) { + await sleep(EXTERNAL_RETRY_DELAY_MS); + } + } + + return lastResult!; + } + + afterEach(async () => { + await kernel?.dispose(); + }); + + itIf(hasHttpGetTest, 'http_get_test reaches a local HTTP server', async () => { + await createKernelWithNet(); + const result = await kernel.exec(`http_get_test 127.0.0.1 ${httpPort} /json`); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain('HTTP/1.1 200'); + expect(result.stdout).toContain('"ok":true'); + }, 15000); + + itIf(hasHttpGetTest, 'http_get_test preserves non-blocking connect diagnostics', async () => { + await createKernelWithNet(); + const result = await kernel.exec(`http_get_test 127.0.0.1 ${httpPort} /json`); + expect(result.exitCode).toBe(0); + expect(result.stderr).toContain('fcntl F_SETFL(NONBLOCK)=0'); + expect(result.stderr).toMatch(/connect=(0|-1 errno=\d+)/); + expect(result.stderr).toContain('getsockopt(SO_ERROR)=0 value=0'); + expect(result.stderr).toContain('poll(POLLOUT)=1'); + }, 15000); + + itIf(hasCurl, 'curl GET returns JSON from a local server', async () => { + await createKernelWithNet(); + const result = await kernel.exec(`curl -s http://127.0.0.1:${httpPort}/json`); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain('"ok":true'); + }, 15000); + + itIf(hasCurl, 'curl --version reports the upstream tool version', async () => { + await createKernelWithNet(); + const result = await kernel.exec('curl --version'); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain('curl 8.11.1'); + expect(result.stdout).toMatch(/Protocols:/); + }, 15000); + + itIf(hasCurl, 'curl -L follows redirects', async () => { + await createKernelWithNet(); + const result = await kernel.exec(`curl -s -L http://127.0.0.1:${httpPort}/redirect`); + expect(result.exitCode).toBe(0); + expect(result.stdout).toBe('followed redirect'); + }, 15000); + + itIf(hasCurl, 'curl POST sends body and headers', async () => { + await createKernelWithNet(); + const result = await kernel.exec( + `curl -s -X POST -H 'X-Test: edge-case' -d 'payload-data' http://127.0.0.1:${httpPort}/echo`, + ); + expect(result.exitCode).toBe(0); + const body = JSON.parse(result.stdout); + expect(body.method).toBe('POST'); + expect(body.body).toBe('payload-data'); + expect(body.header).toBe('edge-case'); + }, 15000); + + itIf(hasCurl, 'curl --json sends JSON with the expected headers', async () => { + await createKernelWithNet(); + const result = await kernel.exec( + `curl -s --json '{\"hello\":\"world\"}' http://127.0.0.1:${httpPort}/json-post`, + ); + expect(result.exitCode).toBe(0); + const body = JSON.parse(result.stdout); + expect(body.method).toBe('POST'); + expect(body.body).toBe('{"hello":"world"}'); + expect(body.contentType).toBe('application/json'); + expect(body.accept).toBe('application/json'); + }, 15000); + + itIf(hasCurl, 'curl -I returns response headers without the body', async () => { + await createKernelWithNet(); + const result = await kernel.exec(`curl -s -I http://127.0.0.1:${httpPort}/head-test`); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain('HTTP/'); + expect(result.stdout).toMatch(/X-Test-Header/i); + expect(result.stdout).not.toContain('body should not appear'); + }, 15000); + + itIf(hasCurl, 'curl -u sends HTTP Basic authentication', async () => { + await createKernelWithNet(); + const result = await kernel.exec(`curl -s -u user:pass http://127.0.0.1:${httpPort}/auth-required`); + expect(result.exitCode).toBe(0); + expect(result.stdout).toBe('authenticated: user:pass'); + }, 15000); + + itIf(hasCurl, 'curl -F uploads multipart form data', async () => { + await createKernelWithNet(); + await kernel.writeFile('/tmp/upload.txt', 'file payload\n'); + const result = await kernel.exec(`curl -s -F file=@/tmp/upload.txt http://127.0.0.1:${httpPort}/upload`); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain('multipart: true'); + expect(result.stdout).toContain('body-contains-file: true'); + }, 15000); + + itIf(hasCurl, 'curl -K reads options from a config file', async () => { + await createKernelWithNet(); + await kernel.writeFile( + '/tmp/curlrc', + `silent\nurl = "http://127.0.0.1:${httpPort}/json"\n`, + ); + const result = await kernel.exec('curl -K /tmp/curlrc'); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain('"ok":true'); + }, 15000); + + itIf(hasCurl, 'curl -o writes text output to a file', async () => { + await createKernelWithNet(); + const result = await kernel.exec(`curl -s -o /tmp/out.json http://127.0.0.1:${httpPort}/json`); + expect(result.exitCode).toBe(0); + expect(result.stdout).toBe(''); + const file = new TextDecoder().decode(await kernel.readFile('/tmp/out.json')); + expect(file).toContain('"ok":true'); + }, 15000); + + itIf(hasCurl, 'curl -o respects the current working directory for relative output paths', async () => { + await createKernelWithNet(); + const result = await kernel.exec( + `mkdir -p /tmp/curl-cwd && cd /tmp/curl-cwd && ` + + `curl -s -o local.txt http://127.0.0.1:${httpPort}/named.txt && cat /tmp/curl-cwd/local.txt`, + ); + expect(result.exitCode).toBe(0); + expect(result.stdout).toBe('downloaded-by-remote-name\n'); + }, 15000); + + itIf(hasCurl, 'curl -o writes binary output without truncation', async () => { + await createKernelWithNet(); + const result = await kernel.exec(`curl -s -o /tmp/out.bin http://127.0.0.1:${httpPort}/binary`); + expect(result.exitCode).toBe(0); + const file = await kernel.readFile('/tmp/out.bin'); + expect(file).toHaveLength(256); + expect(Array.from(file.slice(0, 8))).toEqual([0, 1, 2, 3, 4, 5, 6, 7]); + expect(Array.from(file.slice(-4))).toEqual([252, 253, 254, 255]); + }, 15000); + + itIf(hasCurl, 'curl -D and -o split headers and body into separate files', async () => { + await createKernelWithNet(); + const result = await kernel.exec( + `curl -s -D /tmp/headers.txt -o /tmp/body.txt http://127.0.0.1:${httpPort}/named.txt`, + ); + expect(result.exitCode).toBe(0); + expect(result.stdout).toBe(''); + + const headers = new TextDecoder().decode(await kernel.readFile('/tmp/headers.txt')); + const body = new TextDecoder().decode(await kernel.readFile('/tmp/body.txt')); + expect(headers).toContain('HTTP/1.1 200 OK'); + expect(headers).toMatch(/Content-Type: text\/plain/i); + expect(body).toBe('downloaded-by-remote-name\n'); + }, 15000); + + itIf(hasCurl, 'curl -O writes to the remote filename', async () => { + await createKernelWithNet(); + const result = await kernel.exec( + `mkdir -p /tmp/remote-name && cd /tmp/remote-name && ` + + `curl -s -O http://127.0.0.1:${httpPort}/named.txt && cat /tmp/remote-name/named.txt`, + ); + expect(result.exitCode).toBe(0); + expect(result.stdout).toBe('downloaded-by-remote-name\n'); + }, 15000); + + itIf(hasCurl, 'curl -w writes the HTTP status code', async () => { + await createKernelWithNet(); + const result = await kernel.exec(`curl -s -w '%{http_code}' http://127.0.0.1:${httpPort}/status`); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain('created'); + expect(result.stdout).toContain('201'); + }, 15000); + + itIf(hasCurl, 'curl -f reports HTTP errors with a non-zero exit code', async () => { + await createKernelWithNet(); + const result = await kernel.exec(`curl -fsS http://127.0.0.1:${httpPort}/missing`); + expect(result.exitCode).not.toBe(0); + expect(result.stderr).toMatch(/404|not found|error/i); + }, 15000); + + itIf(hasCurl, 'curl --fail-with-body preserves the response body on HTTP errors', async () => { + await createKernelWithNet(); + const result = await kernel.exec(`curl -sS --fail-with-body http://127.0.0.1:${httpPort}/missing`); + expect(result.exitCode).not.toBe(0); + expect(result.stdout).toBe('not found'); + expect(result.stderr).toMatch(/404|error/i); + }, 15000); + + itIf(hasCurl, 'curl reports refused connections without hanging', async () => { + await createKernelWithNet(); + + const probe = createTcpServer(); + await new Promise((resolveListen) => probe.listen(0, '127.0.0.1', resolveListen)); + const unusedPort = (probe.address() as import('node:net').AddressInfo).port; + await new Promise((resolveClose) => probe.close(() => resolveClose())); + + const startedAt = Date.now(); + const result = await kernel.exec(`curl -sS http://127.0.0.1:${unusedPort}/`); + expect(result.exitCode).not.toBe(0); + expect(Date.now() - startedAt).toBeLessThan(8000); + expect(result.stderr).toMatch(/connect|refused|failed/i); + }, 15000); + + itIf(hasCurl, 'curl reports DNS failures cleanly', async () => { + await createKernelWithNet(); + const result = await kernel.exec('curl -sS http://does-not-exist.invalid/'); + expect(result.exitCode).not.toBe(0); + expect(result.stderr).toMatch(/resolve|host|dns/i); + }, 15000); + + itIf(hasCurl, 'curl handles multiple URLs in one invocation', async () => { + await createKernelWithNet(); + const result = await kernel.exec( + `curl -s http://127.0.0.1:${httpPort}/one http://127.0.0.1:${httpPort}/two`, + ); + expect(result.exitCode).toBe(0); + expect(result.stdout).toBe('first-response\nsecond-response\n'); + }, 15000); + + itIf(hasCurl, 'curl --retry retries transient HTTP failures', async () => { + await createKernelWithNet(); + const result = await kernel.exec( + `curl -fsS --retry 2 --retry-delay 0 http://127.0.0.1:${httpPort}/flaky`, + ); + expect(result.exitCode).toBe(0); + expect(result.stdout).toBe('retry succeeded'); + expect(flakyRequestCount).toBeGreaterThanOrEqual(2); + }, 15000); + + itIf(hasCurl, 'curl exits promptly after a keep-alive response', async () => { + await createKernelWithNet(); + const startedAt = Date.now(); + const result = await kernel.exec(`curl -s http://127.0.0.1:${keepAlivePort}/`); + expect(result.exitCode).toBe(0); + expect(result.stdout).toBe('hello from keepalive'); + expect(Date.now() - startedAt).toBeLessThan(8000); + }, 15000); + + itIf(hasCurl, 'curl --version reports the mbedTLS backend', async () => { + await createKernelWithNet(); + const result = await kernel.exec('curl --version'); + expect(result.exitCode).toBe(0); + // Real in-guest TLS: the SSL version line must name mbedTLS, and the + // retired host shim string must be gone. + expect(result.stdout).toMatch(/mbedTLS/i); + expect(result.stdout).not.toMatch(/WASI-TLS|wasi-tls/i); + expect(result.stdout).toMatch(/^Features:.*\bSSL\b/m); + expect(result.stdout).toMatch(/^Features:.*\bbrotli\b/m); + expect(result.stdout).toMatch(/^Features:.*\bzstd\b/m); + }, 15000); + + itIf(hasCurl && hasOpenssl, 'curl verifies a CA-signed cert against the seeded CA bundle', async () => { + await createKernelWithNet(); + // No -k, no --cacert: trust comes solely from the seeded + // /etc/ssl/certs/ca-certificates.crt, exactly like Debian curl. + const result = await kernel.exec(`curl -sS https://127.0.0.1:${validHttpsPort}/json`); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain('"verified":true'); + }, 15000); + + itIf(hasCurl && hasOpenssl, 'curl fails with exit 60 and a real verify message on an untrusted cert', async () => { + await createKernelWithNet(); + const result = await kernel.exec(`curl -sS https://127.0.0.1:${httpsPort}/json`); + // CURLE_PEER_FAILED_VERIFICATION == 60, the native Linux taxonomy. NOT 35 + // (SSL connect error, the old host-shim behavior). + expect(result.exitCode).toBe(60); + expect(result.stderr).toMatch(/certificate|verify|self[- ]?signed|CA/i); + expect(result.stderr).not.toMatch(/WASI TLS|wasi-tls/i); + expect(result.stdout).toBe(''); + }, 15000); + + itIf(hasCurl && hasOpenssl, 'curl -k skips verification and succeeds on an untrusted cert', async () => { + await createKernelWithNet(); + const result = await kernel.exec(`curl -ks https://127.0.0.1:${httpsPort}/json`); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain('"secure":true'); + }, 15000); + + itIf(hasCurl && hasOpenssl, 'curl --cacert accepts a server signed by that CA', async () => { + await createKernelWithNet(); + // caHttpsServer's CA is NOT in the seeded bundle, so this only passes if + // --cacert is honored (real file read + chain build in-guest). + await kernel.writeFile('/tmp/cacert-only.pem', caOnlyPem); + const result = await kernel.exec( + `curl -sS --cacert /tmp/cacert-only.pem https://127.0.0.1:${caHttpsPort}/json`, + ); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain('"cacert":true'); + }, 15000); + + itIf(hasCurl && hasOpenssl, 'curl --cacert with the wrong CA still fails verification (exit 60)', async () => { + await createKernelWithNet(); + // Point --cacert at the seeded CA, which did NOT sign caHttpsServer's leaf. + await kernel.writeFile('/tmp/wrong-ca.pem', seededCaPem); + const result = await kernel.exec( + `curl -sS --cacert /tmp/wrong-ca.pem https://127.0.0.1:${caHttpsPort}/json`, + ); + expect(result.exitCode).toBe(60); + expect(result.stderr).toMatch(/certificate|verify|CA/i); + }, 15000); + + itIf(hasCurl && hasOpenssl, 'curl -k exits promptly after an HTTPS keep-alive response', async () => { + await createKernelWithNet(); + const startedAt = Date.now(); + const result = await kernel.exec(`curl -ks https://127.0.0.1:${httpsPort}/keepalive`); + expect(result.exitCode).toBe(0); + expect(result.stdout).toBe('hello from tls keepalive'); + expect(Date.now() - startedAt).toBeLessThan(8000); + }, 15000); + + itIf(hasCurl, 'curl --compressed round-trips a gzip response body', async () => { + await createKernelWithNet(); + const result = await kernel.exec(`curl -s --compressed http://127.0.0.1:${httpPort}/gzip`); + expect(result.exitCode).toBe(0); + expect(result.stdout).toBe(COMPRESSION_PAYLOAD); + }, 15000); + + itIf(hasCurl, 'curl --compressed round-trips a brotli response body', async () => { + await createKernelWithNet(); + const result = await kernel.exec(`curl -s --compressed http://127.0.0.1:${httpPort}/brotli`); + expect(result.exitCode).toBe(0); + expect(result.stdout).toBe(COMPRESSION_PAYLOAD); + }, 15000); + + itIf(hasCurl, 'curl --compressed round-trips a zstd response body', async () => { + await createKernelWithNet(); + const result = await kernel.exec(`curl -s --compressed http://127.0.0.1:${httpPort}/zstd`); + expect(result.exitCode).toBe(0); + expect(result.stdout).toBe(COMPRESSION_PAYLOAD); + }, 15000); + + itIf(hasHttpGetTest && !externalNetworkSkipReason, 'http_get_test reaches an external host over real TCP', async () => { + await createKernelWithNet(); + const result = await execWithRetry(`http_get_test ${EXTERNAL_HOST} ${EXTERNAL_TCP_PORT} /`); + expect(result.exitCode).toBe(0); + expect(result.stdout).toMatch(/HTTP\/1\.[01] (200|301|302)/); + }, 30000); + + itIf(hasCurl && !externalNetworkSkipReason, 'curl reaches a real external HTTP endpoint', async () => { + await createKernelWithNet(); + const result = await execWithRetry( + `curl -fsSL --retry 2 --retry-delay 1 --retry-all-errors --connect-timeout 10 --max-time 30 ${EXTERNAL_HTTP_URL}`, + ); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain(EXTERNAL_EXPECTED_BODY); + }, 30000); + + itIf(hasCurl && !externalNetworkSkipReason, 'curl reaches a real external HTTPS endpoint', async () => { + await createKernelWithNet(); + const result = await execWithRetry( + `curl -fsSL --retry 2 --retry-delay 1 --retry-all-errors --connect-timeout 10 --max-time 30 ${EXTERNAL_HTTPS_URL}`, + ); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain(EXTERNAL_EXPECTED_BODY); + }, 30000); +}); diff --git a/software/curl/tsconfig.json b/software/curl/tsconfig.json new file mode 100644 index 0000000000..03ce790ab7 --- /dev/null +++ b/software/curl/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*"] +} diff --git a/software/diffutils/agentos-package.json b/software/diffutils/agentos-package.json new file mode 100644 index 0000000000..c9460d579b --- /dev/null +++ b/software/diffutils/agentos-package.json @@ -0,0 +1,12 @@ +{ + "commands": [ + "diff" + ], + "registry": { + "title": "diffutils", + "description": "GNU diff for comparing files.", + "priority": 6, + "image": "/images/registry/diffutils.svg", + "category": "core" + } +} diff --git a/software/diffutils/native/crates/cmd-diff/Cargo.toml b/software/diffutils/native/crates/cmd-diff/Cargo.toml new file mode 100644 index 0000000000..8cdc14cb7d --- /dev/null +++ b/software/diffutils/native/crates/cmd-diff/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-diff" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "diff standalone binary for secure-exec VM" + +[[bin]] +name = "diff" +path = "src/main.rs" + +[dependencies] +secureexec-diff = { path = "../diff" } diff --git a/registry/native/crates/commands/diff/src/main.rs b/software/diffutils/native/crates/cmd-diff/src/main.rs similarity index 100% rename from registry/native/crates/commands/diff/src/main.rs rename to software/diffutils/native/crates/cmd-diff/src/main.rs diff --git a/software/diffutils/native/crates/diff/Cargo.toml b/software/diffutils/native/crates/diff/Cargo.toml new file mode 100644 index 0000000000..587bcf5968 --- /dev/null +++ b/software/diffutils/native/crates/diff/Cargo.toml @@ -0,0 +1,10 @@ +[package] +workspace = "../../../../../toolchain" +name = "secureexec-diff" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "diff implementation for secure-exec standalone binaries" + +[dependencies] +similar = "2" diff --git a/software/diffutils/native/crates/diff/src/lib.rs b/software/diffutils/native/crates/diff/src/lib.rs new file mode 100644 index 0000000000..77e385a00e --- /dev/null +++ b/software/diffutils/native/crates/diff/src/lib.rs @@ -0,0 +1,476 @@ +//! diff -- compare files line by line using the `similar` crate + +use std::collections::HashSet; +use std::ffi::OsString; +use std::fs; +use std::io::{self, Write}; +use std::path::Path; + +use similar::{ChangeTag, TextDiff}; + +struct Options { + unified: bool, + context_fmt: bool, + context_lines: usize, + recursive: bool, + brief: bool, + ignore_case: bool, + ignore_whitespace: bool, + ignore_blank_lines: bool, +} + +impl Default for Options { + fn default() -> Self { + Self { + unified: false, + context_fmt: false, + context_lines: 3, + recursive: false, + brief: false, + ignore_case: false, + ignore_whitespace: false, + ignore_blank_lines: false, + } + } +} + +pub fn main(args: Vec) -> i32 { + let args: Vec = args + .iter() + .map(|a| a.to_string_lossy().to_string()) + .collect(); + let mut opts = Options::default(); + let mut files: Vec = Vec::new(); + let mut i = 1; + + while i < args.len() { + let arg = &args[i]; + match arg.as_str() { + "-u" | "--unified" => opts.unified = true, + "-c" | "--context" => opts.context_fmt = true, + "-r" | "-R" | "--recursive" => opts.recursive = true, + "-q" | "--brief" => opts.brief = true, + "-i" | "--ignore-case" => opts.ignore_case = true, + "-w" | "--ignore-all-space" => opts.ignore_whitespace = true, + "-B" | "--ignore-blank-lines" => opts.ignore_blank_lines = true, + s if s.starts_with("-U") => { + if let Ok(n) = s[2..].parse::() { + opts.unified = true; + opts.context_lines = n; + } + } + s if s.starts_with("-C") => { + if let Ok(n) = s[2..].parse::() { + opts.context_fmt = true; + opts.context_lines = n; + } + } + _ if !arg.starts_with('-') || arg == "-" => { + files.push(arg.clone()); + } + _ => { + eprintln!("diff: unknown option: {}", arg); + return 2; + } + } + i += 1; + } + + if files.len() != 2 { + eprintln!("diff: requires exactly two file or directory arguments"); + return 2; + } + + let path_a = Path::new(&files[0]); + let path_b = Path::new(&files[1]); + + let mut visited_dirs = HashSet::new(); + match diff_paths(path_a, path_b, &opts, &mut visited_dirs) { + Ok(has_diff) => { + if has_diff { + 1 + } else { + 0 + } + } + Err(e) => { + eprintln!("diff: {}", e); + 2 + } + } +} + +fn diff_paths( + path_a: &Path, + path_b: &Path, + opts: &Options, + visited_dirs: &mut HashSet<(std::path::PathBuf, std::path::PathBuf)>, +) -> Result { + let a_is_dir = path_a.is_dir(); + let b_is_dir = path_b.is_dir(); + + if a_is_dir && b_is_dir { + if opts.recursive { + diff_dirs(path_a, path_b, opts, visited_dirs) + } else { + Err(format!("{} is a directory", path_a.display())) + } + } else if a_is_dir || b_is_dir { + if a_is_dir { + let name = path_b.file_name().ok_or("invalid filename")?; + diff_files(&path_a.join(name), path_b, opts) + } else { + let name = path_a.file_name().ok_or("invalid filename")?; + diff_files(path_a, &path_b.join(name), opts) + } + } else { + diff_files(path_a, path_b, opts) + } +} + +fn diff_dirs( + dir_a: &Path, + dir_b: &Path, + opts: &Options, + visited_dirs: &mut HashSet<(std::path::PathBuf, std::path::PathBuf)>, +) -> Result { + let key = ( + fs::canonicalize(dir_a).map_err(|e| format!("{}: {}", dir_a.display(), e))?, + fs::canonicalize(dir_b).map_err(|e| format!("{}: {}", dir_b.display(), e))?, + ); + if !visited_dirs.insert(key) { + return Ok(false); + } + + let mut entries_a = list_dir(dir_a)?; + let mut entries_b = list_dir(dir_b)?; + entries_a.sort(); + entries_b.sort(); + + let mut all_names: Vec = Vec::new(); + for name in &entries_a { + all_names.push(name.clone()); + } + for name in &entries_b { + if !entries_a.contains(name) { + all_names.push(name.clone()); + } + } + all_names.sort(); + + let mut has_diff = false; + + for name in &all_names { + let pa = dir_a.join(name); + let pb = dir_b.join(name); + let a_exists = entries_a.contains(name); + let b_exists = entries_b.contains(name); + + if a_exists && !b_exists { + print_stdout_line(format_args!("Only in {}: {}", dir_a.display(), name))?; + has_diff = true; + } else if !a_exists && b_exists { + print_stdout_line(format_args!("Only in {}: {}", dir_b.display(), name))?; + has_diff = true; + } else { + match diff_paths(&pa, &pb, opts, visited_dirs) { + Ok(d) => { + if d { + has_diff = true; + } + } + Err(e) => { + return Err(e); + } + } + } + } + + Ok(has_diff) +} + +fn list_dir(dir: &Path) -> Result, String> { + let entries = fs::read_dir(dir).map_err(|e| format!("{}: {}", dir.display(), e))?; + let mut names = Vec::new(); + for entry in entries { + let entry = entry.map_err(|e| format!("{}: {}", dir.display(), e))?; + if let Some(name) = entry.file_name().to_str() { + names.push(name.to_string()); + } + } + Ok(names) +} + +fn preprocess(text: &str, opts: &Options) -> String { + let mut s = text.to_string(); + if opts.ignore_blank_lines { + let trailing = s.ends_with('\n'); + s = s + .lines() + .filter(|line| !line.trim().is_empty()) + .collect::>() + .join("\n"); + if trailing { + s.push('\n'); + } + } + if opts.ignore_case { + s = s.to_lowercase(); + } + if opts.ignore_whitespace { + let trailing = s.ends_with('\n'); + s = s + .lines() + .map(|line| { + let mut result = String::new(); + let mut in_ws = false; + for ch in line.chars() { + if ch.is_whitespace() { + if !in_ws { + result.push(' '); + in_ws = true; + } + } else { + result.push(ch); + in_ws = false; + } + } + let trimmed = result.trim(); + trimmed.to_string() + }) + .collect::>() + .join("\n"); + if trailing { + s.push('\n'); + } + } + s +} + +fn diff_files(path_a: &Path, path_b: &Path, opts: &Options) -> Result { + let text_a = read_file(path_a)?; + let text_b = read_file(path_b)?; + + // Check for differences (possibly with preprocessing) + let needs_pp = opts.ignore_case || opts.ignore_whitespace || opts.ignore_blank_lines; + let has_changes = if needs_pp { + preprocess(&text_a, opts) != preprocess(&text_b, opts) + } else { + text_a != text_b + }; + + if !has_changes { + return Ok(false); + } + + if opts.brief { + print_stdout_line(format_args!( + "Files {} and {} differ", + path_a.display(), + path_b.display() + ))?; + return Ok(true); + } + + if opts.recursive { + print_stdout_line(format_args!( + "diff -r {} {}", + path_a.display(), + path_b.display() + ))?; + } + + // Diff the original text for output (all output logic inline to avoid lifetime issues) + let diff = TextDiff::from_lines(&text_a, &text_b); + let label_a = format!("{}", path_a.display()); + let label_b = format!("{}", path_b.display()); + let stdout = io::stdout(); + let mut out = stdout.lock(); + + if opts.unified { + writeln!(out, "--- {}", label_a).map_err(|e| format!("failed to write output: {e}"))?; + writeln!(out, "+++ {}", label_b).map_err(|e| format!("failed to write output: {e}"))?; + for hunk in diff + .unified_diff() + .context_radius(opts.context_lines) + .iter_hunks() + { + write!(out, "{}", hunk).map_err(|e| format!("failed to write output: {e}"))?; + } + } else if opts.context_fmt { + writeln!(out, "*** {}", label_a).map_err(|e| format!("failed to write output: {e}"))?; + writeln!(out, "--- {}", label_b).map_err(|e| format!("failed to write output: {e}"))?; + for hunk in diff + .unified_diff() + .context_radius(opts.context_lines) + .iter_hunks() + { + let mut old_lines: Vec<(ChangeTag, String)> = Vec::new(); + let mut new_lines: Vec<(ChangeTag, String)> = Vec::new(); + let mut old_start = 0usize; + let mut new_start = 0usize; + let mut first = true; + for change in hunk.iter_changes() { + if first { + old_start = change.old_index().unwrap_or(0) + 1; + new_start = change.new_index().unwrap_or(0) + 1; + first = false; + } + match change.tag() { + ChangeTag::Equal => { + old_lines.push((ChangeTag::Equal, change.value().to_string())); + new_lines.push((ChangeTag::Equal, change.value().to_string())); + } + ChangeTag::Delete => { + old_lines.push((ChangeTag::Delete, change.value().to_string())); + } + ChangeTag::Insert => { + new_lines.push((ChangeTag::Insert, change.value().to_string())); + } + } + } + let old_end = old_start + old_lines.len().saturating_sub(1); + let new_end = new_start + new_lines.len().saturating_sub(1); + writeln!(out, "***************").map_err(|e| format!("failed to write output: {e}"))?; + writeln!(out, "*** {},{} ****", old_start, old_end) + .map_err(|e| format!("failed to write output: {e}"))?; + for (tag, line) in &old_lines { + let prefix = match tag { + ChangeTag::Delete => "- ", + ChangeTag::Equal => " ", + _ => continue, + }; + write!(out, "{}{}", prefix, line) + .map_err(|e| format!("failed to write output: {e}"))?; + if !line.ends_with('\n') { + writeln!(out).map_err(|e| format!("failed to write output: {e}"))?; + } + } + writeln!(out, "--- {},{} ----", new_start, new_end) + .map_err(|e| format!("failed to write output: {e}"))?; + for (tag, line) in &new_lines { + let prefix = match tag { + ChangeTag::Insert => "+ ", + ChangeTag::Equal => " ", + _ => continue, + }; + write!(out, "{}{}", prefix, line) + .map_err(|e| format!("failed to write output: {e}"))?; + if !line.ends_with('\n') { + writeln!(out).map_err(|e| format!("failed to write output: {e}"))?; + } + } + } + } else { + // Normal diff format + let old_text: String = diff.old_slices().concat(); + let new_text: String = diff.new_slices().concat(); + let old_lines: Vec<&str> = old_text.lines().collect(); + let new_lines: Vec<&str> = new_text.lines().collect(); + + for op in diff.ops() { + match op { + similar::DiffOp::Equal { .. } => {} + similar::DiffOp::Delete { + old_index, + old_len, + new_index, + } => { + writeln!( + out, + "{}d{}", + format_range(*old_index + 1, *old_len), + new_index + ) + .map_err(|e| format!("failed to write output: {e}"))?; + for i in *old_index..*old_index + old_len { + if i < old_lines.len() { + writeln!(out, "< {}", old_lines[i]) + .map_err(|e| format!("failed to write output: {e}"))?; + } + } + } + similar::DiffOp::Insert { + old_index, + new_index, + new_len, + } => { + writeln!( + out, + "{}a{}", + old_index, + format_range(*new_index + 1, *new_len) + ) + .map_err(|e| format!("failed to write output: {e}"))?; + for i in *new_index..*new_index + new_len { + if i < new_lines.len() { + writeln!(out, "> {}", new_lines[i]) + .map_err(|e| format!("failed to write output: {e}"))?; + } + } + } + similar::DiffOp::Replace { + old_index, + old_len, + new_index, + new_len, + } => { + writeln!( + out, + "{}c{}", + format_range(*old_index + 1, *old_len), + format_range(*new_index + 1, *new_len) + ) + .map_err(|e| format!("failed to write output: {e}"))?; + for i in *old_index..*old_index + old_len { + if i < old_lines.len() { + writeln!(out, "< {}", old_lines[i]) + .map_err(|e| format!("failed to write output: {e}"))?; + } + } + writeln!(out, "---").map_err(|e| format!("failed to write output: {e}"))?; + for i in *new_index..*new_index + new_len { + if i < new_lines.len() { + writeln!(out, "> {}", new_lines[i]) + .map_err(|e| format!("failed to write output: {e}"))?; + } + } + } + } + } + } + + out.flush() + .map_err(|e| format!("failed to write output: {e}"))?; + + Ok(true) +} + +fn print_stdout_line(args: std::fmt::Arguments<'_>) -> Result<(), String> { + let stdout = io::stdout(); + let mut out = stdout.lock(); + writeln!(out, "{args}").map_err(|e| format!("failed to write output: {e}"))?; + out.flush() + .map_err(|e| format!("failed to write output: {e}")) +} + +fn read_file(path: &Path) -> Result { + if path.to_str() == Some("-") { + use std::io::Read; + let mut buf = String::new(); + io::stdin() + .read_to_string(&mut buf) + .map_err(|e| format!("stdin: {}", e))?; + Ok(buf) + } else { + fs::read_to_string(path).map_err(|e| format!("{}: {}", path.display(), e)) + } +} + +fn format_range(start: usize, len: usize) -> String { + if len == 1 { + format!("{}", start) + } else { + format!("{},{}", start, start + len - 1) + } +} diff --git a/software/diffutils/package.json b/software/diffutils/package.json new file mode 100644 index 0000000000..7943248120 --- /dev/null +++ b/software/diffutils/package.json @@ -0,0 +1,33 @@ +{ + "name": "@agentos-software/diffutils", + "version": "0.3.3", + "type": "module", + "license": "Apache-2.0", + "description": "GNU diffutils for secure-exec VMs (diff)", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "files": [ + "dist", + "!dist/package", + "!dist/package.tar" + ], + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "scripts": { + "build": "agentos-toolchain stage --commands-dir ../../toolchain/target/wasm32-wasip1/release/commands --if-missing skip && tsc && agentos-toolchain build", + "check-types": "tsc --noEmit", + "test": "vitest run test/ --passWithNoTests" + }, + "devDependencies": { + "@agentos-software/manifest": "workspace:*", + "@rivet-dev/agentos-toolchain": "workspace:*", + "@types/node": "^22.10.2", + "typescript": "^5.9.2", + "@agentos/test-harness": "workspace:*", + "vitest": "^2.1.9" + } +} diff --git a/registry/software/codex-cli/src/index.ts b/software/diffutils/src/index.ts similarity index 100% rename from registry/software/codex-cli/src/index.ts rename to software/diffutils/src/index.ts diff --git a/software/diffutils/test/diff.test.ts b/software/diffutils/test/diff.test.ts new file mode 100644 index 0000000000..e3f175b502 --- /dev/null +++ b/software/diffutils/test/diff.test.ts @@ -0,0 +1,145 @@ +import { existsSync } from "node:fs"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { + NodeFileSystem, + createKernel, + createWasmVmRuntime, + describeIf, +} from "@agentos/test-harness"; +import type { Kernel } from "@agentos/test-harness"; +import { afterEach, expect, it } from "vitest"; + +const DIFF_COMMAND_DIR = fileURLToPath(new URL("../bin", import.meta.url)); +const hasDiffPackageBinary = existsSync(join(DIFF_COMMAND_DIR, "diff")); + +let tempRoot: string | undefined; + +async function writeFixture(path: string, contents: string): Promise { + if (!tempRoot) throw new Error("fixture root not initialized"); + const hostPath = join(tempRoot, path.replace(/^\/+/, "")); + await mkdir(dirname(hostPath), { recursive: true }); + await writeFile(hostPath, contents); +} + +async function createTestVFS(): Promise { + tempRoot = await mkdtemp(join(tmpdir(), "agentos-diff-")); + await writeFixture("/project/a.txt", "alpha\nbeta\ngamma\n"); + await writeFixture("/project/b.txt", "alpha\ndelta\ngamma\n"); + await writeFixture("/project/same.txt", "alpha\nbeta\ngamma\n"); + await writeFixture("/project/ws-a.txt", "Alpha\n\nbeta value\n"); + await writeFixture("/project/ws-b.txt", "alpha\n BETA VALUE\n"); + await writeFixture("/project/left/common.txt", "one\ntwo\nthree\n"); + await writeFixture("/project/right/common.txt", "one\n2\nthree\n"); + await writeFixture("/project/right/extra.txt", "only on right\n"); + return new NodeFileSystem({ root: tempRoot }); +} + +describeIf(hasDiffPackageBinary, "diff command", { timeout: 10_000 }, () => { + let kernel: Kernel | undefined; + + afterEach(async () => { + await kernel?.dispose(); + kernel = undefined; + if (tempRoot) { + await rm(tempRoot, { recursive: true, force: true }); + tempRoot = undefined; + } + }); + + async function mountFixture(): Promise { + const vfs = await createTestVFS(); + kernel = createKernel({ filesystem: vfs }); + await kernel.mount(createWasmVmRuntime({ commandDirs: [DIFF_COMMAND_DIR] })); + } + + async function runDiff(args: string[]) { + if (!kernel) throw new Error("kernel not mounted"); + let stdout = ""; + let stderr = ""; + const proc = kernel.spawn("diff", args, { + onStdout: (chunk) => { + stdout += Buffer.from(chunk).toString("utf8"); + }, + onStderr: (chunk) => { + stderr += Buffer.from(chunk).toString("utf8"); + }, + }); + const exitCode = await proc.wait(); + await new Promise((resolve) => setTimeout(resolve, 0)); + return { stdout, stderr, exitCode }; + } + + it("returns zero for identical files", async () => { + await mountFixture(); + + const result = await runDiff(["/project/a.txt", "/project/same.txt"]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(result.stdout).toBe(""); + }); + + it("prints normal diffs for changed files", async () => { + await mountFixture(); + + const result = await runDiff(["/project/a.txt", "/project/b.txt"]); + expect(result.exitCode).toBe(1); + expect(result.stdout).toContain("2c2"); + expect(result.stdout).toContain("< beta"); + expect(result.stdout).toContain("> delta"); + }); + + it("prints unified diffs", async () => { + await mountFixture(); + + const result = await runDiff(["-u", "/project/a.txt", "/project/b.txt"]); + expect(result.exitCode).toBe(1); + expect(result.stdout).toContain("--- /project/a.txt"); + expect(result.stdout).toContain("+++ /project/b.txt"); + expect(result.stdout).toContain("-beta"); + expect(result.stdout).toContain("+delta"); + }); + + it("prints brief output with -q", async () => { + await mountFixture(); + + const result = await runDiff(["-q", "/project/a.txt", "/project/b.txt"]); + expect(result.exitCode).toBe(1); + expect(result.stdout.trim()).toBe( + "Files /project/a.txt and /project/b.txt differ", + ); + }); + + it("honors ignore-case, whitespace, and blank-line flags", async () => { + await mountFixture(); + + const result = await runDiff([ + "-i", + "-w", + "-B", + "/project/ws-a.txt", + "/project/ws-b.txt", + ]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(result.stdout).toBe(""); + }); + + it("compares directories recursively", async () => { + await mountFixture(); + + const result = await runDiff(["-r", "/project/left", "/project/right"]); + expect(result.exitCode).toBe(1); + expect(result.stdout).toContain("Only in /project/right: extra.txt"); + expect(result.stdout).toContain("/project/left/common.txt"); + expect(result.stdout).toContain("/project/right/common.txt"); + }); + + it("returns an error for missing inputs", async () => { + await mountFixture(); + + const result = await runDiff(["/project/a.txt", "/project/missing.txt"]); + expect(result.exitCode).toBe(2); + expect(result.stderr).toContain("/project/missing.txt"); + }); +}); diff --git a/software/diffutils/test/manifest.test.ts b/software/diffutils/test/manifest.test.ts new file mode 100644 index 0000000000..89139d33b8 --- /dev/null +++ b/software/diffutils/test/manifest.test.ts @@ -0,0 +1,19 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +const packageDir = new URL("..", import.meta.url).pathname; + +describe("package manifest", () => { + it("declares command binaries", () => { + const manifest = JSON.parse( + readFileSync(join(packageDir, "agentos-package.json"), "utf8"), + ); + + expect(manifest.commands?.length ?? 0).toBeGreaterThan(0); + for (const command of manifest.commands) { + expect(typeof command).toBe("string"); + expect(command.length).toBeGreaterThan(0); + } + }); +}); diff --git a/software/diffutils/tsconfig.json b/software/diffutils/tsconfig.json new file mode 100644 index 0000000000..03ce790ab7 --- /dev/null +++ b/software/diffutils/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*"] +} diff --git a/software/duckdb/agentos-package.json b/software/duckdb/agentos-package.json new file mode 100644 index 0000000000..bb48072275 --- /dev/null +++ b/software/duckdb/agentos-package.json @@ -0,0 +1,12 @@ +{ + "commands": [ + "duckdb" + ], + "registry": { + "title": "DuckDB", + "description": "In-process analytics database CLI.", + "priority": 70, + "image": "/images/registry/duckdb.svg", + "category": "data" + } +} diff --git a/registry/native/c/patches/duckdb/0001-skip-http-object-when-extension-loading-is-disabled.patch b/software/duckdb/native/c/patches/0001-skip-http-object-when-extension-loading-is-disabled.patch similarity index 100% rename from registry/native/c/patches/duckdb/0001-skip-http-object-when-extension-loading-is-disabled.patch rename to software/duckdb/native/c/patches/0001-skip-http-object-when-extension-loading-is-disabled.patch diff --git a/registry/native/c/patches/duckdb/0002-disable-linenoise-on-wasi.patch b/software/duckdb/native/c/patches/0002-disable-linenoise-on-wasi.patch similarity index 100% rename from registry/native/c/patches/duckdb/0002-disable-linenoise-on-wasi.patch rename to software/duckdb/native/c/patches/0002-disable-linenoise-on-wasi.patch diff --git a/registry/native/c/patches/duckdb/0003-disable-shell-system-command-on-wasi.patch b/software/duckdb/native/c/patches/0003-disable-shell-system-command-on-wasi.patch similarity index 100% rename from registry/native/c/patches/duckdb/0003-disable-shell-system-command-on-wasi.patch rename to software/duckdb/native/c/patches/0003-disable-shell-system-command-on-wasi.patch diff --git a/registry/native/c/patches/duckdb/0004-skip-http-util-init-when-extension-loading-is-disabled.patch b/software/duckdb/native/c/patches/0004-skip-http-util-init-when-extension-loading-is-disabled.patch similarity index 100% rename from registry/native/c/patches/duckdb/0004-skip-http-util-init-when-extension-loading-is-disabled.patch rename to software/duckdb/native/c/patches/0004-skip-http-util-init-when-extension-loading-is-disabled.patch diff --git a/registry/native/c/patches/duckdb/0005-disable-shell-pager-popen-on-wasi.patch b/software/duckdb/native/c/patches/0005-disable-shell-pager-popen-on-wasi.patch similarity index 100% rename from registry/native/c/patches/duckdb/0005-disable-shell-pager-popen-on-wasi.patch rename to software/duckdb/native/c/patches/0005-disable-shell-pager-popen-on-wasi.patch diff --git a/software/duckdb/package.json b/software/duckdb/package.json new file mode 100644 index 0000000000..0c892487f9 --- /dev/null +++ b/software/duckdb/package.json @@ -0,0 +1,33 @@ +{ + "name": "@agentos-software/duckdb", + "version": "0.3.3", + "type": "module", + "license": "Apache-2.0", + "description": "DuckDB CLI for secure-exec VMs", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "files": [ + "dist", + "!dist/package", + "!dist/package.tar" + ], + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "scripts": { + "build": "agentos-toolchain stage --commands-dir ../../toolchain/target/wasm32-wasip1/release/commands --if-missing skip && tsc && agentos-toolchain build", + "check-types": "tsc --noEmit", + "test": "vitest run test/ --passWithNoTests" + }, + "devDependencies": { + "@agentos-software/manifest": "workspace:*", + "@rivet-dev/agentos-toolchain": "workspace:*", + "@types/node": "^22.10.2", + "typescript": "^5.9.2", + "@agentos/test-harness": "workspace:*", + "vitest": "^2.1.9" + } +} diff --git a/registry/software/coreutils/src/index.ts b/software/duckdb/src/index.ts similarity index 100% rename from registry/software/coreutils/src/index.ts rename to software/duckdb/src/index.ts diff --git a/registry/tests/wasmvm/duckdb.test.ts b/software/duckdb/test/duckdb.test.ts similarity index 92% rename from registry/tests/wasmvm/duckdb.test.ts rename to software/duckdb/test/duckdb.test.ts index f1dddc9d46..06349c73fb 100644 --- a/registry/tests/wasmvm/duckdb.test.ts +++ b/software/duckdb/test/duckdb.test.ts @@ -23,8 +23,8 @@ import { createWasmVmRuntime, describeIf, itIf, -} from '../helpers.js'; -import type { Kernel } from '../helpers.js'; +} from '@agentos/test-harness'; +import type { Kernel } from '@agentos/test-harness'; import { createServer, type IncomingMessage, type Server, type ServerResponse } from 'node:http'; import { existsSync } from 'node:fs'; import { resolve } from 'node:path'; @@ -33,7 +33,6 @@ import { setTimeout as sleep } from 'node:timers/promises'; const hasWasmDuckDB = existsSync(resolve(C_BUILD_DIR, 'duckdb')); const hasWasmCurl = (existsSync(resolve(COMMANDS_DIR, 'curl')) || existsSync(resolve(C_BUILD_DIR, 'curl'))); -const hasWasmHttpGet = existsSync(resolve(C_BUILD_DIR, 'http_get')); async function mountKernel( filesystem: ReturnType, @@ -52,7 +51,7 @@ async function mountKernel( commandDirs, permissions: { duckdb: 'read-write', - http_get: 'full', + curl: 'full', }, }) ); @@ -82,8 +81,8 @@ async function waitForFilesystemPath( } } -// TODO(P6): requires duckdb WASM artifact, intentionally excluded from the fast registry-build gate. -describe.skip('duckdb command', { timeout: 120_000 }, () => { +// TODO(P6): requires duckdb WASM artifact, intentionally excluded from the fast software-build gate. +describeIf(hasWasmDuckDB, 'duckdb command', { timeout: 120_000 }, () => { let kernel: Kernel | undefined; afterEach(async () => { @@ -215,7 +214,7 @@ describe.skip('duckdb command', { timeout: 120_000 }, () => { }); itIf( - hasWasmCurl || hasWasmHttpGet, + hasWasmCurl, 'queries data fetched over the network through the shared VFS', async () => { const filesystem = createInMemoryFileSystem(); @@ -243,18 +242,10 @@ describe.skip('duckdb command', { timeout: 120_000 }, () => { loopbackExemptPorts: [address.port], }); - let result; - if (hasWasmHttpGet) { - result = await kernel.exec( - `http_get ${address.port} /remote.csv /tmp/remote.csv` - ); - expect(result.exitCode).toBe(0); - } else { - result = await kernel.exec( - `curl -fsS -o /tmp/remote.csv http://127.0.0.1:${address.port}/remote.csv` - ); - expect(result.exitCode).toBe(0); - } + let result = await kernel.exec( + `curl -fsS -o /tmp/remote.csv http://127.0.0.1:${address.port}/remote.csv` + ); + expect(result.exitCode).toBe(0); expect(await filesystem.readTextFile('/tmp/remote.csv')).toContain('city,value'); diff --git a/software/duckdb/tsconfig.json b/software/duckdb/tsconfig.json new file mode 100644 index 0000000000..03ce790ab7 --- /dev/null +++ b/software/duckdb/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*"] +} diff --git a/software/envsubst/agentos-package.json b/software/envsubst/agentos-package.json new file mode 100644 index 0000000000..4d990285ff --- /dev/null +++ b/software/envsubst/agentos-package.json @@ -0,0 +1,11 @@ +{ + "commands": [ + "envsubst" + ], + "registry": { + "title": "envsubst", + "description": "Substitute environment variables in text streams.", + "priority": 10, + "category": "core" + } +} diff --git a/registry/native/c/programs/envsubst.c b/software/envsubst/native/c/envsubst.c similarity index 100% rename from registry/native/c/programs/envsubst.c rename to software/envsubst/native/c/envsubst.c diff --git a/software/envsubst/package.json b/software/envsubst/package.json new file mode 100644 index 0000000000..397f9dc2c2 --- /dev/null +++ b/software/envsubst/package.json @@ -0,0 +1,33 @@ +{ + "name": "@agentos-software/envsubst", + "version": "0.3.3", + "type": "module", + "license": "Apache-2.0", + "description": "envsubst environment variable substitution command for AgentOS VMs", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "files": [ + "dist", + "!dist/package", + "!dist/package.tar" + ], + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "scripts": { + "build": "agentos-toolchain stage --commands-dir ../../toolchain/target/wasm32-wasip1/release/commands --if-missing skip && tsc && agentos-toolchain build", + "check-types": "tsc --noEmit", + "test": "vitest run test/ --passWithNoTests" + }, + "devDependencies": { + "@agentos-software/manifest": "workspace:*", + "@agentos/test-harness": "workspace:*", + "@rivet-dev/agentos-toolchain": "workspace:*", + "@types/node": "^22.10.2", + "typescript": "^5.9.2", + "vitest": "^2.1.9" + } +} diff --git a/registry/software/curl/src/index.ts b/software/envsubst/src/index.ts similarity index 100% rename from registry/software/curl/src/index.ts rename to software/envsubst/src/index.ts diff --git a/registry/tests/wasmvm/envsubst.test.ts b/software/envsubst/test/envsubst.test.ts similarity index 95% rename from registry/tests/wasmvm/envsubst.test.ts rename to software/envsubst/test/envsubst.test.ts index 3415284a6d..4897d8b700 100644 --- a/registry/tests/wasmvm/envsubst.test.ts +++ b/software/envsubst/test/envsubst.test.ts @@ -10,9 +10,9 @@ */ import { describe, it, expect, afterEach } from 'vitest'; -import { createWasmVmRuntime } from '../helpers.js'; -import { C_BUILD_DIR, COMMANDS_DIR, createKernel } from '../helpers.js'; -import type { Kernel } from '../helpers.js'; +import { createWasmVmRuntime } from '@agentos/test-harness'; +import { C_BUILD_DIR, COMMANDS_DIR, createKernel, describeIf, hasCWasmBinaries } from '@agentos/test-harness'; +import type { Kernel } from '@agentos/test-harness'; // Minimal in-memory VFS for kernel tests class SimpleVFS { @@ -102,7 +102,7 @@ class SimpleVFS { } } -describe('envsubst command', () => { +describeIf(hasCWasmBinaries('envsubst'), 'envsubst command', () => { let kernel: Kernel; afterEach(async () => { diff --git a/software/envsubst/tsconfig.json b/software/envsubst/tsconfig.json new file mode 100644 index 0000000000..73c85c8fee --- /dev/null +++ b/software/envsubst/tsconfig.json @@ -0,0 +1,12 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src", + "composite": true, + "tsBuildInfoFile": "dist/.tsbuildinfo" + }, + "include": [ + "src/**/*.ts" + ] +} diff --git a/software/everything/agentos-package.json b/software/everything/agentos-package.json new file mode 100644 index 0000000000..6dc26907f3 --- /dev/null +++ b/software/everything/agentos-package.json @@ -0,0 +1,8 @@ +{ + "registry": { + "title": "Everything", + "description": "Meta-package: all available WASM command packages.", + "priority": 110, + "category": "meta" + } +} diff --git a/software/everything/package.json b/software/everything/package.json new file mode 100644 index 0000000000..95ec6ff1ea --- /dev/null +++ b/software/everything/package.json @@ -0,0 +1,58 @@ +{ + "name": "@agentos-software/everything", + "version": "0.3.0-rc.2", + "type": "module", + "license": "Apache-2.0", + "description": "All available WASM command packages for secure-exec VMs in a single bundle", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "files": [ + "dist", + "!dist/package", + "!dist/package.tar" + ], + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "scripts": { + "build": "tsc", + "check-types": "tsc --noEmit", + "test": "vitest run test/ --passWithNoTests" + }, + "dependencies": { + "@agentos-software/coreutils": "workspace:*", + "@agentos-software/sed": "workspace:*", + "@agentos-software/grep": "workspace:*", + "@agentos-software/gawk": "workspace:*", + "@agentos-software/findutils": "workspace:*", + "@agentos-software/diffutils": "workspace:*", + "@agentos-software/tar": "workspace:*", + "@agentos-software/gzip": "workspace:*", + "@agentos-software/curl": "workspace:*", + "@agentos-software/wget": "workspace:*", + "@agentos-software/duckdb": "workspace:*", + "@agentos-software/envsubst": "workspace:*", + "@agentos-software/git": "workspace:*", + "@agentos-software/sqlite3": "workspace:*", + "@agentos-software/vim": "workspace:*", + "@agentos-software/zip": "workspace:*", + "@agentos-software/unzip": "workspace:*", + "@agentos-software/jq": "workspace:*", + "@agentos-software/ripgrep": "workspace:*", + "@agentos-software/fd": "workspace:*", + "@agentos-software/tree": "workspace:*", + "@agentos-software/file": "workspace:*", + "@agentos-software/yq": "workspace:*", + "@agentos-software/codex-cli": "workspace:*" + }, + "devDependencies": { + "@agentos-software/manifest": "workspace:*", + "@types/node": "^22.10.2", + "typescript": "^5.9.2", + "@agentos/test-harness": "workspace:*", + "vitest": "^2.1.9" + } +} diff --git a/software/everything/src/index.ts b/software/everything/src/index.ts new file mode 100644 index 0000000000..c45ab24679 --- /dev/null +++ b/software/everything/src/index.ts @@ -0,0 +1,79 @@ +import coreutils from "@agentos-software/coreutils"; +import sed from "@agentos-software/sed"; +import grep from "@agentos-software/grep"; +import gawk from "@agentos-software/gawk"; +import findutils from "@agentos-software/findutils"; +import diffutils from "@agentos-software/diffutils"; +import tar from "@agentos-software/tar"; +import gzip from "@agentos-software/gzip"; +import curl from "@agentos-software/curl"; +import wget from "@agentos-software/wget"; +import duckdb from "@agentos-software/duckdb"; +import envsubst from "@agentos-software/envsubst"; +import git from "@agentos-software/git"; +import sqlite3 from "@agentos-software/sqlite3"; +import vim from "@agentos-software/vim"; +import zip from "@agentos-software/zip"; +import unzip from "@agentos-software/unzip"; +import jq from "@agentos-software/jq"; +import ripgrep from "@agentos-software/ripgrep"; +import fd from "@agentos-software/fd"; +import tree from "@agentos-software/tree"; +import file from "@agentos-software/file"; +import yq from "@agentos-software/yq"; +import codex from "@agentos-software/codex-cli"; + +const everything = [ + coreutils, + sed, + grep, + gawk, + findutils, + diffutils, + tar, + gzip, + curl, + wget, + duckdb, + envsubst, + git, + sqlite3, + vim, + zip, + unzip, + jq, + ripgrep, + fd, + tree, + file, + yq, + codex, +]; + +export default everything; +export { + coreutils, + sed, + grep, + gawk, + findutils, + diffutils, + tar, + gzip, + curl, + wget, + duckdb, + envsubst, + git, + sqlite3, + vim, + zip, + unzip, + jq, + ripgrep, + fd, + tree, + file, + yq, + codex, +}; diff --git a/software/everything/test/everything.test.ts b/software/everything/test/everything.test.ts new file mode 100644 index 0000000000..f9d1ea24af --- /dev/null +++ b/software/everything/test/everything.test.ts @@ -0,0 +1,81 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import everything, { + codex, + coreutils, + curl, + diffutils, + duckdb, + envsubst, + fd, + file, + findutils, + gawk, + git, + grep, + gzip, + jq, + ripgrep, + sed, + sqlite3, + tar, + tree, + unzip, + vim, + wget, + yq, + zip, +} from "../src/index.js"; + +const packageDir = new URL("..", import.meta.url).pathname; + +const expectedMembers = [ + coreutils, + sed, + grep, + gawk, + findutils, + diffutils, + tar, + gzip, + curl, + wget, + duckdb, + envsubst, + git, + sqlite3, + vim, + zip, + unzip, + jq, + ripgrep, + fd, + tree, + file, + yq, + codex, +]; + +describe("everything meta-package", () => { + it("has a registry manifest", () => { + const manifest = JSON.parse( + readFileSync(join(packageDir, "agentos-package.json"), "utf8"), + ); + + expect(manifest.registry).toMatchObject({ + title: "Everything", + category: "meta", + }); + }); + + it("exports every command package descriptor once", () => { + expect(everything).toEqual(expectedMembers); + expect(new Set(everything).size).toBe(everything.length); + for (const descriptor of everything) { + expect(descriptor).toEqual({ + packagePath: expect.stringMatching(/\/package\.aospkg$/), + }); + } + }); +}); diff --git a/software/everything/tsconfig.json b/software/everything/tsconfig.json new file mode 100644 index 0000000000..03ce790ab7 --- /dev/null +++ b/software/everything/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*"] +} diff --git a/software/fd/agentos-package.json b/software/fd/agentos-package.json new file mode 100644 index 0000000000..636140edab --- /dev/null +++ b/software/fd/agentos-package.json @@ -0,0 +1,11 @@ +{ + "commands": [ + "fd" + ], + "registry": { + "title": "fd", + "description": "Fast file finder.", + "priority": 30, + "category": "developer-tools" + } +} diff --git a/software/fd/native/crates/cmd-fd/Cargo.toml b/software/fd/native/crates/cmd-fd/Cargo.toml new file mode 100644 index 0000000000..ef6223d123 --- /dev/null +++ b/software/fd/native/crates/cmd-fd/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-fd" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "fd standalone binary for secure-exec VM" +publish = false + +[dependencies] +# fd-find is a bin-only crate. `toolchain/Makefile` builds its upstream `fd` +# binary directly; this local package exists only so `cmd/fd` is discoverable +# and the upstream package is present in Cargo's resolved/vendor set. +fd-find = { version = "10.4.2", default-features = false } diff --git a/software/fd/native/crates/cmd-fd/src/lib.rs b/software/fd/native/crates/cmd-fd/src/lib.rs new file mode 100644 index 0000000000..a372ecb5e7 --- /dev/null +++ b/software/fd/native/crates/cmd-fd/src/lib.rs @@ -0,0 +1 @@ +//! Build trigger for the upstream `fd-find` command package. diff --git a/software/fd/package.json b/software/fd/package.json new file mode 100644 index 0000000000..014cc77cc2 --- /dev/null +++ b/software/fd/package.json @@ -0,0 +1,33 @@ +{ + "name": "@agentos-software/fd", + "version": "0.3.3", + "type": "module", + "license": "Apache-2.0", + "description": "fd fast file finder for secure-exec VMs", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "files": [ + "dist", + "!dist/package", + "!dist/package.tar" + ], + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "scripts": { + "build": "agentos-toolchain stage --commands-dir ../../toolchain/target/wasm32-wasip1/release/commands --if-missing skip && tsc && agentos-toolchain build", + "check-types": "tsc --noEmit", + "test": "vitest run test/ --passWithNoTests" + }, + "devDependencies": { + "@agentos-software/manifest": "workspace:*", + "@rivet-dev/agentos-toolchain": "workspace:*", + "@types/node": "^22.10.2", + "typescript": "^5.9.2", + "@agentos/test-harness": "workspace:*", + "vitest": "^2.1.9" + } +} diff --git a/registry/software/diffutils/src/index.ts b/software/fd/src/index.ts similarity index 100% rename from registry/software/diffutils/src/index.ts rename to software/fd/src/index.ts diff --git a/software/fd/test/fd.test.ts b/software/fd/test/fd.test.ts new file mode 100644 index 0000000000..0b76049d2f --- /dev/null +++ b/software/fd/test/fd.test.ts @@ -0,0 +1,188 @@ +/** + * Integration tests for fd (fd-find) command. + * + * Verifies file finding with regex patterns, extension filters, type filters, + * hidden file skipping, and empty directory handling via kernel.exec() with + * real WASM binaries. + * + * Note: kernel.exec() wraps commands in sh -c. Brush-shell currently returns + * exit code 17 for all child commands (benign "could not retrieve pid" issue). + * Tests verify stdout correctness rather than exit code. + */ + +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { describe, it, expect, afterEach } from 'vitest'; +import { createWasmVmRuntime } from '@agentos/test-harness'; +import { COMMANDS_DIR, createKernel, describeIf, hasWasmBinaries, NodeFileSystem } from '@agentos/test-harness'; +import type { Kernel } from '@agentos/test-harness'; + +let tempRoot: string | undefined; + +/** Create a VFS pre-populated with a test directory structure */ +async function createTestVFS(): Promise { + tempRoot = await mkdtemp(join(tmpdir(), 'agentos-fd-')); + + // /project/ + // src/ + // main.js + // utils.js + // helpers.ts + // lib/ + // parser.js + // docs/ + // readme.md + // .hidden/ + // secret.txt + // .gitignore + // config.json + await writeFixture('/project/src/main.js', 'console.log("main")'); + await writeFixture('/project/src/utils.js', 'export {}'); + await writeFixture('/project/src/helpers.ts', 'export {}'); + await writeFixture('/project/lib/parser.js', 'module.exports = {}'); + await writeFixture('/project/docs/readme.md', '# Readme'); + await writeFixture('/project/.hidden/secret.txt', 'secret'); + await writeFixture('/project/.gitignore', 'node_modules'); + await writeFixture('/project/config.json', '{}'); + // /empty/ — empty directory + await mkdir(join(tempRoot, 'empty'), { recursive: true }); + return new NodeFileSystem({ root: tempRoot }); +} + +async function writeFixture(path: string, contents: string): Promise { + if (!tempRoot) throw new Error('fixture root not initialized'); + const hostPath = join(tempRoot, path.replace(/^\/+/, '')); + await mkdir(dirname(hostPath), { recursive: true }); + await writeFile(hostPath, contents); +} + +/** Parse fd output lines, sorted for deterministic comparison */ +function parseLines(stdout: string): string[] { + return stdout.split('\n').filter(l => l.length > 0).sort(); +} + +describeIf(hasWasmBinaries, 'fd-find command', { timeout: 10_000 }, () => { + let kernel: Kernel; + + afterEach(async () => { + await kernel?.dispose(); + if (tempRoot) { + await rm(tempRoot, { recursive: true, force: true }); + tempRoot = undefined; + } + }); + + it('reports the upstream fd-find version', async () => { + const vfs = await createTestVFS(); + kernel = createKernel({ filesystem: vfs }); + await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); + + const result = await kernel.exec('fd --version', {}); + expect(result.stdout.trim()).toBe('fd 10.4.2'); + }); + + it('finds files matching regex pattern in current directory', async () => { + const vfs = await createTestVFS(); + kernel = createKernel({ filesystem: vfs }); + await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); + + const result = await kernel.exec('fd main /project', {}); + const lines = parseLines(result.stdout); + expect(lines).toContain('/project/src/main.js'); + }); + + it('finds all .js files with -e js', async () => { + const vfs = await createTestVFS(); + kernel = createKernel({ filesystem: vfs }); + await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); + + const result = await kernel.exec('fd -e js . /project', {}); + const lines = parseLines(result.stdout); + expect(lines).toContain('/project/src/main.js'); + expect(lines).toContain('/project/src/utils.js'); + expect(lines).toContain('/project/lib/parser.js'); + // .ts files should NOT match + expect(lines).not.toContain('/project/src/helpers.ts'); + }); + + it('finds only files with -t f', async () => { + const vfs = await createTestVFS(); + kernel = createKernel({ filesystem: vfs }); + await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); + + const result = await kernel.exec('fd -t f . /project', {}); + const lines = parseLines(result.stdout); + // All entries should be files, not directories + for (const line of lines) { + const stat = await vfs.stat(line); + expect(stat.isDirectory).toBe(false); + } + // Should include known files + expect(lines).toContain('/project/src/main.js'); + expect(lines).toContain('/project/config.json'); + }); + + it('finds only directories with -t d', async () => { + const vfs = await createTestVFS(); + kernel = createKernel({ filesystem: vfs }); + await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); + + const result = await kernel.exec('fd -t d . /project', {}); + const lines = parseLines(result.stdout); + // All entries should be directories + for (const line of lines) { + const stat = await vfs.stat(line); + expect(stat.isDirectory).toBe(true); + } + // Should include known directories (hidden skipped by default) + expect(lines).toContain('/project/src/'); + expect(lines).toContain('/project/lib/'); + expect(lines).toContain('/project/docs/'); + }); + + it('returns no results for empty directory', async () => { + const vfs = await createTestVFS(); + kernel = createKernel({ filesystem: vfs }); + await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); + + const result = await kernel.exec('fd . /empty', {}); + expect(result.stdout.trim()).toBe(''); + }); + + it('returns empty output when no files match pattern', async () => { + const vfs = await createTestVFS(); + kernel = createKernel({ filesystem: vfs }); + await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); + + const result = await kernel.exec('fd zzzznonexistent /project', {}); + expect(result.stdout.trim()).toBe(''); + }); + + it('skips hidden files and directories by default', async () => { + const vfs = await createTestVFS(); + kernel = createKernel({ filesystem: vfs }); + await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); + + const result = await kernel.exec('fd . /project', {}); + const lines = parseLines(result.stdout); + // Hidden files/dirs should NOT appear + const hiddenEntries = lines.filter(l => { + const parts = l.split('/'); + return parts.some(p => p.startsWith('.') && p.length > 1); + }); + expect(hiddenEntries).toEqual([]); + }); + + it('includes hidden files with -H flag', async () => { + const vfs = await createTestVFS(); + kernel = createKernel({ filesystem: vfs }); + await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); + + const result = await kernel.exec('fd -H . /project', {}); + const lines = parseLines(result.stdout); + // Hidden items should now appear + expect(lines).toContain('/project/.gitignore'); + expect(lines).toContain('/project/.hidden/'); + }); +}); diff --git a/software/fd/tsconfig.json b/software/fd/tsconfig.json new file mode 100644 index 0000000000..03ce790ab7 --- /dev/null +++ b/software/fd/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*"] +} diff --git a/software/file/agentos-package.json b/software/file/agentos-package.json new file mode 100644 index 0000000000..ef6f5acc1c --- /dev/null +++ b/software/file/agentos-package.json @@ -0,0 +1,11 @@ +{ + "commands": [ + "file" + ], + "registry": { + "title": "file", + "description": "Detect file types.", + "priority": 2, + "category": "core" + } +} diff --git a/software/file/native/crates/cmd-file/Cargo.toml b/software/file/native/crates/cmd-file/Cargo.toml new file mode 100644 index 0000000000..65e6ee9f90 --- /dev/null +++ b/software/file/native/crates/cmd-file/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-file" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "file standalone binary for secure-exec VM" + +[[bin]] +name = "file" +path = "src/main.rs" + +[dependencies] +secureexec-file-cmd = { path = "../file-cmd" } diff --git a/registry/native/crates/commands/file/src/main.rs b/software/file/native/crates/cmd-file/src/main.rs similarity index 100% rename from registry/native/crates/commands/file/src/main.rs rename to software/file/native/crates/cmd-file/src/main.rs diff --git a/software/file/native/crates/file-cmd/Cargo.toml b/software/file/native/crates/file-cmd/Cargo.toml new file mode 100644 index 0000000000..203113931a --- /dev/null +++ b/software/file/native/crates/file-cmd/Cargo.toml @@ -0,0 +1,10 @@ +[package] +workspace = "../../../../../toolchain" +name = "secureexec-file-cmd" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "file command implementation for secure-exec standalone binaries" + +[dependencies] +infer = "0.16" diff --git a/software/file/native/crates/file-cmd/src/lib.rs b/software/file/native/crates/file-cmd/src/lib.rs new file mode 100644 index 0000000000..38f507dd73 --- /dev/null +++ b/software/file/native/crates/file-cmd/src/lib.rs @@ -0,0 +1,336 @@ +//! file -- determine file type +//! +//! Magic byte detection using the `infer` crate, plus text/binary heuristic. +//! Supports -b (brief, no filename prefix), -i (output MIME type). + +use std::ffi::OsString; +use std::fs; +use std::io::{self, Read, Write}; + +const DETECTION_BYTES: usize = 8192; + +pub fn main(args: Vec) -> i32 { + let str_args: Vec = args + .iter() + .skip(1) + .map(|a| a.to_string_lossy().to_string()) + .collect(); + + let mut brief = false; + let mut mime = false; + let mut filenames: Vec = Vec::new(); + + let mut i = 0; + while i < str_args.len() { + match str_args[i].as_str() { + "-b" | "--brief" => brief = true, + "-i" | "--mime-type" | "--mime" => mime = true, + // Combined short flags + s if s.starts_with('-') && s.len() > 1 && !s.starts_with("--") => { + for ch in s[1..].chars() { + match ch { + 'b' => brief = true, + 'i' => mime = true, + _ => { + eprintln!("file: unknown option '{}'", s); + return 1; + } + } + } + } + "-" => { + // Read from stdin + filenames.push("-".to_string()); + } + s if s.starts_with('-') && s.len() > 1 => { + eprintln!("file: unknown option '{}'", s); + return 1; + } + _ => filenames.push(str_args[i].clone()), + } + i += 1; + } + + if filenames.is_empty() { + eprintln!("Usage: file [-bi] FILE..."); + return 1; + } + + let stdout = io::stdout(); + let mut out = stdout.lock(); + let mut exit_code = 0; + + for filename in &filenames { + let data = if filename == "-" { + match read_head_from_reader(io::stdin().lock(), DETECTION_BYTES) { + Ok(data) => data, + Err(e) => { + eprintln!("file: stdin: {}", e); + exit_code = 1; + continue; + } + } + } else { + // Check if path exists and what kind it is + match fs::metadata(filename) { + Ok(meta) => { + if meta.is_dir() { + if let Err(error) = print_result( + &mut out, + filename, + brief, + "directory", + if mime { "inode/directory" } else { "" }, + mime, + ) { + return output_error(error); + } + continue; + } + if meta.is_symlink() { + if let Err(error) = print_result( + &mut out, + filename, + brief, + "symbolic link", + if mime { "inode/symlink" } else { "" }, + mime, + ) { + return output_error(error); + } + continue; + } + if meta.len() == 0 { + if let Err(error) = print_result( + &mut out, + filename, + brief, + "empty", + if mime { "inode/x-empty" } else { "" }, + mime, + ) { + return output_error(error); + } + continue; + } + // Read file data (up to 8KB for detection) + match read_head(filename, DETECTION_BYTES) { + Ok(data) => data, + Err(e) => { + eprintln!("file: {}: {}", filename, e); + exit_code = 1; + continue; + } + } + } + Err(e) => { + eprintln!("file: {}: {}", filename, e); + exit_code = 1; + continue; + } + } + }; + + let (desc, mime_type) = identify(&data); + + if mime { + if let Err(error) = print_result(&mut out, filename, brief, &desc, &mime_type, true) { + return output_error(error); + } + } else { + if let Err(error) = print_result(&mut out, filename, brief, &desc, &mime_type, false) { + return output_error(error); + } + } + } + + if let Err(error) = out.flush() { + return output_error(error); + } + + exit_code +} + +fn output_error(error: io::Error) -> i32 { + eprintln!("file: failed to write output: {error}"); + 1 +} + +fn read_head(path: &str, max: usize) -> io::Result> { + let mut f = fs::File::open(path)?; + read_head_from_reader(&mut f, max) +} + +fn read_head_from_reader(mut reader: impl Read, max: usize) -> io::Result> { + let mut buf = vec![0u8; max]; + let n = reader.read(&mut buf)?; + buf.truncate(n); + Ok(buf) +} + +fn print_result( + out: &mut W, + filename: &str, + brief: bool, + desc: &str, + mime_type: &str, + use_mime: bool, +) -> io::Result<()> { + if use_mime && !mime_type.is_empty() { + if brief { + writeln!(out, "{}", mime_type) + } else { + writeln!(out, "{}: {}", filename, mime_type) + } + } else if brief { + writeln!(out, "{}", desc) + } else { + writeln!(out, "{}: {}", filename, desc) + } +} + +fn identify(data: &[u8]) -> (String, String) { + if data.is_empty() { + return ("empty".to_string(), "inode/x-empty".to_string()); + } + + // Check for shebang scripts + if data.len() >= 2 && data[0] == b'#' && data[1] == b'!' { + let first_line = data + .iter() + .take(128) + .take_while(|&&b| b != b'\n') + .copied() + .collect::>(); + if let Ok(line) = std::str::from_utf8(&first_line) { + let interp = line.trim_start_matches("#!"); + let interp = interp.trim(); + // Extract interpreter name + let name = interp.split_whitespace().next().unwrap_or(interp); + let basename = match name.rfind('/') { + Some(pos) => &name[pos + 1..], + None => name, + }; + // Handle "env " pattern + if basename == "env" { + let prog = interp.split_whitespace().nth(1).unwrap_or("script"); + return ( + format!("{} script, ASCII text executable", prog), + "text/x-script".to_string(), + ); + } + return ( + format!("{} script, ASCII text executable", basename), + "text/x-script".to_string(), + ); + } + } + + // Try infer crate for magic byte detection + if let Some(kind) = infer::get(data) { + let desc = match kind.mime_type() { + "image/png" => "PNG image data", + "image/jpeg" => "JPEG image data", + "image/gif" => "GIF image data", + "image/webp" => "WebP image data", + "image/bmp" => "BMP image data", + "image/tiff" => "TIFF image data", + "image/svg+xml" => "SVG image data", + "image/x-icon" => "ICO image data", + "application/pdf" => "PDF document", + "application/zip" => "Zip archive data", + "application/gzip" => "gzip compressed data", + "application/x-bzip2" => "bzip2 compressed data", + "application/x-xz" => "XZ compressed data", + "application/x-tar" => "POSIX tar archive", + "application/x-rar-compressed" => "RAR archive data", + "application/x-7z-compressed" => "7-zip archive data", + "application/x-executable" | "application/x-elf" => "ELF executable", + "application/wasm" => "WebAssembly (wasm) binary module", + "application/x-mach-binary" => "Mach-O binary", + "audio/mpeg" => "MPEG audio", + "audio/ogg" => "Ogg audio", + "audio/flac" => "FLAC audio", + "audio/wav" | "audio/x-wav" => "RIFF WAVE audio", + "video/mp4" => "MPEG-4 video", + "video/webm" => "WebM video", + "video/x-matroska" => "Matroska video", + "application/x-sqlite3" => "SQLite 3.x database", + "font/woff" => "Web Open Font Format", + "font/woff2" => "Web Open Font Format 2", + _ => kind.mime_type(), + }; + return (desc.to_string(), kind.mime_type().to_string()); + } + + // Check for known text patterns + if is_json(data) { + return ("JSON text data".to_string(), "application/json".to_string()); + } + if is_xml(data) { + return ("XML document".to_string(), "text/xml".to_string()); + } + if is_html(data) { + return ("HTML document".to_string(), "text/html".to_string()); + } + + // Text vs binary heuristic + if is_text(data) { + return ("ASCII text".to_string(), "text/plain".to_string()); + } + + ("data".to_string(), "application/octet-stream".to_string()) +} + +fn is_text(data: &[u8]) -> bool { + // Check first 8KB for non-text bytes + let check_len = data.len().min(8192); + let mut null_count = 0; + for &b in &data[..check_len] { + if b == 0 { + null_count += 1; + if null_count > 0 { + return false; + } + } + // Allow common control chars: tab, newline, carriage return, form feed, backspace, escape + if b < 0x08 || (b > 0x0D && b < 0x20 && b != 0x1B) { + if b != 0x00 { + // already counted nulls + return false; + } + } + } + true +} + +fn is_json(data: &[u8]) -> bool { + // Quick check: starts with { or [, ignoring leading whitespace + let trimmed = skip_whitespace(data); + !trimmed.is_empty() && (trimmed[0] == b'{' || trimmed[0] == b'[') +} + +fn is_xml(data: &[u8]) -> bool { + let trimmed = skip_whitespace(data); + trimmed.starts_with(b" bool { + let trimmed = skip_whitespace(data); + let lower: Vec = trimmed + .iter() + .take(64) + .map(|b| b.to_ascii_lowercase()) + .collect(); + lower.starts_with(b" &[u8] { + let mut i = 0; + while i < data.len() + && (data[i] == b' ' || data[i] == b'\t' || data[i] == b'\n' || data[i] == b'\r') + { + i += 1; + } + &data[i..] +} diff --git a/software/file/package.json b/software/file/package.json new file mode 100644 index 0000000000..424733c085 --- /dev/null +++ b/software/file/package.json @@ -0,0 +1,33 @@ +{ + "name": "@agentos-software/file", + "version": "0.3.3", + "type": "module", + "license": "Apache-2.0", + "description": "file type detection for secure-exec VMs", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "files": [ + "dist", + "!dist/package", + "!dist/package.tar" + ], + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "scripts": { + "build": "agentos-toolchain stage --commands-dir ../../toolchain/target/wasm32-wasip1/release/commands --if-missing skip && tsc && agentos-toolchain build", + "check-types": "tsc --noEmit", + "test": "vitest run test/ --passWithNoTests" + }, + "devDependencies": { + "@agentos-software/manifest": "workspace:*", + "@rivet-dev/agentos-toolchain": "workspace:*", + "@types/node": "^22.10.2", + "typescript": "^5.9.2", + "@agentos/test-harness": "workspace:*", + "vitest": "^2.1.9" + } +} diff --git a/registry/software/duckdb/src/index.ts b/software/file/src/index.ts similarity index 100% rename from registry/software/duckdb/src/index.ts rename to software/file/src/index.ts diff --git a/software/file/test/file.test.ts b/software/file/test/file.test.ts new file mode 100644 index 0000000000..70657e05b8 --- /dev/null +++ b/software/file/test/file.test.ts @@ -0,0 +1,152 @@ +import { existsSync } from "node:fs"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { + NodeFileSystem, + createKernel, + createWasmVmRuntime, + describeIf, +} from "@agentos/test-harness"; +import type { Kernel } from "@agentos/test-harness"; +import { afterEach, expect, it } from "vitest"; + +const FILE_COMMAND_DIR = fileURLToPath(new URL("../bin", import.meta.url)); +const hasFilePackageBinary = existsSync(join(FILE_COMMAND_DIR, "file")); + +let tempRoot: string | undefined; + +async function writeFixture(path: string, contents: string | Buffer): Promise { + if (!tempRoot) throw new Error("fixture root not initialized"); + const hostPath = join(tempRoot, path.replace(/^\/+/, "")); + await mkdir(dirname(hostPath), { recursive: true }); + await writeFile(hostPath, contents); +} + +async function createTestVFS(): Promise { + tempRoot = await mkdtemp(join(tmpdir(), "agentos-file-")); + await writeFixture("/project/text.txt", "hello from AgentOS\n"); + await writeFixture("/project/data.json", '{ "ok": true }\n'); + await writeFixture("/project/script.sh", "#!/usr/bin/env bash\necho hello\n"); + await writeFixture( + "/project/image.png", + Buffer.from([ + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, + 0x0d, + ]), + ); + await writeFixture("/project/empty", ""); + await mkdir(join(tempRoot, "project/dir"), { recursive: true }); + return new NodeFileSystem({ root: tempRoot }); +} + +describeIf(hasFilePackageBinary, "file command", { timeout: 10_000 }, () => { + let kernel: Kernel | undefined; + + afterEach(async () => { + await kernel?.dispose(); + kernel = undefined; + if (tempRoot) { + await rm(tempRoot, { recursive: true, force: true }); + tempRoot = undefined; + } + }); + + async function mountFixture(): Promise { + const vfs = await createTestVFS(); + kernel = createKernel({ filesystem: vfs }); + await kernel.mount(createWasmVmRuntime({ commandDirs: [FILE_COMMAND_DIR] })); + } + + async function runFile(args: string[], stdin?: string) { + if (!kernel) throw new Error("kernel not mounted"); + let stdout = ""; + let stderr = ""; + const proc = kernel.spawn("file", args, { + streamStdin: stdin !== undefined, + onStdout: (chunk) => { + stdout += Buffer.from(chunk).toString("utf8"); + }, + onStderr: (chunk) => { + stderr += Buffer.from(chunk).toString("utf8"); + }, + }); + if (stdin !== undefined) { + proc.writeStdin(stdin); + proc.closeStdin(); + } + const exitCode = await proc.wait(); + await new Promise((resolve) => setTimeout(resolve, 0)); + return { stdout, stderr, exitCode }; + } + + it("identifies text, JSON, scripts, images, empty files, and directories", async () => { + await mountFixture(); + + const result = await runFile([ + "/project/text.txt", + "/project/data.json", + "/project/script.sh", + "/project/image.png", + "/project/empty", + "/project/dir", + ]); + + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(result.stdout).toContain("/project/text.txt: ASCII text"); + expect(result.stdout).toContain("/project/data.json: JSON text data"); + expect(result.stdout).toContain( + "/project/script.sh: bash script, ASCII text executable", + ); + expect(result.stdout).toContain("/project/image.png: PNG image data"); + expect(result.stdout).toContain("/project/empty: empty"); + expect(result.stdout).toContain("/project/dir: directory"); + }); + + it("prints brief descriptions with -b", async () => { + await mountFixture(); + + const result = await runFile(["-b", "/project/data.json"]); + + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(result.stdout.trim()).toBe("JSON text data"); + }); + + it("prints MIME types with -i", async () => { + await mountFixture(); + + const result = await runFile([ + "-i", + "/project/text.txt", + "/project/image.png", + "/project/dir", + ]); + + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(result.stdout).toContain("/project/text.txt: text/plain"); + expect(result.stdout).toContain("/project/image.png: image/png"); + expect(result.stdout).toContain("/project/dir: inode/directory"); + }); + + it("reads stdin when the operand is -", async () => { + await mountFixture(); + + const result = await runFile( + ["-b", "-"], + "#!/usr/bin/env node\nconsole.log('hello')\n", + ); + + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(result.stdout.trim()).toBe("node script, ASCII text executable"); + }); + + it("returns an error for missing inputs", async () => { + await mountFixture(); + + const result = await runFile(["/project/missing.txt"]); + + expect(result.exitCode).toBe(1); + expect(result.stderr).toContain("/project/missing.txt"); + }); +}); diff --git a/software/file/test/manifest.test.ts b/software/file/test/manifest.test.ts new file mode 100644 index 0000000000..89139d33b8 --- /dev/null +++ b/software/file/test/manifest.test.ts @@ -0,0 +1,19 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +const packageDir = new URL("..", import.meta.url).pathname; + +describe("package manifest", () => { + it("declares command binaries", () => { + const manifest = JSON.parse( + readFileSync(join(packageDir, "agentos-package.json"), "utf8"), + ); + + expect(manifest.commands?.length ?? 0).toBeGreaterThan(0); + for (const command of manifest.commands) { + expect(typeof command).toBe("string"); + expect(command.length).toBeGreaterThan(0); + } + }); +}); diff --git a/software/file/tsconfig.json b/software/file/tsconfig.json new file mode 100644 index 0000000000..03ce790ab7 --- /dev/null +++ b/software/file/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*"] +} diff --git a/software/findutils/agentos-package.json b/software/findutils/agentos-package.json new file mode 100644 index 0000000000..2415eb8b79 --- /dev/null +++ b/software/findutils/agentos-package.json @@ -0,0 +1,13 @@ +{ + "commands": [ + "find", + "xargs" + ], + "registry": { + "title": "findutils", + "description": "GNU find and xargs for file searching and batch execution.", + "priority": 15, + "image": "/images/registry/findutils.svg", + "category": "core" + } +} diff --git a/software/findutils/native/crates/cmd-find/Cargo.toml b/software/findutils/native/crates/cmd-find/Cargo.toml new file mode 100644 index 0000000000..19cc8f38f4 --- /dev/null +++ b/software/findutils/native/crates/cmd-find/Cargo.toml @@ -0,0 +1,15 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-find" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "find standalone binary for secure-exec VM" + +[[bin]] +name = "find" +path = "src/main.rs" + +[dependencies] +findutils = { version = "0.9.0", default-features = false } +uucore = { version = "0.9.0", default-features = false } diff --git a/software/findutils/native/crates/cmd-find/src/main.rs b/software/findutils/native/crates/cmd-find/src/main.rs new file mode 100644 index 0000000000..8990e43353 --- /dev/null +++ b/software/findutils/native/crates/cmd-find/src/main.rs @@ -0,0 +1,11 @@ +fn main() { + uucore::panic::mute_sigpipe_panic(); + + let args = std::env::args().collect::>(); + let args = args + .iter() + .map(std::convert::AsRef::as_ref) + .collect::>(); + let deps = findutils::find::StandardDependencies::new(); + std::process::exit(findutils::find::find_main(&args, &deps)); +} diff --git a/software/findutils/native/crates/cmd-xargs/Cargo.toml b/software/findutils/native/crates/cmd-xargs/Cargo.toml new file mode 100644 index 0000000000..1037c80a58 --- /dev/null +++ b/software/findutils/native/crates/cmd-xargs/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-xargs" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "xargs standalone binary for secure-exec VM" + +[[bin]] +name = "xargs" +path = "src/main.rs" + +[dependencies] +findutils = { version = "0.9.0", default-features = false } diff --git a/software/findutils/native/crates/cmd-xargs/src/main.rs b/software/findutils/native/crates/cmd-xargs/src/main.rs new file mode 100644 index 0000000000..85586163c6 --- /dev/null +++ b/software/findutils/native/crates/cmd-xargs/src/main.rs @@ -0,0 +1,8 @@ +fn main() { + let args = std::env::args().collect::>(); + let args = args + .iter() + .map(std::convert::AsRef::as_ref) + .collect::>(); + std::process::exit(findutils::xargs::xargs_main(&args)); +} diff --git a/software/findutils/package.json b/software/findutils/package.json new file mode 100644 index 0000000000..dd3cdd04c3 --- /dev/null +++ b/software/findutils/package.json @@ -0,0 +1,33 @@ +{ + "name": "@agentos-software/findutils", + "version": "0.3.3", + "type": "module", + "license": "Apache-2.0", + "description": "GNU findutils for secure-exec VMs (find, xargs)", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "files": [ + "dist", + "!dist/package", + "!dist/package.tar" + ], + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "scripts": { + "build": "agentos-toolchain stage --commands-dir ../../toolchain/target/wasm32-wasip1/release/commands --if-missing skip && tsc && agentos-toolchain build", + "check-types": "tsc --noEmit", + "test": "vitest run test/ --passWithNoTests" + }, + "devDependencies": { + "@agentos-software/manifest": "workspace:*", + "@rivet-dev/agentos-toolchain": "workspace:*", + "@types/node": "^22.10.2", + "typescript": "^5.9.2", + "@agentos/test-harness": "workspace:*", + "vitest": "^2.1.9" + } +} diff --git a/registry/software/fd/src/index.ts b/software/findutils/src/index.ts similarity index 100% rename from registry/software/fd/src/index.ts rename to software/findutils/src/index.ts diff --git a/software/findutils/test/findutils.test.ts b/software/findutils/test/findutils.test.ts new file mode 100644 index 0000000000..f343e46864 --- /dev/null +++ b/software/findutils/test/findutils.test.ts @@ -0,0 +1,105 @@ +/** + * Package-owned integration tests for GNU findutils commands. + */ + +import { afterEach, expect, it } from "vitest"; +import { + COMMANDS_DIR, + createInMemoryFileSystem, + createKernel, + createWasmVmRuntime, + describeIf, + hasWasmBinaries, + type Kernel, +} from "@agentos/test-harness"; + +function parseLines(stdout: string): string[] { + return stdout + .split("\n") + .map((line) => line.trim()) + .filter(Boolean) + .sort(); +} + +describeIf(hasWasmBinaries, "findutils commands", { timeout: 10_000 }, () => { + let kernel: Kernel; + + afterEach(async () => { + await kernel?.dispose(); + }); + + it("find matches files by name", async () => { + const vfs = createInMemoryFileSystem(); + await vfs.mkdir("/project/src", { recursive: true }); + await vfs.mkdir("/project/docs", { recursive: true }); + await vfs.writeFile("/project/src/main.js", "console.log('main')\n"); + await vfs.writeFile("/project/src/helper.ts", "export {}\n"); + await vfs.writeFile("/project/docs/readme.md", "# Readme\n"); + + kernel = createKernel({ filesystem: vfs }); + await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); + + const result = await kernel.exec("find /project -name '*.js'"); + expect(parseLines(result.stdout)).toEqual(["/project/src/main.js"]); + }); + + it("find filters directories", async () => { + const vfs = createInMemoryFileSystem(); + await vfs.mkdir("/project/src", { recursive: true }); + await vfs.mkdir("/project/docs", { recursive: true }); + await vfs.writeFile("/project/src/main.js", "console.log('main')\n"); + + kernel = createKernel({ filesystem: vfs }); + await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); + + const result = await kernel.exec("find /project -type d"); + expect(parseLines(result.stdout)).toEqual([ + "/project", + "/project/docs", + "/project/src", + ]); + }); + + it("find supports depth limits", async () => { + const vfs = createInMemoryFileSystem(); + await vfs.mkdir("/project/src/nested", { recursive: true }); + await vfs.mkdir("/project/docs", { recursive: true }); + await vfs.writeFile("/project/src/main.js", "console.log('main')\n"); + await vfs.writeFile("/project/src/nested/helper.ts", "export {}\n"); + await vfs.writeFile("/project/docs/readme.md", "# Readme\n"); + + kernel = createKernel({ filesystem: vfs }); + await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); + + const result = await kernel.exec( + "find /project -mindepth 2 -maxdepth 2 -type f -name '*.js'", + ); + expect(parseLines(result.stdout)).toEqual(["/project/src/main.js"]); + }); + + it("xargs passes stdin arguments to a command", async () => { + const vfs = createInMemoryFileSystem(); + await vfs.writeFile("/args.txt", "alpha\nbeta\n"); + + kernel = createKernel({ filesystem: vfs }); + await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); + + const result = await kernel.exec("xargs echo < /args.txt"); + expect(result.stdout.trim()).toBe("alpha beta"); + }); + + it("xargs batches arguments across spawned commands", async () => { + const vfs = createInMemoryFileSystem(); + await vfs.writeFile("/args.txt", "one two three four five\n"); + + kernel = createKernel({ filesystem: vfs }); + await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); + + const result = await kernel.exec("xargs -n 2 echo < /args.txt"); + expect(result.stdout.trim().split("\n")).toEqual([ + "one two", + "three four", + "five", + ]); + }); +}); diff --git a/software/findutils/tsconfig.json b/software/findutils/tsconfig.json new file mode 100644 index 0000000000..03ce790ab7 --- /dev/null +++ b/software/findutils/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*"] +} diff --git a/software/gawk/agentos-package.json b/software/gawk/agentos-package.json new file mode 100644 index 0000000000..5b103e0e2b --- /dev/null +++ b/software/gawk/agentos-package.json @@ -0,0 +1,12 @@ +{ + "commands": [ + "awk" + ], + "registry": { + "title": "gawk", + "description": "GNU awk text processing and data extraction.", + "priority": 20, + "image": "/images/registry/gawk.svg", + "category": "core" + } +} diff --git a/software/gawk/native/crates/awk/Cargo.toml b/software/gawk/native/crates/awk/Cargo.toml new file mode 100644 index 0000000000..c630320b93 --- /dev/null +++ b/software/gawk/native/crates/awk/Cargo.toml @@ -0,0 +1,10 @@ +[package] +workspace = "../../../../../toolchain" +name = "secureexec-awk" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "awk implementation for secure-exec standalone binaries" + +[dependencies] +awk-rs = "0.1.0" diff --git a/registry/native/crates/libs/awk/src/lib.rs b/software/gawk/native/crates/awk/src/lib.rs similarity index 100% rename from registry/native/crates/libs/awk/src/lib.rs rename to software/gawk/native/crates/awk/src/lib.rs diff --git a/software/gawk/native/crates/cmd-awk/Cargo.toml b/software/gawk/native/crates/cmd-awk/Cargo.toml new file mode 100644 index 0000000000..af4cd03274 --- /dev/null +++ b/software/gawk/native/crates/cmd-awk/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-awk" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "awk standalone binary for secure-exec VM" + +[[bin]] +name = "awk" +path = "src/main.rs" + +[dependencies] +secureexec-awk = { path = "../awk" } diff --git a/registry/native/crates/commands/awk/src/main.rs b/software/gawk/native/crates/cmd-awk/src/main.rs similarity index 100% rename from registry/native/crates/commands/awk/src/main.rs rename to software/gawk/native/crates/cmd-awk/src/main.rs diff --git a/software/gawk/package.json b/software/gawk/package.json new file mode 100644 index 0000000000..e2d3831013 --- /dev/null +++ b/software/gawk/package.json @@ -0,0 +1,33 @@ +{ + "name": "@agentos-software/gawk", + "version": "0.3.3", + "type": "module", + "license": "Apache-2.0", + "description": "GNU awk text processing for secure-exec VMs", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "files": [ + "dist", + "!dist/package", + "!dist/package.tar" + ], + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "scripts": { + "build": "agentos-toolchain stage --commands-dir ../../toolchain/target/wasm32-wasip1/release/commands --if-missing skip && tsc && agentos-toolchain build", + "check-types": "tsc --noEmit", + "test": "vitest run test/ --passWithNoTests" + }, + "devDependencies": { + "@agentos-software/manifest": "workspace:*", + "@rivet-dev/agentos-toolchain": "workspace:*", + "@types/node": "^22.10.2", + "typescript": "^5.9.2", + "@agentos/test-harness": "workspace:*", + "vitest": "^2.1.9" + } +} diff --git a/registry/software/file/src/index.ts b/software/gawk/src/index.ts similarity index 100% rename from registry/software/file/src/index.ts rename to software/gawk/src/index.ts diff --git a/software/gawk/test/gawk.test.ts b/software/gawk/test/gawk.test.ts new file mode 100644 index 0000000000..c900d622d1 --- /dev/null +++ b/software/gawk/test/gawk.test.ts @@ -0,0 +1,130 @@ +import { existsSync } from "node:fs"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { + NodeFileSystem, + createKernel, + createWasmVmRuntime, + describeIf, +} from "@agentos/test-harness"; +import type { Kernel } from "@agentos/test-harness"; +import { afterEach, expect, it } from "vitest"; + +const AWK_COMMAND_DIR = fileURLToPath(new URL("../bin", import.meta.url)); +const hasAwkPackageBinary = existsSync(join(AWK_COMMAND_DIR, "awk")); + +let tempRoot: string | undefined; + +async function writeFixture(path: string, contents: string): Promise { + if (!tempRoot) throw new Error("fixture root not initialized"); + const hostPath = join(tempRoot, path.replace(/^\/+/, "")); + await mkdir(dirname(hostPath), { recursive: true }); + await writeFile(hostPath, contents); +} + +async function createTestVFS(): Promise { + tempRoot = await mkdtemp(join(tmpdir(), "agentos-gawk-")); + await writeFixture( + "/project/scores.txt", + [ + "Ada runtime 7", + "Grace docs 5", + "Linus runtime 3", + "Barbara infra 9", + ].join("\n") + "\n", + ); + await writeFixture( + "/project/colon.txt", + ["alpha:build:4", "beta:test:6", "gamma:deploy:2"].join("\n") + "\n", + ); + await writeFixture( + "/project/runtime.awk", + '/runtime/ { print $1 ":" $3 }\nEND { print "done" }\n', + ); + return new NodeFileSystem({ root: tempRoot }); +} + +function lines(stdout: string): string[] { + return stdout.split("\n").filter((line) => line.length > 0); +} + +describeIf(hasAwkPackageBinary, "awk command", { timeout: 10_000 }, () => { + let kernel: Kernel | undefined; + + afterEach(async () => { + await kernel?.dispose(); + kernel = undefined; + if (tempRoot) { + await rm(tempRoot, { recursive: true, force: true }); + tempRoot = undefined; + } + }); + + async function mountFixture(): Promise { + const vfs = await createTestVFS(); + kernel = createKernel({ filesystem: vfs }); + await kernel.mount(createWasmVmRuntime({ commandDirs: [AWK_COMMAND_DIR] })); + } + + async function runAwk(args: string[]) { + if (!kernel) throw new Error("kernel not mounted"); + let stdout = ""; + let stderr = ""; + const proc = kernel.spawn("awk", args, { + onStdout: (chunk) => { + stdout += Buffer.from(chunk).toString("utf8"); + }, + onStderr: (chunk) => { + stderr += Buffer.from(chunk).toString("utf8"); + }, + }); + const exitCode = await proc.wait(); + await new Promise((resolve) => setTimeout(resolve, 0)); + return { stdout, stderr, exitCode }; + } + + it("extracts fields from a file", async () => { + await mountFixture(); + + const result = await runAwk(["{print $1}", "/project/scores.txt"]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(lines(result.stdout)).toEqual(["Ada", "Grace", "Linus", "Barbara"]); + }); + + it("uses explicit field separators", async () => { + await mountFixture(); + + const result = await runAwk(["-F:", "{print $2}", "/project/colon.txt"]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(lines(result.stdout)).toEqual(["build", "test", "deploy"]); + }); + + it("aggregates numeric columns", async () => { + await mountFixture(); + + const result = await runAwk([ + "{sum += $3} END {print sum}", + "/project/scores.txt", + ]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(result.stdout.trim()).toBe("24"); + }); + + it("runs awk programs from files", async () => { + await mountFixture(); + + const result = await runAwk(["-f", "/project/runtime.awk", "/project/scores.txt"]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(lines(result.stdout)).toEqual(["Ada:7", "Linus:3", "done"]); + }); + + it("fails when an input file is missing", async () => { + await mountFixture(); + + const result = await runAwk(["{print $1}", "/project/missing.txt"]); + expect(result.exitCode).not.toBe(0); + expect(result.stderr).toContain("/project/missing.txt"); + }); +}); diff --git a/software/gawk/test/manifest.test.ts b/software/gawk/test/manifest.test.ts new file mode 100644 index 0000000000..89139d33b8 --- /dev/null +++ b/software/gawk/test/manifest.test.ts @@ -0,0 +1,19 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +const packageDir = new URL("..", import.meta.url).pathname; + +describe("package manifest", () => { + it("declares command binaries", () => { + const manifest = JSON.parse( + readFileSync(join(packageDir, "agentos-package.json"), "utf8"), + ); + + expect(manifest.commands?.length ?? 0).toBeGreaterThan(0); + for (const command of manifest.commands) { + expect(typeof command).toBe("string"); + expect(command.length).toBeGreaterThan(0); + } + }); +}); diff --git a/software/gawk/tsconfig.json b/software/gawk/tsconfig.json new file mode 100644 index 0000000000..03ce790ab7 --- /dev/null +++ b/software/gawk/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*"] +} diff --git a/software/git/agentos-package.json b/software/git/agentos-package.json new file mode 100644 index 0000000000..b1c4f9c536 --- /dev/null +++ b/software/git/agentos-package.json @@ -0,0 +1,19 @@ +{ + "commands": [ + "git", + "git-remote-http" + ], + "aliases": { + "git-remote-https": "git-remote-http", + "git-upload-pack": "git", + "git-receive-pack": "git", + "git-upload-archive": "git" + }, + "registry": { + "title": "git", + "description": "Upstream Git version control for AgentOS VMs.", + "priority": 90, + "image": "/images/registry/git.svg", + "category": "developer-tools" + } +} diff --git a/software/git/package.json b/software/git/package.json new file mode 100644 index 0000000000..c37a93b30f --- /dev/null +++ b/software/git/package.json @@ -0,0 +1,33 @@ +{ + "name": "@agentos-software/git", + "version": "0.3.3", + "type": "module", + "license": "GPL-2.0-only", + "description": "Upstream Git version control for AgentOS VMs", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "files": [ + "dist", + "!dist/package", + "!dist/package.tar" + ], + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "scripts": { + "build": "agentos-toolchain stage --commands-dir ../../toolchain/target/wasm32-wasip1/release/commands --if-missing skip && tsc && agentos-toolchain build", + "check-types": "tsc --noEmit", + "test": "vitest run test/ --passWithNoTests" + }, + "devDependencies": { + "@agentos-software/manifest": "workspace:*", + "@rivet-dev/agentos-toolchain": "workspace:*", + "@types/node": "^22.10.2", + "typescript": "^5.9.2", + "@agentos/test-harness": "workspace:*", + "vitest": "^2.1.9" + } +} diff --git a/registry/software/findutils/src/index.ts b/software/git/src/index.ts similarity index 100% rename from registry/software/findutils/src/index.ts rename to software/git/src/index.ts diff --git a/software/git/test/git.test.ts b/software/git/test/git.test.ts new file mode 100644 index 0000000000..f159618173 --- /dev/null +++ b/software/git/test/git.test.ts @@ -0,0 +1,719 @@ +/** + * Integration tests for git command. + * + * Verifies init, add, commit, branch, checkout (with DWIM), plus local and + * smart-HTTP remote clone via kernel.exec() with real WASM binaries. + */ + +import { describe, it, expect, afterEach, beforeAll, afterAll, vi } from 'vitest'; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { resolve, join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { createServer as createHttpsServer, type Server as HttpsServer } from 'node:https'; +import { spawn, spawnSync, execSync } from 'node:child_process'; +import { createWasmVmRuntime } from '@agentos/test-harness'; +import { + allowAll, + COMMANDS_DIR, + createInMemoryFileSystem, + createKernel, + describeIf, + hasWasmBinaries, +} from '@agentos/test-harness'; +import type { Kernel } from '@agentos/test-harness'; + +vi.setConfig({ testTimeout: 30_000 }); + +/** Check git binary exists in addition to base WASM binaries */ +const hasGit = hasWasmBinaries && existsSync(resolve(COMMANDS_DIR, 'git')); +const hasHostGit = spawnSync('git', ['--version'], { stdio: 'ignore' }).status === 0; +// Smart HTTP needs Git's libcurl-backed remote helper. It is now a real second +// WASM binary (git-remote-http links the overlaid mbedTLS libcurl in-process); +// git-remote-https aliases to it. +const hasGitHttpHelper = + hasGit && existsSync(resolve(COMMANDS_DIR, 'git-remote-http')); + +const gitConfig = [ + '-c safe.directory=*', + '-c init.defaultBranch=main', + '-c user.name=agentos', + '-c user.email=agentos@example.invalid', +].join(' '); + +function git(args: string) { + return `git ${gitConfig} ${args}`; +} + +/** Create a kernel with a world-writable in-memory filesystem */ +async function createGitKernel() { + const vfs = createInMemoryFileSystem(); + // Make root and /tmp writable by all users (WASM processes run as non-root) + await (vfs as any).chmod('/', 0o1777); + await vfs.mkdir('/tmp', { recursive: true }); + await (vfs as any).chmod('/tmp', 0o1777); + const kernel = createKernel({ filesystem: vfs, syncFilesystemOnDispose: false }); + await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); + return { kernel, vfs, dispose: () => kernel.dispose() }; +} + +async function createGitKernelWithNet(loopbackExemptPorts: number[], seededCaPem?: string) { + const vfs = createInMemoryFileSystem(); + await (vfs as any).chmod('/', 0o1777); + await vfs.mkdir('/tmp', { recursive: true }); + await (vfs as any).chmod('/tmp', 0o1777); + const kernel = createKernel({ + filesystem: vfs, + permissions: allowAll, + loopbackExemptPorts, + syncFilesystemOnDispose: false, + }); + await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); + // Seed the Debian-shaped trust store the way the native VM bootstrap does, so + // libcurl's compile-time default CA bundle (/etc/ssl/certs/ca-certificates.crt) + // resolves in-guest for git-remote-http's mbedTLS backend. + if (seededCaPem) { + await vfs.mkdir('/etc/ssl/certs', { recursive: true }); + await kernel.writeFile('/etc/ssl/certs/ca-certificates.crt', seededCaPem); + } + return { kernel, vfs, dispose: () => kernel.dispose() }; +} + +// Build a real CA and a leaf server certificate signed by it, with a SAN that +// covers the 127.0.0.1 loopback endpoint the VM connects to. This lets +// git-remote-http's mbedTLS backend perform genuine chain + hostname +// verification, exactly like Linux git against a private CA. +function makeCaSignedCert(caCommonName: string): { + caPem: string; + serverKey: string; + serverCert: string; +} { + const dir = mkdtempSync(join(tmpdir(), 'git-ca-')); + try { + execSync(`openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out "${dir}/ca.key" 2>/dev/null`); + execSync( + `openssl req -x509 -new -key "${dir}/ca.key" -days 3650 -subj "/CN=${caCommonName}" -out "${dir}/ca.crt" 2>/dev/null`, + ); + execSync(`openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out "${dir}/srv.key" 2>/dev/null`); + execSync(`openssl req -new -key "${dir}/srv.key" -subj "/CN=localhost" -out "${dir}/srv.csr" 2>/dev/null`); + writeFileSync(`${dir}/ext.cnf`, 'subjectAltName=DNS:localhost,IP:127.0.0.1\n'); + execSync( + `openssl x509 -req -in "${dir}/srv.csr" -CA "${dir}/ca.crt" -CAkey "${dir}/ca.key" ` + + `-CAcreateserial -days 3650 -extfile "${dir}/ext.cnf" -out "${dir}/srv.crt" 2>/dev/null`, + ); + return { + caPem: readFileSync(`${dir}/ca.crt`, 'utf8'), + serverKey: readFileSync(`${dir}/srv.key`, 'utf8'), + serverCert: readFileSync(`${dir}/srv.crt`, 'utf8'), + }; + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +function runHostGit(args: string[], cwd?: string) { + const result = spawnSync('git', args, { + cwd, + encoding: 'utf8', + }); + if (result.status !== 0) { + throw new Error( + `host git failed: git ${args.join(' ')}\nstdout: ${result.stdout}\nstderr: ${result.stderr}`, + ); + } +} + +/** Helper: run command and assert success */ +async function run(kernel: Kernel, cmd: string): Promise<{ stdout: string; stderr: string; exitCode: number }> { + const r = await kernel.exec(cmd); + if (r.exitCode !== 0) { + throw new Error(`Command failed (exit ${r.exitCode}): ${cmd}\nstdout: ${r.stdout}\nstderr: ${r.stderr}`); + } + return r; +} + +async function expectGitRef(kernel: Kernel, repo: string, ref: string) { + const result = await run(kernel, git(`-C ${repo} rev-parse --verify ${ref}`)); + expect(result.stdout.trim()).toMatch(/^[0-9a-f]{40,64}$/); +} + +// TODO(P6): requires git WASM artifact, intentionally excluded from the fast software-build gate. +describeIf(hasGit, 'git command', () => { + let kernel: Kernel; + let vfs: any; + let dispose: () => Promise; + + afterEach(async () => { + await dispose?.(); + }); + + it('init creates .git directory structure', async () => { + ({ kernel, vfs, dispose } = await createGitKernel()); + + const result = await run(kernel, git('init /repo')); + expect(result.stdout).toContain('Initialized empty Git repository'); + + expect(await vfs.exists('/repo/.git/HEAD')).toBe(true); + expect(await vfs.exists('/repo/.git/objects')).toBe(true); + expect(await vfs.exists('/repo/.git/refs/heads')).toBe(true); + + const head = new TextDecoder().decode(await vfs.readFile('/repo/.git/HEAD')); + expect(head.trim()).toBe('ref: refs/heads/main'); + }); + + it('add + commit creates objects and updates ref', async () => { + ({ kernel, vfs, dispose } = await createGitKernel()); + + await run(kernel, git('init /repo')); + await kernel.writeFile('/repo/hello.txt', 'hello world\n'); + await run(kernel, git('-C /repo add hello.txt')); + await run(kernel, git("-C /repo commit -m 'first commit'")); + + expect(await vfs.exists('/repo/.git/refs/heads/main')).toBe(true); + }); + + it('branch lists branches with current marked', async () => { + ({ kernel, vfs, dispose } = await createGitKernel()); + + await run(kernel, git('init /repo')); + await kernel.writeFile('/repo/file.txt', 'content\n'); + await run(kernel, git('-C /repo add file.txt')); + await run(kernel, git("-C /repo commit -m 'init'")); + + const result = await run(kernel, git('-C /repo branch')); + expect(result.stdout.trim()).toBe('* main'); + }); + + it('checkout -b creates a new branch', async () => { + ({ kernel, vfs, dispose } = await createGitKernel()); + + await run(kernel, git('init /repo')); + await kernel.writeFile('/repo/file.txt', 'content\n'); + await run(kernel, git('-C /repo add file.txt')); + await run(kernel, git("-C /repo commit -m 'init'")); + + await run(kernel, git('-C /repo checkout -b feature')); + + const result = await run(kernel, git('-C /repo branch')); + const lines = result.stdout.trim().split('\n').map((l: string) => l.trim()); + expect(lines).toContain('* feature'); + expect(lines).toContain('main'); + }); + + it('full quickstart scenario: init, commit, branch, clone, checkout', async () => { + ({ kernel, vfs, dispose } = await createGitKernel()); + + // Create origin repo + await run(kernel, git('init /tmp/origin')); + await kernel.writeFile('/tmp/origin/README.md', '# demo repo\n'); + await run(kernel, git('-C /tmp/origin add README.md')); + await run(kernel, git("-C /tmp/origin commit -m 'initial commit'")); + + // Check default branch + let r = await run(kernel, git('-C /tmp/origin branch')); + expect(r.stdout.trim()).toBe('* main'); + + // Create feature branch with a new file + await run(kernel, git('-C /tmp/origin checkout -b feature')); + await kernel.writeFile('/tmp/origin/feature.txt', 'checked out from feature\n'); + await run(kernel, git('-C /tmp/origin add feature.txt')); + await run(kernel, git("-C /tmp/origin commit -m 'add feature file'")); + + // Switch back to main + await run(kernel, git('-C /tmp/origin checkout main')); + + // Clone + await run(kernel, git('clone /tmp/origin /tmp/clone')); + + // Clone should only show main branch initially + r = await run(kernel, git('-C /tmp/clone branch')); + expect(r.stdout.trim()).toBe('* main'); + + // Checkout feature (DWIM from remote tracking) + await run(kernel, git('-C /tmp/clone checkout feature')); + + // Now both branches should be listed + r = await run(kernel, git('-C /tmp/clone branch')); + const branches = r.stdout.trim().split('\n').map((l: string) => l.trim()); + expect(branches).toContain('* feature'); + expect(branches).toContain('main'); + + // Verify feature file exists in clone + const featureContent = new TextDecoder().decode(await vfs.readFile('/tmp/clone/feature.txt')); + expect(featureContent).toBe('checked out from feature\n'); + + // Verify README exists too + const readmeContent = new TextDecoder().decode(await vfs.readFile('/tmp/clone/README.md')); + expect(readmeContent).toBe('# demo repo\n'); + }); + + it('clone without an explicit destination uses the source basename', async () => { + ({ kernel, vfs, dispose } = await createGitKernel()); + + await run(kernel, git('init /tmp/origin')); + await kernel.writeFile('/tmp/origin/README.md', 'default destination\n'); + await run(kernel, git('-C /tmp/origin add README.md')); + await run(kernel, git("-C /tmp/origin commit -m 'seed'")); + + await run(kernel, 'mkdir -p /work'); + await run(kernel, git('-C /work clone /tmp/origin')); + + expect(await vfs.exists('/work/origin/.git/HEAD')).toBe(true); + const readmeContent = new TextDecoder().decode(await vfs.readFile('/work/origin/README.md')); + expect(readmeContent).toBe('default destination\n'); + }); + + it('clone without an explicit destination strips a trailing .git suffix', async () => { + ({ kernel, vfs, dispose } = await createGitKernel()); + + await run(kernel, git('init /tmp/origin.git')); + await kernel.writeFile('/tmp/origin.git/README.md', 'suffix destination\n'); + await run(kernel, git('-C /tmp/origin.git add README.md')); + await run(kernel, git("-C /tmp/origin.git commit -m 'seed'")); + + await run(kernel, 'mkdir -p /work'); + await run(kernel, git('-C /work clone /tmp/origin.git')); + + expect(await vfs.exists('/work/origin/.git/HEAD')).toBe(true); + const readmeContent = new TextDecoder().decode(await vfs.readFile('/work/origin/README.md')); + expect(readmeContent).toBe('suffix destination\n'); + }); + + it('clone into an existing empty destination directory succeeds', async () => { + ({ kernel, vfs, dispose } = await createGitKernel()); + + await run(kernel, git('init /tmp/origin')); + await kernel.writeFile('/tmp/origin/README.md', 'empty destination\n'); + await run(kernel, git('-C /tmp/origin add README.md')); + await run(kernel, git("-C /tmp/origin commit -m 'seed'")); + + await run(kernel, 'mkdir -p /tmp/clone'); + await run(kernel, git('clone /tmp/origin /tmp/clone')); + + expect(await vfs.exists('/tmp/clone/.git/HEAD')).toBe(true); + const readmeContent = new TextDecoder().decode(await vfs.readFile('/tmp/clone/README.md')); + expect(readmeContent).toBe('empty destination\n'); + }); + + it('clone rejects a non-empty destination directory', async () => { + ({ kernel, vfs, dispose } = await createGitKernel()); + + await run(kernel, git('init /tmp/origin')); + await kernel.writeFile('/tmp/origin/README.md', 'origin\n'); + await run(kernel, git('-C /tmp/origin add README.md')); + await run(kernel, git("-C /tmp/origin commit -m 'seed'")); + + await run(kernel, 'mkdir -p /tmp/clone'); + await kernel.writeFile('/tmp/clone/existing.txt', 'keep me\n'); + + const result = await kernel.exec(git('clone /tmp/origin /tmp/clone')); + expect(result.exitCode).not.toBe(0); + expect(result.stderr).toMatch(/already exists|not an empty directory|destination/i); + + const existing = new TextDecoder().decode(await vfs.readFile('/tmp/clone/existing.txt')); + expect(existing).toBe('keep me\n'); + expect(await vfs.exists('/tmp/clone/.git')).toBe(false); + }); + + it('clone of a missing repository fails without leaving a partial destination', async () => { + ({ kernel, vfs, dispose } = await createGitKernel()); + + const result = await kernel.exec(git('clone /tmp/missing /tmp/clone')); + expect(result.exitCode).not.toBe(0); + expect(result.stderr).toMatch(/not a git repository|missing|no such file|fatal/i); + expect(await vfs.exists('/tmp/clone')).toBe(false); + }); + + it('clone of an empty repository succeeds and leaves an empty worktree', async () => { + ({ kernel, vfs, dispose } = await createGitKernel()); + + await run(kernel, git('init /tmp/origin')); + await run(kernel, git('clone /tmp/origin /tmp/clone')); + + const head = new TextDecoder().decode(await vfs.readFile('/tmp/clone/.git/HEAD')); + expect(head.trim()).toBe('ref: refs/heads/main'); + expect(await vfs.exists('/tmp/clone/.git/config')).toBe(true); + expect(await vfs.exists('/tmp/clone/.git/refs/heads/main')).toBe(false); + expect(await vfs.exists('/tmp/clone/README.md')).toBe(false); + }); + + it('clone preserves nested directory trees', async () => { + ({ kernel, vfs, dispose } = await createGitKernel()); + + await run(kernel, git('init /tmp/origin')); + await run(kernel, 'mkdir -p /tmp/origin/src/nested'); + await kernel.writeFile('/tmp/origin/src/nested/file.txt', 'nested payload\n'); + await kernel.writeFile('/tmp/origin/src/root.txt', 'root payload\n'); + await run(kernel, git('-C /tmp/origin add src/nested/file.txt src/root.txt')); + await run(kernel, git("-C /tmp/origin commit -m 'nested tree'")); + + await run(kernel, git('clone /tmp/origin /tmp/clone')); + + const nested = new TextDecoder().decode(await vfs.readFile('/tmp/clone/src/nested/file.txt')); + const root = new TextDecoder().decode(await vfs.readFile('/tmp/clone/src/root.txt')); + expect(nested).toBe('nested payload\n'); + expect(root).toBe('root payload\n'); + }); + + it('clone honors the source default branch when HEAD is not main', async () => { + ({ kernel, vfs, dispose } = await createGitKernel()); + + await run(kernel, git('init /tmp/origin')); + await kernel.writeFile('/tmp/origin/README.md', 'main branch\n'); + await run(kernel, git('-C /tmp/origin add README.md')); + await run(kernel, git("-C /tmp/origin commit -m 'main'")); + + await run(kernel, git('-C /tmp/origin checkout -b trunk')); + await kernel.writeFile('/tmp/origin/trunk.txt', 'trunk branch\n'); + await run(kernel, git('-C /tmp/origin add trunk.txt')); + await run(kernel, git("-C /tmp/origin commit -m 'trunk'")); + + await run(kernel, git('clone /tmp/origin /tmp/clone')); + + const head = new TextDecoder().decode(await vfs.readFile('/tmp/clone/.git/HEAD')); + expect(head.trim()).toBe('ref: refs/heads/trunk'); + expect(await vfs.exists('/tmp/clone/.git/refs/heads/trunk')).toBe(true); + const trunk = new TextDecoder().decode(await vfs.readFile('/tmp/clone/trunk.txt')); + expect(trunk).toBe('trunk branch\n'); + }); + + it('clone copies nested branch refs and checkout DWIM works for branch names with slashes', async () => { + ({ kernel, vfs, dispose } = await createGitKernel()); + + await run(kernel, git('init /tmp/origin')); + await kernel.writeFile('/tmp/origin/README.md', '# demo repo\n'); + await run(kernel, git('-C /tmp/origin add README.md')); + await run(kernel, git("-C /tmp/origin commit -m 'initial commit'")); + + await run(kernel, git('-C /tmp/origin checkout -b feature/deep')); + await kernel.writeFile('/tmp/origin/feature.txt', 'nested branch payload\n'); + await run(kernel, git('-C /tmp/origin add feature.txt')); + await run(kernel, git("-C /tmp/origin commit -m 'nested branch'")); + await run(kernel, git('-C /tmp/origin checkout main')); + + await run(kernel, git('clone /tmp/origin /tmp/clone')); + + await expectGitRef(kernel, '/tmp/clone', 'refs/remotes/origin/feature/deep'); + + await run(kernel, git('-C /tmp/clone checkout feature/deep')); + const featureContent = new TextDecoder().decode(await vfs.readFile('/tmp/clone/feature.txt')); + expect(featureContent).toBe('nested branch payload\n'); + const head = new TextDecoder().decode(await vfs.readFile('/tmp/clone/.git/HEAD')); + expect(head.trim()).toBe('ref: refs/heads/feature/deep'); + }); + + it('clone works with relative source and destination paths', async () => { + ({ kernel, vfs, dispose } = await createGitKernel()); + + await run(kernel, 'mkdir -p /tmp/work'); + await run(kernel, git('init /tmp/work/origin')); + await kernel.writeFile('/tmp/work/origin/README.md', 'relative clone\n'); + await run(kernel, git('-C /tmp/work/origin add README.md')); + await run(kernel, git("-C /tmp/work/origin commit -m 'seed'")); + + await run(kernel, git('-C /tmp/work clone ./origin ./clone')); + + expect(await vfs.exists('/tmp/work/clone/.git/HEAD')).toBe(true); + const readmeContent = new TextDecoder().decode(await vfs.readFile('/tmp/work/clone/README.md')); + expect(readmeContent).toBe('relative clone\n'); + }); + + it('push fails with a real Git remote/ref error', async () => { + ({ kernel, dispose } = await createGitKernel()); + + await run(kernel, git('init /tmp/repo')); + + const result = await kernel.exec(git('-C /tmp/repo push origin main')); + expect(result.exitCode).not.toBe(0); + expect(result.stderr).toMatch(/fatal|error|origin|refspec|repository/i); + expect(result.stderr).not.toContain('GitSubcommandUnsupported'); + }); + + it('clone rejects SSH-style remotes with a real Git spawn failure', async () => { + ({ kernel, dispose } = await createGitKernel()); + + const result = await kernel.exec(git('clone git@github.com:rivet-dev/agentos.git /tmp/clone')); + expect(result.exitCode).not.toBe(0); + expect(result.stderr).toMatch(/cannot run ssh|unable to fork|ssh|fatal/i); + expect(result.stderr).not.toContain('GitSubcommandUnsupported'); + }); + + // Real smart-HTTP over TLS: git-remote-http (libcurl + in-guest mbedTLS) + // clones/fetches/pushes against `git http-backend` behind a Node HTTPS + // endpoint. Certificate trust comes from a private CA seeded into the guest's + // /etc/ssl/certs bundle (the "trusted" server) — exactly the Debian trust + // path — while a second server signed by a CA absent from the bundle exercises + // verify-fail, http.sslVerify=false, GIT_SSL_NO_VERIFY, and http.sslCAInfo. + describeIf(hasHostGit && hasGitHttpHelper, 'smart-HTTP clone/fetch/push over TLS', () => { + let repoRoot: string; + let trustedServer: HttpsServer; + let untrustedServer: HttpsServer; + let trustedPort: number; + let untrustedPort: number; + let trustedCaPem = ''; + let untrustedCaPem = ''; + + // A CGI bridge to `git http-backend`. receive-pack is enabled on the origin + // (below) so pushes are accepted; GIT_HTTP_EXPORT_ALL allows anonymous read. + function makeBackendHandler() { + return (req: import('node:http').IncomingMessage, res: import('node:http').ServerResponse) => { + const url = new URL(req.url ?? '/', 'https://127.0.0.1'); + const bodyChunks: Buffer[] = []; + req.on('data', (chunk) => { + bodyChunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + }); + req.on('end', () => { + const requestBody = Buffer.concat(bodyChunks); + const gitProtocol = req.headers['git-protocol']; + const env = { + ...process.env, + GIT_HTTP_EXPORT_ALL: '1', + GIT_PROJECT_ROOT: repoRoot, + PATH_INFO: url.pathname, + QUERY_STRING: url.search.startsWith('?') ? url.search.slice(1) : url.search, + REQUEST_METHOD: req.method ?? 'GET', + CONTENT_TYPE: String(req.headers['content-type'] ?? ''), + CONTENT_LENGTH: String(requestBody.length), + REMOTE_ADDR: '127.0.0.1', + GIT_PROTOCOL: typeof gitProtocol === 'string' ? gitProtocol : '', + HTTP_GIT_PROTOCOL: typeof gitProtocol === 'string' ? gitProtocol : '', + }; + + const child = spawn('git', ['http-backend'], { env }); + const stdout: Buffer[] = []; + const stderr: Buffer[] = []; + child.stdout.on('data', (chunk) => { + stdout.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + }); + child.stderr.on('data', (chunk) => { + stderr.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + }); + child.on('error', (error) => { + res.writeHead(500, { 'Content-Type': 'text/plain' }); + res.end(String(error)); + }); + child.on('close', (code) => { + const output = Buffer.concat(stdout); + const headerSep = output.indexOf(Buffer.from('\r\n\r\n')); + const altSep = output.indexOf(Buffer.from('\n\n')); + const sepIndex = headerSep >= 0 ? headerSep : altSep; + const sepLen = headerSep >= 0 ? 4 : altSep >= 0 ? 2 : 0; + if (code !== 0 && sepIndex === -1) { + res.writeHead(500, { 'Content-Type': 'text/plain' }); + res.end(Buffer.concat(stderr)); + return; + } + if (sepIndex === -1) { + res.writeHead(500, { 'Content-Type': 'text/plain' }); + res.end(output); + return; + } + const headerText = output.subarray(0, sepIndex).toString('utf8'); + const responseBody = output.subarray(sepIndex + sepLen); + let status = 200; + const headers: Record = {}; + for (const line of headerText.split(/\r?\n/)) { + if (!line) continue; + const colon = line.indexOf(':'); + if (colon === -1) continue; + const name = line.slice(0, colon); + const value = line.slice(colon + 1).trim(); + if (name.toLowerCase() === 'status') { + status = Number.parseInt(value, 10) || 200; + } else { + headers[name] = value; + } + } + res.writeHead(status, headers); + res.end(responseBody); + }); + child.stdin.end(requestBody); + }); + }; + } + + async function listen(server: HttpsServer): Promise { + await new Promise((r) => server.listen(0, '127.0.0.1', r)); + return (server.address() as import('node:net').AddressInfo).port; + } + + beforeAll(async () => { + repoRoot = mkdtempSync(join(tmpdir(), 'agentos-git-https-')); + const worktree = join(repoRoot, 'worktree'); + const origin = join(repoRoot, 'origin.git'); + + runHostGit(['-c', 'init.defaultBranch=main', 'init', worktree]); + writeFileSync(join(worktree, 'README.md'), 'remote smart clone\n'); + runHostGit(['-C', worktree, 'add', 'README.md']); + runHostGit([ + '-C', worktree, + '-c', 'user.name=secure-exec', '-c', 'user.email=agent@example.com', + 'commit', '-m', 'seed', + ]); + runHostGit(['-C', worktree, 'checkout', '-b', 'feature/deep']); + writeFileSync(join(worktree, 'feature.txt'), 'remote branch payload\n'); + runHostGit(['-C', worktree, 'add', 'feature.txt']); + runHostGit([ + '-C', worktree, + '-c', 'user.name=secure-exec', '-c', 'user.email=agent@example.com', + 'commit', '-m', 'feature branch', + ]); + runHostGit(['-C', worktree, 'checkout', 'main']); + runHostGit(['clone', '--bare', worktree, origin]); + runHostGit(['-C', origin, 'repack', '-a', '-d', '-f', '--depth=50', '--window=50']); + // Accept anonymous pushes over smart HTTP. + runHostGit(['-C', origin, 'config', 'http.receivepack', 'true']); + + const trusted = makeCaSignedCert('AgentOS Git Test Root CA'); + trustedCaPem = trusted.caPem; + trustedServer = createHttpsServer( + { key: trusted.serverKey, cert: trusted.serverCert }, + makeBackendHandler(), + ); + trustedPort = await listen(trustedServer); + + const untrusted = makeCaSignedCert('AgentOS Git Untrusted CA'); + untrustedCaPem = untrusted.caPem; + untrustedServer = createHttpsServer( + { key: untrusted.serverKey, cert: untrusted.serverCert }, + makeBackendHandler(), + ); + untrustedPort = await listen(untrustedServer); + }); + + afterAll(async () => { + if (trustedServer) await new Promise((r) => trustedServer.close(() => r())); + if (untrustedServer) await new Promise((r) => untrustedServer.close(() => r())); + rmSync(repoRoot, { recursive: true, force: true }); + }); + + const trustedUrl = () => `https://127.0.0.1:${trustedPort}/origin.git`; + const untrustedUrl = () => `https://127.0.0.1:${untrustedPort}/origin.git`; + + it('clone fetches refs and worktree contents over HTTPS with a trusted CA', async () => { + ({ kernel, vfs, dispose } = await createGitKernelWithNet([trustedPort], trustedCaPem)); + + const res = await kernel.exec(git(`clone ${trustedUrl()} /tmp/clone`), { + env: { GIT_CURL_VERBOSE: '1' }, + }); + expect(res.exitCode).toBe(0); + // Proof a real TLS handshake happened in-guest (mbedTLS via libcurl). + expect(res.stderr).toMatch(/SSL connection|TLS|SSL certificate|CAfile/i); + + const head = new TextDecoder().decode(await kernel.readFile('/tmp/clone/.git/HEAD')); + expect(head.trim()).toBe('ref: refs/heads/main'); + const readme = new TextDecoder().decode(await kernel.readFile('/tmp/clone/README.md')); + expect(readme).toBe('remote smart clone\n'); + await expectGitRef(kernel, '/tmp/clone', 'refs/remotes/origin/feature/deep'); + + await run(kernel, git('-C /tmp/clone checkout feature/deep')); + const feature = new TextDecoder().decode(await kernel.readFile('/tmp/clone/feature.txt')); + expect(feature).toBe('remote branch payload\n'); + }); + + it('fetch picks up a new remote branch over HTTPS', async () => { + ({ kernel, vfs, dispose } = await createGitKernelWithNet([trustedPort], trustedCaPem)); + await run(kernel, git(`clone ${trustedUrl()} /tmp/clone`)); + + // Add a new branch on the origin (host side), then fetch it in-guest. + const bareBranch = 'fetched-branch'; + runHostGit(['-C', join(repoRoot, 'origin.git'), 'branch', bareBranch, 'main']); + + await run(kernel, git('-C /tmp/clone fetch origin')); + await expectGitRef(kernel, '/tmp/clone', `refs/remotes/origin/${bareBranch}`); + }); + + it('push sends a small commit over HTTPS smart-HTTP', async () => { + ({ kernel, vfs, dispose } = await createGitKernelWithNet([trustedPort], trustedCaPem)); + await run(kernel, git(`clone ${trustedUrl()} /tmp/clone`)); + + await kernel.writeFile('/tmp/clone/pushed.txt', 'pushed over https\n'); + await run(kernel, git('-C /tmp/clone add pushed.txt')); + await run(kernel, git("-C /tmp/clone commit -m 'push small'")); + const pushed = await kernel.exec(git('-C /tmp/clone push origin HEAD:refs/heads/small-push')); + expect(pushed.exitCode).toBe(0); + + // Verify the ref really landed in the origin bare repo (host side). + const originRef = spawnSync( + 'git', + ['-C', join(repoRoot, 'origin.git'), 'rev-parse', '--verify', 'refs/heads/small-push'], + { encoding: 'utf8' }, + ); + expect(originRef.status).toBe(0); + expect(originRef.stdout.trim()).toMatch(/^[0-9a-f]{40,64}$/); + }); + + it('push streams a >1 MiB commit over HTTPS (chunked POST)', async () => { + ({ kernel, vfs, dispose } = await createGitKernelWithNet([trustedPort], trustedCaPem)); + await run(kernel, git(`clone ${trustedUrl()} /tmp/clone`)); + + // Incompressible >1 MiB payload so the pack exceeds http.postBuffer (1 MiB) + // and libcurl must use chunked Transfer-Encoding + Expect: 100-continue. + const { randomBytes } = await import('node:crypto'); + const big = randomBytes(2 * 1024 * 1024); + await kernel.writeFile('/tmp/clone/big.bin', big); + await run(kernel, git('-C /tmp/clone add big.bin')); + await run(kernel, git("-C /tmp/clone commit -m 'push large'")); + const pushed = await kernel.exec(git('-C /tmp/clone push origin HEAD:refs/heads/large-push')); + expect(pushed.exitCode).toBe(0); + + const originRef = spawnSync( + 'git', + ['-C', join(repoRoot, 'origin.git'), 'rev-parse', '--verify', 'refs/heads/large-push'], + { encoding: 'utf8' }, + ); + expect(originRef.status).toBe(0); + // Confirm the large object is actually present in the origin object store. + const cat = spawnSync( + 'git', + ['-C', join(repoRoot, 'origin.git'), 'cat-file', '-s', `${originRef.stdout.trim()}^{tree}`], + { encoding: 'utf8' }, + ); + expect(cat.status).toBe(0); + }); + + it('clone fails with a real certificate-verification error on an untrusted CA', async () => { + // Only the trusted CA is seeded; the untrusted server's CA is absent. + ({ kernel, vfs, dispose } = await createGitKernelWithNet([untrustedPort], trustedCaPem)); + + const res = await kernel.exec(git(`clone ${untrustedUrl()} /tmp/clone`)); + expect(res.exitCode).not.toBe(0); + expect(res.stderr).toMatch(/certificate|SSL|TLS|verify|CAfile|unable to (access|get local)/i); + expect(res.stderr).not.toContain('GitSubcommandUnsupported'); + expect(await vfs.exists('/tmp/clone/.git')).toBe(false); + }); + + it('http.sslVerify=false bypasses verification for an untrusted CA', async () => { + ({ kernel, vfs, dispose } = await createGitKernelWithNet([untrustedPort], trustedCaPem)); + + const res = await kernel.exec(git(`-c http.sslVerify=false clone ${untrustedUrl()} /tmp/clone`)); + expect(res.exitCode).toBe(0); + const readme = new TextDecoder().decode(await kernel.readFile('/tmp/clone/README.md')); + expect(readme).toBe('remote smart clone\n'); + }); + + it('GIT_SSL_NO_VERIFY bypasses verification for an untrusted CA', async () => { + ({ kernel, vfs, dispose } = await createGitKernelWithNet([untrustedPort], trustedCaPem)); + + const res = await kernel.exec(git(`clone ${untrustedUrl()} /tmp/clone`), { + env: { GIT_SSL_NO_VERIFY: '1' }, + }); + expect(res.exitCode).toBe(0); + expect(await vfs.exists('/tmp/clone/.git/HEAD')).toBe(true); + }); + + it('http.sslCAInfo trusts an explicitly supplied CA bundle', async () => { + // Seed only the trusted CA in the default bundle; supply the untrusted + // server's CA via a VFS file referenced with http.sslCAInfo. + ({ kernel, vfs, dispose } = await createGitKernelWithNet([untrustedPort], trustedCaPem)); + await vfs.mkdir('/tmp/ca', { recursive: true }); + await kernel.writeFile('/tmp/ca/untrusted.pem', untrustedCaPem); + + const res = await kernel.exec( + git(`-c http.sslCAInfo=/tmp/ca/untrusted.pem clone ${untrustedUrl()} /tmp/clone`), + ); + expect(res.exitCode).toBe(0); + const readme = new TextDecoder().decode(await kernel.readFile('/tmp/clone/README.md')); + expect(readme).toBe('remote smart clone\n'); + }); + }); +}); diff --git a/software/git/tsconfig.json b/software/git/tsconfig.json new file mode 100644 index 0000000000..03ce790ab7 --- /dev/null +++ b/software/git/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*"] +} diff --git a/software/grep/agentos-package.json b/software/grep/agentos-package.json new file mode 100644 index 0000000000..2b14ba7c37 --- /dev/null +++ b/software/grep/agentos-package.json @@ -0,0 +1,14 @@ +{ + "commands": [ + "grep", + "egrep", + "fgrep" + ], + "registry": { + "title": "grep", + "description": "GNU grep pattern matching (grep, egrep, fgrep).", + "priority": 40, + "image": "/images/registry/grep.svg", + "category": "core" + } +} diff --git a/software/grep/native/crates/cmd-egrep/Cargo.toml b/software/grep/native/crates/cmd-egrep/Cargo.toml new file mode 100644 index 0000000000..b3eb4c9222 --- /dev/null +++ b/software/grep/native/crates/cmd-egrep/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-egrep" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "egrep launcher for GNU grep" + +[[bin]] +name = "egrep" +path = "src/main.rs" + +[dependencies] +shims = { package = "secureexec-shims", path = "../../../../../toolchain/crates/libs/shims" } diff --git a/software/grep/native/crates/cmd-egrep/src/main.rs b/software/grep/native/crates/cmd-egrep/src/main.rs new file mode 100644 index 0000000000..b6671b6c38 --- /dev/null +++ b/software/grep/native/crates/cmd-egrep/src/main.rs @@ -0,0 +1,4 @@ +fn main() { + let args: Vec = std::env::args_os().collect(); + std::process::exit(shims::grep_alias::egrep(args)); +} diff --git a/software/grep/native/crates/cmd-fgrep/Cargo.toml b/software/grep/native/crates/cmd-fgrep/Cargo.toml new file mode 100644 index 0000000000..573b6b100e --- /dev/null +++ b/software/grep/native/crates/cmd-fgrep/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-fgrep" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "fgrep launcher for GNU grep" + +[[bin]] +name = "fgrep" +path = "src/main.rs" + +[dependencies] +shims = { package = "secureexec-shims", path = "../../../../../toolchain/crates/libs/shims" } diff --git a/software/grep/native/crates/cmd-fgrep/src/main.rs b/software/grep/native/crates/cmd-fgrep/src/main.rs new file mode 100644 index 0000000000..3f81c7a4d5 --- /dev/null +++ b/software/grep/native/crates/cmd-fgrep/src/main.rs @@ -0,0 +1,4 @@ +fn main() { + let args: Vec = std::env::args_os().collect(); + std::process::exit(shims::grep_alias::fgrep(args)); +} diff --git a/software/grep/package.json b/software/grep/package.json new file mode 100644 index 0000000000..ed30841797 --- /dev/null +++ b/software/grep/package.json @@ -0,0 +1,33 @@ +{ + "name": "@agentos-software/grep", + "version": "0.3.3", + "type": "module", + "license": "Apache-2.0", + "description": "GNU grep pattern matching for secure-exec VMs (grep, egrep, fgrep)", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "files": [ + "dist", + "!dist/package", + "!dist/package.tar" + ], + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "scripts": { + "build": "agentos-toolchain stage --commands-dir ../../toolchain/target/wasm32-wasip1/release/commands --if-missing skip && tsc && agentos-toolchain build", + "check-types": "tsc --noEmit", + "test": "vitest run test/ --passWithNoTests" + }, + "devDependencies": { + "@agentos-software/manifest": "workspace:*", + "@rivet-dev/agentos-toolchain": "workspace:*", + "@types/node": "^22.10.2", + "typescript": "^5.9.2", + "@agentos/test-harness": "workspace:*", + "vitest": "^2.1.9" + } +} diff --git a/registry/software/gawk/src/index.ts b/software/grep/src/index.ts similarity index 100% rename from registry/software/gawk/src/index.ts rename to software/grep/src/index.ts diff --git a/software/grep/test/grep.test.ts b/software/grep/test/grep.test.ts new file mode 100644 index 0000000000..1816f9cbcb --- /dev/null +++ b/software/grep/test/grep.test.ts @@ -0,0 +1,125 @@ +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { createWasmVmRuntime } from "@agentos/test-harness"; +import { + C_BUILD_DIR, + COMMANDS_DIR, + NodeFileSystem, + createKernel, + describeIf, + hasCWasmBinaries, + hasWasmBinaries, +} from "@agentos/test-harness"; +import type { Kernel } from "@agentos/test-harness"; +import { afterEach, describe, expect, it } from "vitest"; + +let tempRoot: string | undefined; + +async function createTestVFS(): Promise { + tempRoot = await mkdtemp(join(tmpdir(), "agentos-grep-")); + await writeFixture( + "/project/notes.txt", + ["Alpha", "beta", "alphabet", "delta"].join("\n") + "\n", + ); + await writeFixture( + "/project/other.txt", + ["gamma", "Beta blocker", "literal a+b"].join("\n") + "\n", + ); + return new NodeFileSystem({ root: tempRoot }); +} + +async function writeFixture(path: string, contents: string): Promise { + if (!tempRoot) throw new Error("fixture root not initialized"); + const hostPath = join(tempRoot, path.replace(/^\/+/, "")); + await mkdir(dirname(hostPath), { recursive: true }); + await writeFile(hostPath, contents); +} + +describeIf( + hasWasmBinaries && hasCWasmBinaries("grep"), + "GNU grep command", + { timeout: 10_000 }, + () => { + let kernel: Kernel; + + afterEach(async () => { + await kernel?.dispose(); + if (tempRoot) { + await rm(tempRoot, { recursive: true, force: true }); + tempRoot = undefined; + } + }); + + async function mountFixture(): Promise { + const vfs = await createTestVFS(); + kernel = createKernel({ filesystem: vfs }); + await kernel.mount( + createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] }), + ); + return vfs; + } + + it("reports the upstream GNU grep version", async () => { + await mountFixture(); + + const result = await kernel.exec("grep --version", {}); + expect(result.stdout).toContain("GNU grep 3.12"); + }); + + it("prints matching lines from a file", async () => { + await mountFixture(); + + const result = await kernel.exec("grep alpha /project/notes.txt", {}); + expect(result.stdout).toBe("alphabet\n"); + }); + + it("supports case-insensitive search", async () => { + await mountFixture(); + + const result = await kernel.exec("grep -i beta /project/notes.txt", {}); + expect(result.stdout).toBe("beta\n"); + }); + + it("supports inverted matches", async () => { + await mountFixture(); + + const result = await kernel.exec("grep -v Alpha /project/notes.txt", {}); + expect(result.stdout).toBe("beta\nalphabet\ndelta\n"); + }); + + it("counts matches", async () => { + await mountFixture(); + + const result = await kernel.exec("grep -c a /project/notes.txt", {}); + expect(result.stdout).toBe("4\n"); + }); + + it("prints file names with matches", async () => { + await mountFixture(); + + const result = await kernel.exec( + "grep -l gamma /project/notes.txt /project/other.txt", + {}, + ); + expect(result.stdout).toBe("/project/other.txt\n"); + }); + + it("supports egrep extended regex alias", async () => { + await mountFixture(); + + const result = await kernel.exec( + "egrep 'Alpha|delta' /project/notes.txt", + {}, + ); + expect(result.stdout).toBe("Alpha\ndelta\n"); + }); + + it("supports fgrep fixed-string alias", async () => { + await mountFixture(); + + const result = await kernel.exec("fgrep 'a+b' /project/other.txt", {}); + expect(result.stdout).toBe("literal a+b\n"); + }); + }, +); diff --git a/software/grep/test/manifest.test.ts b/software/grep/test/manifest.test.ts new file mode 100644 index 0000000000..89139d33b8 --- /dev/null +++ b/software/grep/test/manifest.test.ts @@ -0,0 +1,19 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +const packageDir = new URL("..", import.meta.url).pathname; + +describe("package manifest", () => { + it("declares command binaries", () => { + const manifest = JSON.parse( + readFileSync(join(packageDir, "agentos-package.json"), "utf8"), + ); + + expect(manifest.commands?.length ?? 0).toBeGreaterThan(0); + for (const command of manifest.commands) { + expect(typeof command).toBe("string"); + expect(command.length).toBeGreaterThan(0); + } + }); +}); diff --git a/software/grep/tsconfig.json b/software/grep/tsconfig.json new file mode 100644 index 0000000000..03ce790ab7 --- /dev/null +++ b/software/grep/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*"] +} diff --git a/software/gzip/agentos-package.json b/software/gzip/agentos-package.json new file mode 100644 index 0000000000..33ceeb2f6e --- /dev/null +++ b/software/gzip/agentos-package.json @@ -0,0 +1,16 @@ +{ + "commands": [ + "gzip" + ], + "aliases": { + "gunzip": "gzip", + "zcat": "gzip" + }, + "registry": { + "title": "gzip", + "description": "GNU gzip compression (gzip, gunzip, zcat).", + "priority": 8, + "image": "/images/registry/gzip.svg", + "category": "core" + } +} diff --git a/software/gzip/native/crates/cmd-gzip/Cargo.toml b/software/gzip/native/crates/cmd-gzip/Cargo.toml new file mode 100644 index 0000000000..dafe689b6d --- /dev/null +++ b/software/gzip/native/crates/cmd-gzip/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-gzip" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "gzip standalone binary for secure-exec VM" + +[[bin]] +name = "gzip" +path = "src/main.rs" + +[dependencies] +secureexec-gzip = { path = "../gzip" } diff --git a/registry/native/crates/commands/gzip/src/main.rs b/software/gzip/native/crates/cmd-gzip/src/main.rs similarity index 100% rename from registry/native/crates/commands/gzip/src/main.rs rename to software/gzip/native/crates/cmd-gzip/src/main.rs diff --git a/software/gzip/native/crates/gzip/Cargo.toml b/software/gzip/native/crates/gzip/Cargo.toml new file mode 100644 index 0000000000..897731b6d4 --- /dev/null +++ b/software/gzip/native/crates/gzip/Cargo.toml @@ -0,0 +1,10 @@ +[package] +workspace = "../../../../../toolchain" +name = "secureexec-gzip" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "gzip/gunzip/zcat implementations for secure-exec standalone binaries" + +[dependencies] +flate2 = { version = "1.0", default-features = false, features = ["rust_backend"] } diff --git a/registry/native/crates/libs/gzip/src/lib.rs b/software/gzip/native/crates/gzip/src/lib.rs similarity index 100% rename from registry/native/crates/libs/gzip/src/lib.rs rename to software/gzip/native/crates/gzip/src/lib.rs diff --git a/software/gzip/package.json b/software/gzip/package.json new file mode 100644 index 0000000000..f35d2a16a1 --- /dev/null +++ b/software/gzip/package.json @@ -0,0 +1,33 @@ +{ + "name": "@agentos-software/gzip", + "version": "0.3.3", + "type": "module", + "license": "Apache-2.0", + "description": "GNU gzip compression for secure-exec VMs (gzip, gunzip, zcat)", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "files": [ + "dist", + "!dist/package", + "!dist/package.tar" + ], + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "scripts": { + "build": "agentos-toolchain stage --commands-dir ../../toolchain/target/wasm32-wasip1/release/commands --if-missing skip && tsc && agentos-toolchain build", + "check-types": "tsc --noEmit", + "test": "vitest run test/ --passWithNoTests" + }, + "devDependencies": { + "@agentos-software/manifest": "workspace:*", + "@rivet-dev/agentos-toolchain": "workspace:*", + "@types/node": "^22.10.2", + "typescript": "^5.9.2", + "@agentos/test-harness": "workspace:*", + "vitest": "^2.1.9" + } +} diff --git a/registry/software/git/src/index.ts b/software/gzip/src/index.ts similarity index 100% rename from registry/software/git/src/index.ts rename to software/gzip/src/index.ts diff --git a/software/gzip/test/gzip.test.ts b/software/gzip/test/gzip.test.ts new file mode 100644 index 0000000000..477499a35a --- /dev/null +++ b/software/gzip/test/gzip.test.ts @@ -0,0 +1,138 @@ +import { existsSync } from "node:fs"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { + NodeFileSystem, + createKernel, + createWasmVmRuntime, + describeIf, +} from "@agentos/test-harness"; +import type { Kernel } from "@agentos/test-harness"; +import { afterEach, expect, it } from "vitest"; + +const GZIP_COMMAND_DIR = fileURLToPath(new URL("../bin", import.meta.url)); +const hasGzipPackageBinary = existsSync(join(GZIP_COMMAND_DIR, "gzip")); + +let tempRoot: string | undefined; + +async function writeFixture(path: string, contents: string): Promise { + if (!tempRoot) throw new Error("fixture root not initialized"); + const hostPath = join(tempRoot, path.replace(/^\/+/, "")); + await mkdir(dirname(hostPath), { recursive: true }); + await writeFile(hostPath, contents); +} + +async function createTestVFS(): Promise { + tempRoot = await mkdtemp(join(tmpdir(), "agentos-gzip-")); + await writeFixture("/project/report.txt", "alpha\nbeta\ngamma\n"); + await writeFixture("/project/remove.txt", "temporary payload\n"); + return new NodeFileSystem({ root: tempRoot }); +} + +const textDecoder = new TextDecoder(); + +describeIf(hasGzipPackageBinary, "gzip command", { timeout: 10_000 }, () => { + let kernel: Kernel | undefined; + + afterEach(async () => { + await kernel?.dispose(); + kernel = undefined; + if (tempRoot) { + await rm(tempRoot, { recursive: true, force: true }); + tempRoot = undefined; + } + }); + + async function mountFixture(): Promise { + const vfs = await createTestVFS(); + kernel = createKernel({ filesystem: vfs }); + await kernel.mount(createWasmVmRuntime({ commandDirs: [GZIP_COMMAND_DIR] })); + } + + async function runCommand(command: string, args: string[]) { + if (!kernel) throw new Error("kernel not mounted"); + let stdout = ""; + let stderr = ""; + const proc = kernel.spawn(command, args, { + onStdout: (chunk) => { + stdout += Buffer.from(chunk).toString("utf8"); + }, + onStderr: (chunk) => { + stderr += Buffer.from(chunk).toString("utf8"); + }, + }); + const exitCode = await proc.wait(); + await new Promise((resolve) => setTimeout(resolve, 0)); + return { stdout, stderr, exitCode }; + } + + async function readGuestText(path: string): Promise { + if (!kernel) throw new Error("kernel not mounted"); + return textDecoder.decode(await kernel.readFile(path)); + } + + it("compresses files while keeping originals with -k", async () => { + await mountFixture(); + + const result = await runCommand("gzip", ["-k", "/project/report.txt"]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + await expect(readGuestText("/project/report.txt")).resolves.toBe( + "alpha\nbeta\ngamma\n", + ); + + const decompressed = await runCommand("gunzip", [ + "-c", + "/project/report.txt.gz", + ]); + expect(decompressed.exitCode, decompressed.stderr || decompressed.stdout).toBe(0); + expect(decompressed.stdout).toBe("alpha\nbeta\ngamma\n"); + }); + + it("removes the source file unless -k is set", async () => { + await mountFixture(); + + const result = await runCommand("gzip", ["/project/remove.txt"]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + await expect(readGuestText("/project/remove.txt")).rejects.toThrow(); + await expect(readGuestText("/project/remove.txt.gz")).resolves.toBeTruthy(); + }); + + it("decompresses files with gunzip -k", async () => { + await mountFixture(); + const compressed = await runCommand("gzip", ["-k", "/project/report.txt"]); + expect(compressed.exitCode, compressed.stderr || compressed.stdout).toBe(0); + await writeFixture("/project/report.txt", "replacement\n"); + + const result = await runCommand("gunzip", [ + "-fk", + "/project/report.txt.gz", + ]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + await expect(readGuestText("/project/report.txt")).resolves.toBe( + "alpha\nbeta\ngamma\n", + ); + await expect(readGuestText("/project/report.txt.gz")).resolves.toBeTruthy(); + }); + + it("streams decompressed content with zcat", async () => { + await mountFixture(); + const compressed = await runCommand("gzip", ["-k", "/project/report.txt"]); + expect(compressed.exitCode, compressed.stderr || compressed.stdout).toBe(0); + + const result = await runCommand("zcat", ["/project/report.txt.gz"]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(result.stdout).toBe("alpha\nbeta\ngamma\n"); + }); + + it("fails instead of overwriting compressed outputs without -f", async () => { + await mountFixture(); + const first = await runCommand("gzip", ["-k", "/project/report.txt"]); + expect(first.exitCode, first.stderr || first.stdout).toBe(0); + + const second = await runCommand("gzip", ["-k", "/project/report.txt"]); + expect(second.exitCode).not.toBe(0); + expect(second.stderr).toContain("already exists"); + }); +}); diff --git a/software/gzip/test/manifest.test.ts b/software/gzip/test/manifest.test.ts new file mode 100644 index 0000000000..89139d33b8 --- /dev/null +++ b/software/gzip/test/manifest.test.ts @@ -0,0 +1,19 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +const packageDir = new URL("..", import.meta.url).pathname; + +describe("package manifest", () => { + it("declares command binaries", () => { + const manifest = JSON.parse( + readFileSync(join(packageDir, "agentos-package.json"), "utf8"), + ); + + expect(manifest.commands?.length ?? 0).toBeGreaterThan(0); + for (const command of manifest.commands) { + expect(typeof command).toBe("string"); + expect(command.length).toBeGreaterThan(0); + } + }); +}); diff --git a/software/gzip/tsconfig.json b/software/gzip/tsconfig.json new file mode 100644 index 0000000000..03ce790ab7 --- /dev/null +++ b/software/gzip/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*"] +} diff --git a/software/jq/agentos-package.json b/software/jq/agentos-package.json new file mode 100644 index 0000000000..43e0a6f3c8 --- /dev/null +++ b/software/jq/agentos-package.json @@ -0,0 +1,12 @@ +{ + "commands": [ + "jq" + ], + "registry": { + "title": "jq", + "description": "Lightweight JSON processor.", + "priority": 80, + "image": "/images/registry/jq.svg", + "category": "data" + } +} diff --git a/software/jq/native/crates/cmd-jq/Cargo.toml b/software/jq/native/crates/cmd-jq/Cargo.toml new file mode 100644 index 0000000000..bdc904ce10 --- /dev/null +++ b/software/jq/native/crates/cmd-jq/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-jq" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "jq standalone binary for secure-exec VM" + +[[bin]] +name = "jq" +path = "src/main.rs" + +[dependencies] +secureexec-jq = { path = "../jq" } diff --git a/registry/native/crates/commands/jq/src/main.rs b/software/jq/native/crates/cmd-jq/src/main.rs similarity index 100% rename from registry/native/crates/commands/jq/src/main.rs rename to software/jq/native/crates/cmd-jq/src/main.rs diff --git a/software/jq/native/crates/jq/Cargo.toml b/software/jq/native/crates/jq/Cargo.toml new file mode 100644 index 0000000000..6a86e30e47 --- /dev/null +++ b/software/jq/native/crates/jq/Cargo.toml @@ -0,0 +1,13 @@ +[package] +workspace = "../../../../../toolchain" +name = "secureexec-jq" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "jq implementation for secure-exec standalone binaries" + +[dependencies] +jaq-core = "2.2" +jaq-std = "2.1" +jaq-json = { version = "1.1", features = ["serde_json"] } +serde_json = "1" diff --git a/software/jq/native/crates/jq/src/lib.rs b/software/jq/native/crates/jq/src/lib.rs new file mode 100644 index 0000000000..66fc2544cc --- /dev/null +++ b/software/jq/native/crates/jq/src/lib.rs @@ -0,0 +1,349 @@ +//! jq implementation using the jaq crate (pure Rust, jq-compatible). +//! +//! Wraps jaq-core/jaq-std/jaq-json to provide a standard jq CLI interface. + +use std::ffi::OsString; +use std::fs::File as FsFile; +use std::io::{self, Read, Write}; + +use jaq_core::load::{Arena, File, Loader}; +use jaq_core::{Compiler, Ctx, RcIter}; +use jaq_json::Val; + +const MAX_INPUT_BYTES: usize = 16 * 1024 * 1024; +const MAX_INPUT_VALUES: usize = 100_000; + +/// Entry point for jq command. +pub fn main(args: Vec) -> i32 { + let str_args: Vec = args + .iter() + .skip(1) // skip argv[0] + .map(|a| a.to_string_lossy().to_string()) + .collect(); + + match run_jq(&str_args) { + Ok(code) => code, + Err(msg) => { + eprintln!("jq: {}", msg); + 2 + } + } +} + +struct JqOptions { + filter: String, + raw_output: bool, + raw_input: bool, + slurp: bool, + compact: bool, + null_input: bool, + exit_status: bool, + join_output: bool, + args: Vec<(String, String)>, + jsonargs: Vec<(String, Val)>, + input_paths: Vec, +} + +fn parse_args(args: &[String]) -> Result { + let mut opts = JqOptions { + filter: String::new(), + raw_output: false, + raw_input: false, + slurp: false, + compact: false, + null_input: false, + exit_status: false, + join_output: false, + args: Vec::new(), + jsonargs: Vec::new(), + input_paths: Vec::new(), + }; + + let mut filter_set = false; + let mut i = 0; + + while i < args.len() { + let arg = &args[i]; + + if arg == "--" { + let remaining = &args[i + 1..]; + if !filter_set { + let Some(filter) = remaining.first() else { + return Err("no filter provided".to_string()); + }; + opts.filter = filter.clone(); + filter_set = true; + opts.input_paths.extend(remaining.iter().skip(1).cloned()); + } else { + opts.input_paths.extend(remaining.iter().cloned()); + } + break; + } + + if arg.starts_with('-') && arg.len() > 1 && !arg.starts_with("--") { + // Handle combined short flags like -rn, -crS, etc. + let flags = &arg[1..]; + for c in flags.chars() { + match c { + 'r' => opts.raw_output = true, + 'R' => opts.raw_input = true, + 's' => opts.slurp = true, + 'c' => opts.compact = true, + 'n' => opts.null_input = true, + 'e' => opts.exit_status = true, + 'j' => opts.join_output = true, + 'S' => {} // sort keys - jaq sorts by default + _ => return Err(format!("unknown option: -{}", c)), + } + } + } else if arg == "--raw-output" || arg == "--raw-output0" { + opts.raw_output = true; + } else if arg == "--raw-input" { + opts.raw_input = true; + } else if arg == "--slurp" { + opts.slurp = true; + } else if arg == "--compact-output" { + opts.compact = true; + } else if arg == "--null-input" { + opts.null_input = true; + } else if arg == "--exit-status" { + opts.exit_status = true; + } else if arg == "--join-output" { + opts.join_output = true; + } else if arg == "--arg" { + if i + 2 >= args.len() { + return Err("--arg requires name and value".to_string()); + } + let name = args[i + 1].clone(); + let value = args[i + 2].clone(); + opts.args.push((name, value)); + i += 2; + } else if arg == "--argjson" { + if i + 2 >= args.len() { + return Err("--argjson requires name and value".to_string()); + } + let name = args[i + 1].clone(); + let json_str = &args[i + 2]; + let value: serde_json::Value = serde_json::from_str(json_str) + .map_err(|e| format!("invalid JSON for --argjson: {}", e))?; + opts.jsonargs.push((name, Val::from(value))); + i += 2; + } else if !filter_set { + opts.filter = arg.clone(); + filter_set = true; + } else { + opts.input_paths.push(arg.clone()); + } + + i += 1; + } + + if !filter_set { + return Err("no filter provided".to_string()); + } + + Ok(opts) +} + +fn read_sources(opts: &JqOptions) -> Result, String> { + if opts.input_paths.is_empty() { + let mut stdin_data = String::new(); + io::stdin() + .take((MAX_INPUT_BYTES + 1) as u64) + .read_to_string(&mut stdin_data) + .map_err(|e| format!("failed to read stdin: {}", e))?; + if stdin_data.len() > MAX_INPUT_BYTES { + return Err("stdin exceeds size limit".to_string()); + } + return Ok(vec![stdin_data]); + } + + let mut total = 0usize; + let mut sources = Vec::new(); + for path in &opts.input_paths { + let mut data = String::new(); + if path == "-" { + io::stdin() + .take((MAX_INPUT_BYTES.saturating_sub(total) + 1) as u64) + .read_to_string(&mut data) + .map_err(|e| format!("failed to read stdin: {}", e))?; + } else { + FsFile::open(path) + .map_err(|e| format!("failed to open {}: {}", path, e))? + .take((MAX_INPUT_BYTES.saturating_sub(total) + 1) as u64) + .read_to_string(&mut data) + .map_err(|e| format!("failed to read {}: {}", path, e))?; + } + total = total + .checked_add(data.len()) + .ok_or_else(|| "input exceeds size limit".to_string())?; + if total > MAX_INPUT_BYTES { + return Err("input exceeds size limit".to_string()); + } + sources.push(data); + } + Ok(sources) +} + +fn read_inputs(opts: &JqOptions) -> Result, String> { + if opts.null_input { + return Ok(vec![Val::from(serde_json::Value::Null)]); + } + + let sources = read_sources(opts)?; + + if opts.raw_input { + let raw_data = sources.concat(); + if opts.slurp { + Ok(vec![Val::from(serde_json::Value::String(raw_data))]) + } else { + let mut lines = Vec::new(); + for line in raw_data.lines() { + push_input_value( + &mut lines, + Val::from(serde_json::Value::String(line.to_string())), + )?; + } + Ok(lines) + } + } else { + let mut values = Vec::new(); + for source in &sources { + let trimmed = source.trim(); + if trimmed.is_empty() { + continue; + } + + let decoder = + serde_json::Deserializer::from_str(trimmed).into_iter::(); + for result in decoder { + let value = result.map_err(|e| format!("parse error: {}", e))?; + push_input_value(&mut values, value)?; + } + } + + if values.is_empty() && opts.input_paths.is_empty() { + values.push(serde_json::Value::Null); + } + + if opts.slurp { + Ok(vec![Val::from(serde_json::Value::Array(values))]) + } else { + Ok(values.into_iter().map(Val::from).collect()) + } + } +} + +fn push_input_value(values: &mut Vec, value: T) -> Result<(), String> { + if values.len() >= MAX_INPUT_VALUES { + return Err("too many input values".to_string()); + } + values.push(value); + Ok(()) +} + +/// Format a jaq Val as a string for output. +fn format_output(val: &Val, opts: &JqOptions) -> Result { + let compact_str = format!("{}", val); + + // For raw output, unquote strings + if opts.raw_output { + if compact_str.starts_with('"') && compact_str.ends_with('"') && compact_str.len() >= 2 { + if let Ok(unescaped) = serde_json::from_str::(&compact_str) { + return Ok(unescaped); + } + } + } + + if opts.compact { + Ok(compact_str) + } else { + // Pretty print via serde_json + if let Ok(v) = serde_json::from_str::(&compact_str) { + serde_json::to_string_pretty(&v).map_err(|e| format!("output format error: {}", e)) + } else { + Ok(compact_str) + } + } +} + +fn run_jq(args: &[String]) -> Result { + if matches!(args, [arg] if arg == "--version" || arg == "-V") { + println!("jq-jaq-{}", env!("CARGO_PKG_VERSION")); + return Ok(0); + } + + let opts = parse_args(args)?; + let inputs = read_inputs(&opts)?; + + // Set up variable bindings for --arg and --argjson + let mut var_vals: Vec = Vec::new(); + + for (_name, value) in &opts.args { + var_vals.push(Val::from(serde_json::Value::String(value.clone()))); + } + for (_name, value) in &opts.jsonargs { + var_vals.push(value.clone()); + } + + // Load and compile filter + let loader = Loader::new(jaq_std::defs().chain(jaq_json::defs())); + let arena = Arena::default(); + let program = File { + code: opts.filter.as_str(), + path: (), + }; + let modules = loader + .load(&arena, program) + .map_err(|errs| format!("parse error: {:?}", errs))?; + + let filter = Compiler::default() + .with_funs(jaq_std::funs().chain(jaq_json::funs())) + .compile(modules) + .map_err(|errs| format!("compile error: {:?}", errs))?; + + let empty_inputs = RcIter::new(core::iter::empty()); + let stdout = io::stdout(); + let mut out = stdout.lock(); + let mut had_false_or_null = false; + + for input in inputs { + let ctx = Ctx::new(var_vals.iter().cloned(), &empty_inputs); + let results = filter.run((ctx, input)); + + for result in results { + match result { + Ok(val) => { + let s = format_output(&val, &opts)?; + + // Track for --exit-status + let compact = format!("{}", val); + if compact == "null" || compact == "false" { + had_false_or_null = true; + } + + if opts.join_output { + write!(out, "{}", s) + .map_err(|e| format!("failed to write stdout: {}", e))?; + } else { + writeln!(out, "{}", s) + .map_err(|e| format!("failed to write stdout: {}", e))?; + } + } + Err(e) => { + eprintln!("jq: error: {}", e); + return Ok(5); + } + } + } + } + + if opts.exit_status && had_false_or_null { + return Ok(1); + } + + out.flush() + .map_err(|e| format!("failed to flush stdout: {}", e))?; + + Ok(0) +} diff --git a/software/jq/package.json b/software/jq/package.json new file mode 100644 index 0000000000..a7fa98ecf1 --- /dev/null +++ b/software/jq/package.json @@ -0,0 +1,33 @@ +{ + "name": "@agentos-software/jq", + "version": "0.3.3", + "type": "module", + "license": "Apache-2.0", + "description": "jq JSON processor for secure-exec VMs", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "files": [ + "dist", + "!dist/package", + "!dist/package.tar" + ], + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "scripts": { + "build": "agentos-toolchain stage --commands-dir ../../toolchain/target/wasm32-wasip1/release/commands --if-missing skip && tsc && agentos-toolchain build", + "check-types": "tsc --noEmit", + "test": "vitest run test/ --passWithNoTests" + }, + "devDependencies": { + "@agentos-software/manifest": "workspace:*", + "@rivet-dev/agentos-toolchain": "workspace:*", + "@types/node": "^22.10.2", + "typescript": "^5.9.2", + "@agentos/test-harness": "workspace:*", + "vitest": "^2.1.9" + } +} diff --git a/registry/software/grep/src/index.ts b/software/jq/src/index.ts similarity index 100% rename from registry/software/grep/src/index.ts rename to software/jq/src/index.ts diff --git a/software/jq/test/jq.test.ts b/software/jq/test/jq.test.ts new file mode 100644 index 0000000000..5d15eaebc3 --- /dev/null +++ b/software/jq/test/jq.test.ts @@ -0,0 +1,153 @@ +import { existsSync } from "node:fs"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { + NodeFileSystem, + createKernel, + createWasmVmRuntime, + describeIf, +} from "@agentos/test-harness"; +import type { Kernel } from "@agentos/test-harness"; +import { afterEach, expect, it } from "vitest"; + +const JQ_COMMAND_DIR = fileURLToPath(new URL("../bin", import.meta.url)); +const hasJqPackageBinary = existsSync(join(JQ_COMMAND_DIR, "jq")); + +let tempRoot: string | undefined; + +async function writeFixture(path: string, contents: string): Promise { + if (!tempRoot) throw new Error("fixture root not initialized"); + const hostPath = join(tempRoot, path.replace(/^\/+/, "")); + await mkdir(dirname(hostPath), { recursive: true }); + await writeFile(hostPath, contents); +} + +async function createTestVFS(): Promise { + tempRoot = await mkdtemp(join(tmpdir(), "agentos-jq-")); + await writeFixture( + "/project/users.json", + `${JSON.stringify( + { + users: [ + { name: "Ada", active: true, team: "runtime", score: 7 }, + { name: "Grace", active: false, team: "docs", score: 5 }, + { name: "Linus", active: true, team: "runtime", score: 3 }, + ], + }, + null, + 2, + )}\n`, + ); + await writeFixture( + "/project/events.ndjson", + [ + JSON.stringify({ type: "build", value: 4 }), + JSON.stringify({ type: "test", value: 6 }), + JSON.stringify({ type: "deploy", value: 2 }), + ].join("\n") + "\n", + ); + await writeFixture("/project/broken.json", '{"users": ['); + return new NodeFileSystem({ root: tempRoot }); +} + +function lines(stdout: string): string[] { + return stdout.split("\n").filter((line) => line.length > 0); +} + +describeIf(hasJqPackageBinary, "jq command", { timeout: 10_000 }, () => { + let kernel: Kernel | undefined; + + afterEach(async () => { + await kernel?.dispose(); + kernel = undefined; + if (tempRoot) { + await rm(tempRoot, { recursive: true, force: true }); + tempRoot = undefined; + } + }); + + async function mountFixture(): Promise { + const vfs = await createTestVFS(); + kernel = createKernel({ filesystem: vfs }); + await kernel.mount(createWasmVmRuntime({ commandDirs: [JQ_COMMAND_DIR] })); + } + + async function runJq(args: string[]) { + if (!kernel) throw new Error("kernel not mounted"); + let stdout = ""; + let stderr = ""; + const proc = kernel.spawn("jq", args, { + onStdout: (chunk) => { + stdout += Buffer.from(chunk).toString("utf8"); + }, + onStderr: (chunk) => { + stderr += Buffer.from(chunk).toString("utf8"); + }, + }); + const exitCode = await proc.wait(); + await new Promise((resolve) => setTimeout(resolve, 0)); + return { stdout, stderr, exitCode }; + } + + it("reports a jq-compatible version", async () => { + await mountFixture(); + + const result = await runJq(["--version"]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(result.stdout).toMatch(/^jq-/); + }); + + it("filters arrays and emits raw strings", async () => { + await mountFixture(); + + const result = await runJq([ + "-r", + '.users[] | select(.active) | "\\(.name):\\(.score)"', + "/project/users.json", + ]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(lines(result.stdout)).toEqual(["Ada:7", "Linus:3"]); + }); + + it("builds aggregate JSON objects", async () => { + await mountFixture(); + + const result = await runJq([ + "-c", + '{activeNames: [.users[] | select(.active) | .name], runtimeTotal: ([.users[] | select(.team == "runtime") | .score] | add)}', + "/project/users.json", + ]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(JSON.parse(result.stdout.trim())).toEqual({ + activeNames: ["Ada", "Linus"], + runtimeTotal: 10, + }); + }); + + it("slurps newline-delimited JSON records", async () => { + await mountFixture(); + + const result = await runJq([ + "-s", + "-c", + "{count: length, total: (map(.value) | add), types: map(.type)}", + "/project/events.ndjson", + ]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(JSON.parse(result.stdout.trim())).toEqual({ + count: 3, + total: 12, + types: ["build", "test", "deploy"], + }); + }); + + it("fails with a parse error for invalid JSON", async () => { + await mountFixture(); + + const result = await runJq([".", "/project/broken.json"]); + expect(result.exitCode).not.toBe(0); + expect(result.stderr).toContain("parse error"); + }); +}); diff --git a/software/jq/test/manifest.test.ts b/software/jq/test/manifest.test.ts new file mode 100644 index 0000000000..89139d33b8 --- /dev/null +++ b/software/jq/test/manifest.test.ts @@ -0,0 +1,19 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +const packageDir = new URL("..", import.meta.url).pathname; + +describe("package manifest", () => { + it("declares command binaries", () => { + const manifest = JSON.parse( + readFileSync(join(packageDir, "agentos-package.json"), "utf8"), + ); + + expect(manifest.commands?.length ?? 0).toBeGreaterThan(0); + for (const command of manifest.commands) { + expect(typeof command).toBe("string"); + expect(command.length).toBeGreaterThan(0); + } + }); +}); diff --git a/software/jq/tsconfig.json b/software/jq/tsconfig.json new file mode 100644 index 0000000000..03ce790ab7 --- /dev/null +++ b/software/jq/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*"] +} diff --git a/software/opencode/agentos-package.json b/software/opencode/agentos-package.json new file mode 100644 index 0000000000..c32dead093 --- /dev/null +++ b/software/opencode/agentos-package.json @@ -0,0 +1,19 @@ +{ + "name": "opencode", + "agent": { + "acpEntrypoint": "agentos-opencode-acp", + "env": { + "OPENCODE_DISABLE_CONFIG_DEP_INSTALL": "1", + "OPENCODE_DISABLE_EMBEDDED_WEB_UI": "1" + } + }, + "registry": { + "title": "OpenCode", + "description": "Run OpenCode, an open-source coding agent, inside agentOS.", + "docsHref": "/docs/agents/opencode", + "image": "/images/registry/opencode.svg", + "priority": 70, + "category": "agents" + }, + "kind": "agent" +} diff --git a/software/opencode/package.json b/software/opencode/package.json new file mode 100644 index 0000000000..b0d1546147 --- /dev/null +++ b/software/opencode/package.json @@ -0,0 +1,38 @@ +{ + "name": "@agentos-software/opencode", + "version": "0.2.1", + "type": "module", + "license": "Apache-2.0", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "bin": { + "agentos-opencode-acp": "./dist/adapter.js" + }, + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js", + "default": "./dist/index.js" + } + }, + "files": [ + "dist", + "!dist/package", + "!dist/package.tar", + "agentos-package.json" + ], + "scripts": { + "build": "node ./scripts/build-opencode-acp.mjs && tsc && rm -rf dist/package dist/package.tar && agentos-toolchain pack . --out dist/package --agent agentos-opencode-acp", + "check-types": "tsc --noEmit", + "test": "vitest run test/ --passWithNoTests" + }, + "devDependencies": { + "@agentos-software/manifest": "workspace:*", + "@rivet-dev/agentos-toolchain": "workspace:*", + "@types/node": "^22.10.2", + "bun": "1.3.11", + "typescript": "^5.7.2", + "@agentos/test-harness": "workspace:*", + "vitest": "^2.1.9" + } +} diff --git a/registry/agent/opencode/scripts/build-opencode-acp.mjs b/software/opencode/scripts/build-opencode-acp.mjs similarity index 100% rename from registry/agent/opencode/scripts/build-opencode-acp.mjs rename to software/opencode/scripts/build-opencode-acp.mjs diff --git a/registry/agent/opencode/src/adapter.ts b/software/opencode/src/adapter.ts similarity index 100% rename from registry/agent/opencode/src/adapter.ts rename to software/opencode/src/adapter.ts diff --git a/registry/software/gzip/src/index.ts b/software/opencode/src/index.ts similarity index 100% rename from registry/software/gzip/src/index.ts rename to software/opencode/src/index.ts diff --git a/registry/agent/opencode/src/opencode-acp.mjs.d.ts b/software/opencode/src/opencode-acp.mjs.d.ts similarity index 100% rename from registry/agent/opencode/src/opencode-acp.mjs.d.ts rename to software/opencode/src/opencode-acp.mjs.d.ts diff --git a/software/opencode/tsconfig.json b/software/opencode/tsconfig.json new file mode 100644 index 0000000000..73a06ddbdb --- /dev/null +++ b/software/opencode/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "declaration": true, + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist"] +} diff --git a/registry/agent/opencode/upstream/opencode-v1.3.13.patch b/software/opencode/upstream/opencode-v1.3.13.patch similarity index 100% rename from registry/agent/opencode/upstream/opencode-v1.3.13.patch rename to software/opencode/upstream/opencode-v1.3.13.patch diff --git a/software/pi-cli/agentos-package.json b/software/pi-cli/agentos-package.json new file mode 100644 index 0000000000..9d25dd6f54 --- /dev/null +++ b/software/pi-cli/agentos-package.json @@ -0,0 +1,10 @@ +{ + "name": "pi-cli", + "agent": { + "acpEntrypoint": "pi-acp", + "env": { + "PI_ACP_PI_COMMAND": "pi" + } + }, + "kind": "agent" +} diff --git a/software/pi-cli/package.json b/software/pi-cli/package.json new file mode 100644 index 0000000000..a6c9463342 --- /dev/null +++ b/software/pi-cli/package.json @@ -0,0 +1,42 @@ +{ + "name": "@agentos-software/pi-cli", + "version": "0.2.1", + "type": "module", + "license": "Apache-2.0", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "bin": { + "pi-acp": "./node_modules/pi-acp/dist/index.js", + "pi": "./node_modules/@mariozechner/pi-coding-agent/dist/cli.js" + }, + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js", + "default": "./dist/index.js" + } + }, + "files": [ + "dist", + "!dist/package", + "!dist/package.tar", + "agentos-package.json" + ], + "scripts": { + "build": "tsc && rm -rf dist/package dist/package.tar && agentos-toolchain pack . --out dist/package --agent pi-acp --prune-native", + "check-types": "tsc --noEmit", + "test": "vitest run test/ --passWithNoTests" + }, + "dependencies": { + "@mariozechner/pi-coding-agent": "^0.60.0", + "pi-acp": "^0.0.23" + }, + "devDependencies": { + "@agentos-software/manifest": "workspace:*", + "@rivet-dev/agentos-toolchain": "workspace:*", + "@types/node": "^22.10.2", + "typescript": "^5.7.2", + "@agentos/test-harness": "workspace:*", + "vitest": "^2.1.9" + } +} diff --git a/registry/software/http-get/src/index.ts b/software/pi-cli/src/index.ts similarity index 100% rename from registry/software/http-get/src/index.ts rename to software/pi-cli/src/index.ts diff --git a/software/pi-cli/tsconfig.json b/software/pi-cli/tsconfig.json new file mode 100644 index 0000000000..73a06ddbdb --- /dev/null +++ b/software/pi-cli/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "declaration": true, + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist"] +} diff --git a/software/pi/adapter-browser-entry.mjs b/software/pi/adapter-browser-entry.mjs new file mode 100644 index 0000000000..2bb013fee2 --- /dev/null +++ b/software/pi/adapter-browser-entry.mjs @@ -0,0 +1,38 @@ +// Browser-bundle entry for the pi ACP adapter. The adapter normally loads its SDK via +// computed dynamic import() from a VM-mounted node_modules; in the browser converged +// executor there is no such mount, so we statically import the SDK submodules (which +// esbuild bundles into a single self-contained file) and hand them to the adapter via +// the `__piSdkModules` override (see loadPiSdkRuntime). The adapter is otherwise +// unchanged — same ACP behavior, just bundled instead of VFS-resolved. +// +// Imports use relative node_modules file paths (not bare package subpaths) so esbuild +// resolves the dist files directly, bypassing the packages' restrictive `exports`. +import * as agentCore from "./node_modules/@mariozechner/pi-agent-core/dist/index.js"; +import * as authStorage from "./node_modules/@mariozechner/pi-coding-agent/dist/core/auth-storage.js"; +import * as config from "./node_modules/@mariozechner/pi-coding-agent/dist/config.js"; +import * as defaults from "./node_modules/@mariozechner/pi-coding-agent/dist/core/defaults.js"; +import * as messages from "./node_modules/@mariozechner/pi-coding-agent/dist/core/messages.js"; +import * as modelRegistry from "./node_modules/@mariozechner/pi-coding-agent/dist/core/model-registry.js"; +import * as resourceLoader from "./node_modules/@mariozechner/pi-coding-agent/dist/core/resource-loader.js"; +import * as sdk from "./node_modules/@mariozechner/pi-coding-agent/dist/core/sdk.js"; +import * as sessionManager from "./node_modules/@mariozechner/pi-coding-agent/dist/core/session-manager.js"; +import * as settingsManager from "./node_modules/@mariozechner/pi-coding-agent/dist/core/settings-manager.js"; +import * as tools from "./node_modules/@mariozechner/pi-coding-agent/dist/core/tools/index.js"; + +// loadPiSdkRuntime reads this lazily (at session/new), so setting it after the adapter +// import (ESM-hoisted) is fine. +globalThis.__piSdkModules = { + agentCore, + authStorage, + config, + defaults, + messages, + modelRegistry, + resourceLoader, + sdk, + sessionManager, + settingsManager, + tools, +}; + +import "./dist/adapter.js"; diff --git a/software/pi/agentos-package.json b/software/pi/agentos-package.json new file mode 100644 index 0000000000..07ebc84234 --- /dev/null +++ b/software/pi/agentos-package.json @@ -0,0 +1,15 @@ +{ + "name": "pi", + "agent": { + "acpEntrypoint": "pi-sdk-acp", + "snapshot": true + }, + "registry": { + "title": "PI", + "description": "Run the PI coding agent with lightweight, fast execution.", + "image": "/images/registry/pi.svg", + "priority": 100, + "category": "agents" + }, + "kind": "agent" +} diff --git a/software/pi/package.json b/software/pi/package.json new file mode 100644 index 0000000000..c32c77b09b --- /dev/null +++ b/software/pi/package.json @@ -0,0 +1,44 @@ +{ + "name": "@agentos-software/pi", + "version": "0.2.1", + "type": "module", + "license": "Apache-2.0", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "bin": { + "pi": "./node_modules/@mariozechner/pi-coding-agent/dist/cli.js", + "pi-sdk-acp": "./dist/adapter.js" + }, + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js", + "default": "./dist/index.js" + } + }, + "files": [ + "dist", + "!dist/package", + "!dist/package.tar", + "agentos-package.json" + ], + "scripts": { + "build": "tsc && node scripts/build-snapshot-bundle.mjs && rm -rf dist/package dist/package.tar && agentos-toolchain pack . --out dist/package --agent pi-sdk-acp --prune-native && node scripts/copy-snapshot-into-package.mjs", + "build:snapshot": "node scripts/build-snapshot-bundle.mjs", + "check-types": "tsc --noEmit", + "test": "pnpm build && node --test --test-force-exit tests/*.test.mjs" + }, + "dependencies": { + "@agentclientprotocol/sdk": "^0.16.1", + "@mariozechner/pi-coding-agent": "0.60.0", + "@mariozechner/pi-ai": "0.60.0" + }, + "devDependencies": { + "@agentos-software/manifest": "workspace:*", + "@rivet-dev/agentos-toolchain": "workspace:*", + "@types/node": "^22.10.2", + "esbuild": "^0.27.4", + "typescript": "^5.7.2", + "@agentos/test-harness": "workspace:*" + } +} diff --git a/registry/agent/pi/scripts/build-snapshot-bundle.mjs b/software/pi/scripts/build-snapshot-bundle.mjs similarity index 100% rename from registry/agent/pi/scripts/build-snapshot-bundle.mjs rename to software/pi/scripts/build-snapshot-bundle.mjs diff --git a/registry/agent/pi/scripts/copy-snapshot-into-package.mjs b/software/pi/scripts/copy-snapshot-into-package.mjs similarity index 100% rename from registry/agent/pi/scripts/copy-snapshot-into-package.mjs rename to software/pi/scripts/copy-snapshot-into-package.mjs diff --git a/registry/agent/pi/src/adapter.ts b/software/pi/src/adapter.ts similarity index 97% rename from registry/agent/pi/src/adapter.ts rename to software/pi/src/adapter.ts index 1eea462bdb..6fb8a3bf6d 100644 --- a/registry/agent/pi/src/adapter.ts +++ b/software/pi/src/adapter.ts @@ -103,32 +103,8 @@ Object.defineProperty(process, "stdin", { type SessionManagerLike = { inMemory(cwd?: string): unknown; - continueRecent(cwd: string, sessionDir: string): unknown; }; -/** - * Choose the Pi `SessionManager` for a new session. - * - * By default sessions are in-memory (nothing is written to disk), so a - * conversation is lost when the adapter process restarts. When the embedder - * provides a session directory via the `PI_SESSION_DIR` env var, use pi's - * `continueRecent` instead: it persists the session's `.jsonl` under that - * directory and resumes the most recent one, so conversations survive an adapter - * restart. No behavior change unless `PI_SESSION_DIR` is set. - * - * Exported for unit testing (the real `newSession` path needs the Pi SDK). - */ -export function resolveSessionManager( - SessionManager: SessionManagerLike, - cwd: string, - env: Record = process.env, -): unknown { - const sessionDir = env.PI_SESSION_DIR?.trim(); - return sessionDir - ? SessionManager.continueRecent(cwd, sessionDir) - : SessionManager.inMemory(cwd); -} - type ModelLike = { id: string; provider: string; @@ -897,7 +873,7 @@ export class PiSdkAgent implements Agent { const { session } = await __trace.span("createAgentSession", () => createAgentSession({ cwd: params.cwd, - sessionManager: resolveSessionManager(SessionManager, params.cwd), + sessionManager: SessionManager.inMemory(params.cwd), resourceLoader, tools: this.wrapTools( createCodingTools(params.cwd, { diff --git a/registry/software/jq/src/index.ts b/software/pi/src/index.ts similarity index 100% rename from registry/software/jq/src/index.ts rename to software/pi/src/index.ts diff --git a/registry/agent/pi/src/snapshot-entry.ts b/software/pi/src/snapshot-entry.ts similarity index 100% rename from registry/agent/pi/src/snapshot-entry.ts rename to software/pi/src/snapshot-entry.ts diff --git a/registry/agent/pi/tests/adapter.test.mjs b/software/pi/tests/adapter.test.mjs similarity index 100% rename from registry/agent/pi/tests/adapter.test.mjs rename to software/pi/tests/adapter.test.mjs diff --git a/software/pi/tsconfig.json b/software/pi/tsconfig.json new file mode 100644 index 0000000000..3ecc60a8e9 --- /dev/null +++ b/software/pi/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "declaration": true, + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist", "src/snapshot-entry.ts"] +} diff --git a/software/ripgrep/agentos-package.json b/software/ripgrep/agentos-package.json new file mode 100644 index 0000000000..dbc08da455 --- /dev/null +++ b/software/ripgrep/agentos-package.json @@ -0,0 +1,11 @@ +{ + "commands": [ + "rg" + ], + "registry": { + "title": "ripgrep", + "description": "Fast recursive search (rg).", + "priority": 85, + "category": "developer-tools" + } +} diff --git a/software/ripgrep/native/crates/cmd-rg/Cargo.toml b/software/ripgrep/native/crates/cmd-rg/Cargo.toml new file mode 100644 index 0000000000..f6184892c7 --- /dev/null +++ b/software/ripgrep/native/crates/cmd-rg/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-rg" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "rg (ripgrep) standalone binary for secure-exec VM" +publish = false + +[dependencies] +# ripgrep is a bin-only crate. `toolchain/Makefile` builds its upstream `rg` +# binary directly; this local package exists only so `cmd/rg` is discoverable +# and the upstream package is present in Cargo's resolved/vendor set. +ripgrep = { version = "15.1.0", default-features = false } diff --git a/software/ripgrep/native/crates/cmd-rg/src/lib.rs b/software/ripgrep/native/crates/cmd-rg/src/lib.rs new file mode 100644 index 0000000000..1bd8717252 --- /dev/null +++ b/software/ripgrep/native/crates/cmd-rg/src/lib.rs @@ -0,0 +1 @@ +//! Build trigger for the upstream `ripgrep` command package. diff --git a/software/ripgrep/package.json b/software/ripgrep/package.json new file mode 100644 index 0000000000..7241515e70 --- /dev/null +++ b/software/ripgrep/package.json @@ -0,0 +1,33 @@ +{ + "name": "@agentos-software/ripgrep", + "version": "0.3.3", + "type": "module", + "license": "Apache-2.0", + "description": "ripgrep fast search for secure-exec VMs", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "files": [ + "dist", + "!dist/package", + "!dist/package.tar" + ], + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "scripts": { + "build": "agentos-toolchain stage --commands-dir ../../toolchain/target/wasm32-wasip1/release/commands --if-missing skip && tsc && agentos-toolchain build", + "check-types": "tsc --noEmit", + "test": "vitest run test/ --passWithNoTests" + }, + "devDependencies": { + "@agentos-software/manifest": "workspace:*", + "@rivet-dev/agentos-toolchain": "workspace:*", + "@types/node": "^22.10.2", + "typescript": "^5.9.2", + "@agentos/test-harness": "workspace:*", + "vitest": "^2.1.9" + } +} diff --git a/registry/software/ripgrep/src/index.ts b/software/ripgrep/src/index.ts similarity index 100% rename from registry/software/ripgrep/src/index.ts rename to software/ripgrep/src/index.ts diff --git a/software/ripgrep/test/manifest.test.ts b/software/ripgrep/test/manifest.test.ts new file mode 100644 index 0000000000..89139d33b8 --- /dev/null +++ b/software/ripgrep/test/manifest.test.ts @@ -0,0 +1,19 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +const packageDir = new URL("..", import.meta.url).pathname; + +describe("package manifest", () => { + it("declares command binaries", () => { + const manifest = JSON.parse( + readFileSync(join(packageDir, "agentos-package.json"), "utf8"), + ); + + expect(manifest.commands?.length ?? 0).toBeGreaterThan(0); + for (const command of manifest.commands) { + expect(typeof command).toBe("string"); + expect(command.length).toBeGreaterThan(0); + } + }); +}); diff --git a/software/ripgrep/test/ripgrep.test.ts b/software/ripgrep/test/ripgrep.test.ts new file mode 100644 index 0000000000..af8d77885e --- /dev/null +++ b/software/ripgrep/test/ripgrep.test.ts @@ -0,0 +1,128 @@ +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { createWasmVmRuntime } from "@agentos/test-harness"; +import { + COMMANDS_DIR, + NodeFileSystem, + createKernel, + describeIf, + hasWasmBinaries, +} from "@agentos/test-harness"; +import type { Kernel } from "@agentos/test-harness"; +import { afterEach, describe, expect, it } from "vitest"; + +let tempRoot: string | undefined; + +async function createTestVFS(): Promise { + tempRoot = await mkdtemp(join(tmpdir(), "agentos-ripgrep-")); + + await writeFixture( + "/project/src/main.rs", + ["fn main() {", ' println!("needle");', "}"].join("\n") + "\n", + ); + await writeFixture( + "/project/src/lib.rs", + ["pub fn helper() {", " // Needle in a comment", "}"].join("\n") + "\n", + ); + await writeFixture("/project/docs/readme.md", "needle in docs\n"); + await writeFixture("/project/vendor/generated.rs", "needle in vendor\n"); + await writeFixture("/project/.hidden.txt", "needle hidden\n"); + await writeFixture("/project/.gitignore", "vendor/\n"); + await writeFixture("/project/.git/HEAD", "ref: refs/heads/main\n"); + + return new NodeFileSystem({ root: tempRoot }); +} + +async function writeFixture(path: string, contents: string): Promise { + if (!tempRoot) throw new Error("fixture root not initialized"); + const hostPath = join(tempRoot, path.replace(/^\/+/, "")); + await mkdir(dirname(hostPath), { recursive: true }); + await writeFile(hostPath, contents); +} + +function lines(stdout: string): string[] { + return stdout + .split("\n") + .filter((line) => line.length > 0) + .sort(); +} + +describeIf(hasWasmBinaries, "ripgrep command", { timeout: 10_000 }, () => { + let kernel: Kernel; + + afterEach(async () => { + await kernel?.dispose(); + if (tempRoot) { + await rm(tempRoot, { recursive: true, force: true }); + tempRoot = undefined; + } + }); + + async function mountFixture(): Promise { + const vfs = await createTestVFS(); + kernel = createKernel({ filesystem: vfs }); + await kernel.mount(createWasmVmRuntime({ commandDirs: [COMMANDS_DIR] })); + } + + it("reports the upstream ripgrep version", async () => { + await mountFixture(); + + const result = await kernel.exec("rg --version", {}); + expect(result.stdout).toContain("ripgrep 15.1.0"); + expect(result.stdout).not.toContain("secure-exec"); + }); + + it("searches recursively and respects .gitignore by default", async () => { + await mountFixture(); + + const result = await kernel.exec("rg needle /project", {}); + const output = lines(result.stdout); + + expect(output).toContain('/project/src/main.rs: println!("needle");'); + expect(output).toContain("/project/docs/readme.md:needle in docs"); + expect(output).not.toContain("/project/vendor/generated.rs:needle in vendor"); + expect(output).not.toContain("/project/.hidden.txt:needle hidden"); + }); + + it("supports case-insensitive search", async () => { + await mountFixture(); + + const result = await kernel.exec("rg -i needle /project/src", {}); + expect(lines(result.stdout)).toContain("/project/src/lib.rs: // Needle in a comment"); + }); + + it("supports fixed-string search", async () => { + await mountFixture(); + + const result = await kernel.exec("rg -F 'println!(\"needle\")' /project/src", {}); + expect(result.stdout.trim()).toBe('/project/src/main.rs: println!("needle");'); + }); + + it("supports glob filtering", async () => { + await mountFixture(); + + const result = await kernel.exec("rg needle /project -g '*.md'", {}); + expect(result.stdout.trim()).toBe("/project/docs/readme.md:needle in docs"); + }); + + it("can include hidden and ignored files when requested", async () => { + await mountFixture(); + + const result = await kernel.exec("rg -uu needle /project", {}); + const output = lines(result.stdout); + + expect(output).toContain("/project/.hidden.txt:needle hidden"); + expect(output).toContain("/project/vendor/generated.rs:needle in vendor"); + }); + + it("emits JSON search records", async () => { + await mountFixture(); + + const result = await kernel.exec("rg --json needle /project/docs", {}); + const records = lines(result.stdout).map((line) => JSON.parse(line)); + + expect(records.some((record) => record.type === "match")).toBe(true); + expect(records.some((record) => record.type === "summary")).toBe(true); + }); +}); diff --git a/software/ripgrep/tsconfig.json b/software/ripgrep/tsconfig.json new file mode 100644 index 0000000000..03ce790ab7 --- /dev/null +++ b/software/ripgrep/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*"] +} diff --git a/software/sed/agentos-package.json b/software/sed/agentos-package.json new file mode 100644 index 0000000000..7f66a6f0ed --- /dev/null +++ b/software/sed/agentos-package.json @@ -0,0 +1,12 @@ +{ + "commands": [ + "sed" + ], + "registry": { + "title": "sed", + "description": "GNU stream editor for text transformation.", + "priority": 35, + "image": "/images/registry/sed.svg", + "category": "core" + } +} diff --git a/software/sed/native/crates/cmd-sed/Cargo.toml b/software/sed/native/crates/cmd-sed/Cargo.toml new file mode 100644 index 0000000000..990de688db --- /dev/null +++ b/software/sed/native/crates/cmd-sed/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-sed" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "sed standalone binary for secure-exec VM" + +[[bin]] +name = "sed" +path = "src/main.rs" + +[dependencies] +sed = "0.1.1" diff --git a/registry/native/crates/commands/sed/src/main.rs b/software/sed/native/crates/cmd-sed/src/main.rs similarity index 100% rename from registry/native/crates/commands/sed/src/main.rs rename to software/sed/native/crates/cmd-sed/src/main.rs diff --git a/software/sed/package.json b/software/sed/package.json new file mode 100644 index 0000000000..24609bb457 --- /dev/null +++ b/software/sed/package.json @@ -0,0 +1,33 @@ +{ + "name": "@agentos-software/sed", + "version": "0.3.3", + "type": "module", + "license": "Apache-2.0", + "description": "GNU sed stream editor for secure-exec VMs", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "files": [ + "dist", + "!dist/package", + "!dist/package.tar" + ], + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "scripts": { + "build": "agentos-toolchain stage --commands-dir ../../toolchain/target/wasm32-wasip1/release/commands --if-missing skip && tsc && agentos-toolchain build", + "check-types": "tsc --noEmit", + "test": "vitest run test/ --passWithNoTests" + }, + "devDependencies": { + "@agentos-software/manifest": "workspace:*", + "@rivet-dev/agentos-toolchain": "workspace:*", + "@types/node": "^22.10.2", + "typescript": "^5.9.2", + "@agentos/test-harness": "workspace:*", + "vitest": "^2.1.9" + } +} diff --git a/registry/software/sed/src/index.ts b/software/sed/src/index.ts similarity index 100% rename from registry/software/sed/src/index.ts rename to software/sed/src/index.ts diff --git a/software/sed/test/manifest.test.ts b/software/sed/test/manifest.test.ts new file mode 100644 index 0000000000..89139d33b8 --- /dev/null +++ b/software/sed/test/manifest.test.ts @@ -0,0 +1,19 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +const packageDir = new URL("..", import.meta.url).pathname; + +describe("package manifest", () => { + it("declares command binaries", () => { + const manifest = JSON.parse( + readFileSync(join(packageDir, "agentos-package.json"), "utf8"), + ); + + expect(manifest.commands?.length ?? 0).toBeGreaterThan(0); + for (const command of manifest.commands) { + expect(typeof command).toBe("string"); + expect(command.length).toBeGreaterThan(0); + } + }); +}); diff --git a/software/sed/test/sed.test.ts b/software/sed/test/sed.test.ts new file mode 100644 index 0000000000..c6ccf420a2 --- /dev/null +++ b/software/sed/test/sed.test.ts @@ -0,0 +1,142 @@ +import { existsSync } from "node:fs"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { + NodeFileSystem, + createKernel, + createWasmVmRuntime, + describeIf, +} from "@agentos/test-harness"; +import type { Kernel } from "@agentos/test-harness"; +import { afterEach, expect, it } from "vitest"; + +const SED_COMMAND_DIR = fileURLToPath(new URL("../bin", import.meta.url)); +const hasSedPackageBinary = existsSync(join(SED_COMMAND_DIR, "sed")); + +let tempRoot: string | undefined; + +async function writeFixture(path: string, contents: string): Promise { + if (!tempRoot) throw new Error("fixture root not initialized"); + const hostPath = join(tempRoot, path.replace(/^\/+/, "")); + await mkdir(dirname(hostPath), { recursive: true }); + await writeFile(hostPath, contents); +} + +async function createTestVFS(): Promise { + tempRoot = await mkdtemp(join(tmpdir(), "agentos-sed-")); + await writeFixture( + "/project/log.txt", + [ + "ERROR: disk full", + "INFO: retrying", + "ERROR: timeout", + "DEBUG: ignored", + ].join("\n") + "\n", + ); + await writeFixture( + "/project/records.txt", + ["alpha:build:4", "beta:test:6", "gamma:deploy:2"].join("\n") + "\n", + ); + return new NodeFileSystem({ root: tempRoot }); +} + +function lines(stdout: string): string[] { + return stdout.split("\n").filter((line) => line.length > 0); +} + +describeIf(hasSedPackageBinary, "sed command", { timeout: 10_000 }, () => { + let kernel: Kernel | undefined; + + afterEach(async () => { + await kernel?.dispose(); + kernel = undefined; + if (tempRoot) { + await rm(tempRoot, { recursive: true, force: true }); + tempRoot = undefined; + } + }); + + async function mountFixture(): Promise { + const vfs = await createTestVFS(); + kernel = createKernel({ filesystem: vfs }); + await kernel.mount(createWasmVmRuntime({ commandDirs: [SED_COMMAND_DIR] })); + } + + async function runSed(args: string[]) { + if (!kernel) throw new Error("kernel not mounted"); + let stdout = ""; + let stderr = ""; + const proc = kernel.spawn("sed", args, { + onStdout: (chunk) => { + stdout += Buffer.from(chunk).toString("utf8"); + }, + onStderr: (chunk) => { + stderr += Buffer.from(chunk).toString("utf8"); + }, + }); + const exitCode = await proc.wait(); + await new Promise((resolve) => setTimeout(resolve, 0)); + return { stdout, stderr, exitCode }; + } + + it("substitutes text in file operands", async () => { + await mountFixture(); + + const result = await runSed(["s/ERROR/WARN/", "/project/log.txt"]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(lines(result.stdout)).toEqual([ + "WARN: disk full", + "INFO: retrying", + "WARN: timeout", + "DEBUG: ignored", + ]); + }); + + it("prints addressed matches with -n", async () => { + await mountFixture(); + + const result = await runSed(["-n", "/ERROR/p", "/project/log.txt"]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(lines(result.stdout)).toEqual(["ERROR: disk full", "ERROR: timeout"]); + }); + + it("deletes addressed records", async () => { + await mountFixture(); + + const result = await runSed(["/DEBUG/d", "/project/log.txt"]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(lines(result.stdout)).toEqual([ + "ERROR: disk full", + "INFO: retrying", + "ERROR: timeout", + ]); + }); + + it("applies multiple expressions in order", async () => { + await mountFixture(); + + const result = await runSed([ + "-e", + "s/:/ /g", + "-e", + "s/^/row /", + "/project/records.txt", + ]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(lines(result.stdout)).toEqual([ + "row alpha build 4", + "row beta test 6", + "row gamma deploy 2", + ]); + }); + + it("fails when an input file is missing", async () => { + await mountFixture(); + + const result = await runSed(["s/a/b/", "/project/missing.txt"]); + expect(result.exitCode).not.toBe(0); + expect(result.stderr).toContain("/project/missing.txt"); + }); +}); diff --git a/software/sed/tsconfig.json b/software/sed/tsconfig.json new file mode 100644 index 0000000000..03ce790ab7 --- /dev/null +++ b/software/sed/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*"] +} diff --git a/software/sqlite3/agentos-package.json b/software/sqlite3/agentos-package.json new file mode 100644 index 0000000000..8722328cdf --- /dev/null +++ b/software/sqlite3/agentos-package.json @@ -0,0 +1,12 @@ +{ + "commands": [ + "sqlite3" + ], + "registry": { + "title": "SQLite3", + "description": "SQLite database command-line interface.", + "priority": 75, + "image": "/images/registry/sqlite3.svg", + "category": "data" + } +} diff --git a/software/sqlite3/package.json b/software/sqlite3/package.json new file mode 100644 index 0000000000..11434344f6 --- /dev/null +++ b/software/sqlite3/package.json @@ -0,0 +1,33 @@ +{ + "name": "@agentos-software/sqlite3", + "version": "0.3.3", + "type": "module", + "license": "Apache-2.0", + "description": "SQLite3 CLI for secure-exec VMs", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "files": [ + "dist", + "!dist/package", + "!dist/package.tar" + ], + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "scripts": { + "build": "agentos-toolchain stage --commands-dir ../../toolchain/target/wasm32-wasip1/release/commands --if-missing skip && tsc && agentos-toolchain build", + "check-types": "tsc --noEmit", + "test": "vitest run test/ --passWithNoTests" + }, + "devDependencies": { + "@agentos-software/manifest": "workspace:*", + "@rivet-dev/agentos-toolchain": "workspace:*", + "@types/node": "^22.10.2", + "typescript": "^5.9.2", + "@agentos/test-harness": "workspace:*", + "vitest": "^2.1.9" + } +} diff --git a/registry/software/sqlite3/src/index.ts b/software/sqlite3/src/index.ts similarity index 100% rename from registry/software/sqlite3/src/index.ts rename to software/sqlite3/src/index.ts diff --git a/software/sqlite3/test/sqlite3.test.ts b/software/sqlite3/test/sqlite3.test.ts new file mode 100644 index 0000000000..2b7144304e --- /dev/null +++ b/software/sqlite3/test/sqlite3.test.ts @@ -0,0 +1,356 @@ +/** + * Integration tests for sqlite3 C command. + * + * Verifies SQLite CLI operations via kernel.exec() with real WASM binaries: + * - In-memory databases (:memory:) + * - Stdin pipe mode for simple queries + * - SQL from command line arguments for multi-statement operations + * - Meta-commands (.dump, .schema, .tables) + * + * The command is the official SQLite shell compiled against the AgentOS C + * sysroot, not the former local sqlite3_cli.c reimplementation. + */ + +import { describe, it, expect, afterEach } from 'vitest'; +import { existsSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { createWasmVmRuntime } from '@agentos/test-harness'; +import { + C_BUILD_DIR, + COMMANDS_DIR, + createKernel, + describeIf, +} from '@agentos/test-harness'; +import type { Kernel } from '@agentos/test-harness'; + +const SQLITE3_COMMAND_DIRS = [C_BUILD_DIR, COMMANDS_DIR].filter((dir) => + existsSync(dir) +); +const hasSqlite3Binary = SQLITE3_COMMAND_DIRS.some((dir) => + existsSync(resolve(dir, 'sqlite3')) +); + +// Minimal in-memory VFS for kernel tests +class SimpleVFS { + private files = new Map(); + private dirs = new Set(['/']); + + async readFile(path: string): Promise { + const data = this.files.get(path); + if (!data) throw new Error(`ENOENT: ${path}`); + return data; + } + async readTextFile(path: string): Promise { + return new TextDecoder().decode(await this.readFile(path)); + } + async readDir(path: string): Promise { + const prefix = path === '/' ? '/' : path + '/'; + const entries: string[] = []; + for (const p of [...this.files.keys(), ...this.dirs]) { + if (p !== path && p.startsWith(prefix)) { + const rest = p.slice(prefix.length); + if (!rest.includes('/')) entries.push(rest); + } + } + return entries; + } + async readDirWithTypes(path: string) { + return (await this.readDir(path)).map(name => ({ + name, + isDirectory: this.dirs.has(path === '/' ? `/${name}` : `${path}/${name}`), + })); + } + async writeFile(path: string, content: string | Uint8Array): Promise { + const data = typeof content === 'string' ? new TextEncoder().encode(content) : content; + this.files.set(path, new Uint8Array(data)); + const parts = path.split('/').filter(Boolean); + for (let i = 1; i < parts.length; i++) { + this.dirs.add('/' + parts.slice(0, i).join('/')); + } + } + async createDir(path: string) { this.dirs.add(path); } + async mkdir(path: string, _options?: { recursive?: boolean }) { + this.dirs.add(path); + const parts = path.split('/').filter(Boolean); + for (let i = 1; i < parts.length; i++) { + this.dirs.add('/' + parts.slice(0, i).join('/')); + } + } + async exists(path: string): Promise { + return this.files.has(path) || this.dirs.has(path); + } + async stat(path: string) { + const isDir = this.dirs.has(path); + const data = this.files.get(path); + if (!isDir && !data) throw new Error(`ENOENT: ${path}`); + return { + mode: isDir ? 0o40755 : 0o100644, + size: data?.length ?? 0, + isDirectory: isDir, + isSymbolicLink: false, + atimeMs: Date.now(), + mtimeMs: Date.now(), + ctimeMs: Date.now(), + birthtimeMs: Date.now(), + ino: 0, + nlink: 1, + uid: 1000, + gid: 1000, + }; + } + async chmod(_path: string, _mode: number) {} + async lstat(path: string) { return this.stat(path); } + async removeFile(path: string) { this.files.delete(path); } + async removeDir(path: string) { this.dirs.delete(path); } + async rename(oldPath: string, newPath: string) { + const data = this.files.get(oldPath); + if (data) { + this.files.set(newPath, data); + this.files.delete(oldPath); + } + } + async pread(path: string, buffer: Uint8Array, offset: number, length: number, position: number): Promise { + const data = this.files.get(path); + if (!data) throw new Error(`ENOENT: ${path}`); + const available = Math.min(length, data.length - position); + if (available <= 0) return 0; + buffer.set(data.subarray(position, position + available), offset); + return available; + } + async pwrite(path: string, offset: number, data: Uint8Array): Promise { + const current = this.files.get(path); + if (!current) throw new Error(`ENOENT: ${path}`); + const next = new Uint8Array(Math.max(current.length, offset + data.length)); + next.set(current); + next.set(data, offset); + this.files.set(path, next); + } +} + +describeIf(hasSqlite3Binary, 'sqlite3 command', () => { + let kernel: Kernel; + + afterEach(async () => { + await kernel?.dispose(); + }); + + it('executes SQL from stdin pipe on in-memory database', async () => { + const vfs = new SimpleVFS(); + kernel = createKernel({ filesystem: vfs as any }); + await kernel.mount(createWasmVmRuntime({ commandDirs: SQLITE3_COMMAND_DIRS })); + + const result = await kernel.exec('sqlite3 :memory:', { + stdin: 'SELECT 1+1 AS result;\n', + }); + expect(result.stdout.trim()).toBe('2'); + }); + + it('executes multi-statement SQL as command argument', async () => { + const vfs = new SimpleVFS(); + kernel = createKernel({ filesystem: vfs as any }); + await kernel.mount(createWasmVmRuntime({ commandDirs: SQLITE3_COMMAND_DIRS })); + + // Multi-statement SQL passed as command argument (more reliable than stdin in WASM) + const sql = 'CREATE TABLE t(x INTEGER); INSERT INTO t VALUES(10); INSERT INTO t VALUES(20); INSERT INTO t VALUES(30); SELECT * FROM t ORDER BY x;'; + const result = await kernel.exec(`sqlite3 :memory: "${sql}"`); + expect(result.stdout.trim()).toBe('10\n20\n30'); + }); + + it('supports .tables meta-command', async () => { + const vfs = new SimpleVFS(); + await vfs.mkdir('/tmp', { recursive: true }); + kernel = createKernel({ filesystem: vfs as any }); + await kernel.mount(createWasmVmRuntime({ commandDirs: SQLITE3_COMMAND_DIRS })); + + const createResult = await kernel.exec( + 'sqlite3 /tmp/meta.db "CREATE TABLE alpha(x); CREATE TABLE beta(y);"' + ); + expect(createResult.stderr).toBe(''); + + const result = await kernel.exec('sqlite3 /tmp/meta.db ".tables"'); + expect(result.stdout).toContain('alpha'); + expect(result.stdout).toContain('beta'); + }); + + it('supports .schema meta-command', async () => { + const vfs = new SimpleVFS(); + await vfs.mkdir('/tmp', { recursive: true }); + kernel = createKernel({ filesystem: vfs as any }); + await kernel.mount(createWasmVmRuntime({ commandDirs: SQLITE3_COMMAND_DIRS })); + + const createResult = await kernel.exec( + 'sqlite3 /tmp/schema.db "CREATE TABLE users(id INTEGER PRIMARY KEY, name TEXT NOT NULL);"' + ); + expect(createResult.stderr).toBe(''); + + const result = await kernel.exec('sqlite3 /tmp/schema.db ".schema users"'); + expect(result.stdout).toContain('CREATE TABLE users'); + expect(result.stdout).toContain('id INTEGER PRIMARY KEY'); + }); + + it('supports .dump meta-command', async () => { + const vfs = new SimpleVFS(); + await vfs.mkdir('/tmp', { recursive: true }); + kernel = createKernel({ filesystem: vfs as any }); + await kernel.mount(createWasmVmRuntime({ commandDirs: SQLITE3_COMMAND_DIRS })); + + const createResult = await kernel.exec( + `sqlite3 /tmp/dump.db "CREATE TABLE t(x INTEGER, y TEXT); INSERT INTO t VALUES(1,'hello');"` + ); + expect(createResult.stderr).toBe(''); + + const result = await kernel.exec('sqlite3 /tmp/dump.db ".dump"'); + const output = result.stdout; + expect(output).toContain('BEGIN TRANSACTION'); + expect(output).toContain('CREATE TABLE t'); + expect(output).toContain("INSERT INTO t VALUES(1,'hello')"); + expect(output).toContain('COMMIT'); + }); + + it('handles SELECT with multiple columns', async () => { + const vfs = new SimpleVFS(); + kernel = createKernel({ filesystem: vfs as any }); + await kernel.mount(createWasmVmRuntime({ commandDirs: SQLITE3_COMMAND_DIRS })); + + const result = await kernel.exec('sqlite3 :memory:', { + stdin: "SELECT 'hello' AS greeting, 42 AS number, 3.14 AS pi;\n", + }); + expect(result.stdout.trim()).toBe('hello|42|3.14'); + }); + + it('handles NULL values in output', async () => { + const vfs = new SimpleVFS(); + kernel = createKernel({ filesystem: vfs as any }); + await kernel.mount(createWasmVmRuntime({ commandDirs: SQLITE3_COMMAND_DIRS })); + + const result = await kernel.exec('sqlite3 :memory:', { + stdin: 'SELECT NULL;\n', + }); + // SQLite CLI outputs empty string for NULL + expect(result.stdout.trim()).toBe(''); + }); + + it('reports SQL errors on stderr', async () => { + const vfs = new SimpleVFS(); + kernel = createKernel({ filesystem: vfs as any }); + await kernel.mount(createWasmVmRuntime({ commandDirs: SQLITE3_COMMAND_DIRS })); + + const result = await kernel.exec(`sqlite3 :memory: "SELECT * FROM nonexistent_table;"`); + expect(result.stderr).toContain('no such table'); + }); + + it('defaults to :memory: when no database specified', async () => { + const vfs = new SimpleVFS(); + kernel = createKernel({ filesystem: vfs as any }); + await kernel.mount(createWasmVmRuntime({ commandDirs: SQLITE3_COMMAND_DIRS })); + + const result = await kernel.exec('sqlite3', { + stdin: 'SELECT 99;\n', + }); + expect(result.stdout.trim()).toBe('99'); + }); + + it('CREATE TABLE, INSERT, SELECT roundtrip via piped SQL', async () => { + const vfs = new SimpleVFS(); + kernel = createKernel({ filesystem: vfs as any }); + await kernel.mount(createWasmVmRuntime({ commandDirs: SQLITE3_COMMAND_DIRS })); + + // Multi-statement SQL via command arg (stdin multi-statement has fgetc buffering issues in WASM) + const sql = "CREATE TABLE items(id INTEGER PRIMARY KEY, name TEXT); INSERT INTO items VALUES(1,'apple'); INSERT INTO items VALUES(2,'banana'); SELECT id, name FROM items ORDER BY id;"; + const result = await kernel.exec(`sqlite3 :memory: "${sql}"`); + expect(result.stdout.trim()).toBe('1|apple\n2|banana'); + }); + + it('.tables meta-command lists created tables', async () => { + const vfs = new SimpleVFS(); + await vfs.mkdir('/tmp', { recursive: true }); + kernel = createKernel({ filesystem: vfs as any }); + await kernel.mount(createWasmVmRuntime({ commandDirs: SQLITE3_COMMAND_DIRS })); + + const createResult = await kernel.exec( + 'sqlite3 /tmp/tables.db "CREATE TABLE alpha(x); CREATE TABLE beta(y);"' + ); + expect(createResult.stderr).toBe(''); + + const result = await kernel.exec('sqlite3 /tmp/tables.db ".tables"'); + expect(result.stdout).toContain('alpha'); + expect(result.stdout).toContain('beta'); + }); + + it('.schema meta-command shows CREATE TABLE statements', async () => { + const vfs = new SimpleVFS(); + await vfs.mkdir('/tmp', { recursive: true }); + kernel = createKernel({ filesystem: vfs as any }); + await kernel.mount(createWasmVmRuntime({ commandDirs: SQLITE3_COMMAND_DIRS })); + + const createResult = await kernel.exec( + 'sqlite3 /tmp/schema-meta.db "CREATE TABLE users(id INTEGER PRIMARY KEY, name TEXT NOT NULL);"' + ); + expect(createResult.stderr).toBe(''); + + const result = await kernel.exec('sqlite3 /tmp/schema-meta.db ".schema users"'); + expect(result.stdout).toContain('CREATE TABLE users'); + expect(result.stdout).toContain('id INTEGER PRIMARY KEY'); + }); + + it('.dump meta-command outputs INSERT statements for data', async () => { + const vfs = new SimpleVFS(); + await vfs.mkdir('/tmp', { recursive: true }); + kernel = createKernel({ filesystem: vfs as any }); + await kernel.mount(createWasmVmRuntime({ commandDirs: SQLITE3_COMMAND_DIRS })); + + const createResult = await kernel.exec( + `sqlite3 /tmp/dump-meta.db "CREATE TABLE t(x INTEGER, y TEXT); INSERT INTO t VALUES(1,'hello'); INSERT INTO t VALUES(2,'world');"` + ); + expect(createResult.stderr).toBe(''); + + const result = await kernel.exec('sqlite3 /tmp/dump-meta.db ".dump"'); + const output = result.stdout; + expect(output).toContain('BEGIN TRANSACTION'); + expect(output).toContain('CREATE TABLE t'); + expect(output).toContain("INSERT INTO t VALUES(1,'hello')"); + expect(output).toContain("INSERT INTO t VALUES(2,'world')"); + expect(output).toContain('COMMIT'); + }); + + it('file-based DB persists data across separate exec calls', async () => { + const vfs = new SimpleVFS(); + await vfs.mkdir('/tmp', { recursive: true }); + kernel = createKernel({ filesystem: vfs as any }); + await kernel.mount(createWasmVmRuntime({ commandDirs: SQLITE3_COMMAND_DIRS })); + + // File-backed SQLite must persist through the kernel VFS across exec calls. + const createSql = "CREATE TABLE t(x INTEGER); INSERT INTO t VALUES(42); INSERT INTO t VALUES(99);"; + const createResult = await kernel.exec(`sqlite3 /tmp/test.db "${createSql}"`); + expect(createResult.stderr).toBe(''); + + // Verify file was created in VFS + const dbData = await vfs.readFile('/tmp/test.db'); + expect(dbData.length).toBeGreaterThan(0); + + // Second exec: reopen and query persisted data. + const result = await kernel.exec('sqlite3 /tmp/test.db "SELECT * FROM t ORDER BY x;"'); + expect(result.stdout.trim()).toBe('42\n99'); + }); + + it('multi-statement input separated by semicolons', async () => { + const vfs = new SimpleVFS(); + kernel = createKernel({ filesystem: vfs as any }); + await kernel.mount(createWasmVmRuntime({ commandDirs: SQLITE3_COMMAND_DIRS })); + + // Multi-statement SQL via command arg (semicolons separate statements) + const sql = "CREATE TABLE nums(v); INSERT INTO nums VALUES(10); INSERT INTO nums VALUES(20); SELECT v FROM nums ORDER BY v;"; + const result = await kernel.exec(`sqlite3 :memory: "${sql}"`); + expect(result.stdout.trim()).toBe('10\n20'); + }); + + it('SQL syntax error produces error on stderr with non-zero exit', async () => { + const vfs = new SimpleVFS(); + kernel = createKernel({ filesystem: vfs as any }); + await kernel.mount(createWasmVmRuntime({ commandDirs: SQLITE3_COMMAND_DIRS })); + + // Syntax error via command arg (reliable error output) + const result = await kernel.exec('sqlite3 :memory: "SELEC INVALID SYNTAX;"'); + expect(result.stderr).toContain('Error'); + }); +}); diff --git a/software/sqlite3/tsconfig.json b/software/sqlite3/tsconfig.json new file mode 100644 index 0000000000..03ce790ab7 --- /dev/null +++ b/software/sqlite3/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*"] +} diff --git a/software/tar/agentos-package.json b/software/tar/agentos-package.json new file mode 100644 index 0000000000..94501679ac --- /dev/null +++ b/software/tar/agentos-package.json @@ -0,0 +1,12 @@ +{ + "commands": [ + "tar" + ], + "registry": { + "title": "tar", + "description": "GNU tar archiver.", + "priority": 60, + "image": "/images/registry/tar.svg", + "category": "core" + } +} diff --git a/software/tar/native/crates/cmd-tar/Cargo.toml b/software/tar/native/crates/cmd-tar/Cargo.toml new file mode 100644 index 0000000000..a5ca3d0b4b --- /dev/null +++ b/software/tar/native/crates/cmd-tar/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-tar" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "tar standalone binary for secure-exec VM" + +[[bin]] +name = "tar" +path = "src/main.rs" + +[dependencies] +secureexec-tar = { path = "../tar" } diff --git a/registry/native/crates/commands/tar/src/main.rs b/software/tar/native/crates/cmd-tar/src/main.rs similarity index 100% rename from registry/native/crates/commands/tar/src/main.rs rename to software/tar/native/crates/cmd-tar/src/main.rs diff --git a/software/tar/native/crates/tar/Cargo.toml b/software/tar/native/crates/tar/Cargo.toml new file mode 100644 index 0000000000..f10f711e29 --- /dev/null +++ b/software/tar/native/crates/tar/Cargo.toml @@ -0,0 +1,11 @@ +[package] +workspace = "../../../../../toolchain" +name = "secureexec-tar" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "tar implementation for secure-exec standalone binaries" + +[dependencies] +flate2 = { version = "1.0", default-features = false, features = ["rust_backend"] } +tar = "0.4" diff --git a/software/tar/native/crates/tar/src/lib.rs b/software/tar/native/crates/tar/src/lib.rs new file mode 100644 index 0000000000..db039cff45 --- /dev/null +++ b/software/tar/native/crates/tar/src/lib.rs @@ -0,0 +1,753 @@ +//! tar implementation — create, extract, and list tape archives. +//! +//! Supports -c create, -x extract, -t list. +//! Options: -f archive, -z gzip, -v verbose, -C directory, --strip-components=N. + +use std::collections::HashSet; +use std::ffi::OsString; +use std::fs::{self, File, OpenOptions}; +use std::io::{self, Read, Write}; +use std::path::{Component, Path, PathBuf}; + +use flate2::read::GzDecoder; +use flate2::write::GzEncoder; +use flate2::Compression; + +const MAX_ARCHIVE_ENTRIES: usize = 100_000; +const MAX_CREATE_DEPTH: usize = 256; +const MAX_DIRECTORY_ENTRIES: usize = 100_000; + +#[derive(PartialEq)] +enum Mode { + None, + Create, + Extract, + List, +} + +/// Unified tar entry point. +pub fn main(args: Vec) -> i32 { + let str_args: Vec = args + .iter() + .skip(1) + .map(|a| a.to_string_lossy().to_string()) + .collect(); + + if str_args.is_empty() { + eprintln!("tar: must specify one of -c, -x, -t"); + return 1; + } + + let mut mode = Mode::None; + let mut archive_file: Option = None; + let mut gzip = false; + let mut verbose = false; + let mut directory: Option = None; + let mut strip_components: usize = 0; + let mut paths: Vec = Vec::new(); + + let mut i = 0; + let mut first_arg = true; + + while i < str_args.len() { + let arg = &str_args[i]; + + if arg.starts_with("--strip-components=") { + if let Ok(n) = arg["--strip-components=".len()..].parse() { + strip_components = n; + } + first_arg = false; + } else if arg == "--strip-components" { + i += 1; + if i < str_args.len() { + strip_components = str_args[i].parse().unwrap_or(0); + } + first_arg = false; + } else if arg == "-C" || arg == "--directory" { + i += 1; + if i < str_args.len() { + directory = Some(str_args[i].clone()); + } + first_arg = false; + } else if arg == "--help" { + print_usage(); + return 0; + } else if arg.starts_with('-') || first_arg { + // tar's first argument can omit the leading dash (e.g., `tar czf`) + let flags = if arg.starts_with('-') { + &arg[1..] + } else { + &arg[..] + }; + let mut chars = flags.chars().peekable(); + while let Some(ch) = chars.next() { + match ch { + 'c' => mode = Mode::Create, + 'x' => mode = Mode::Extract, + 't' => mode = Mode::List, + 'z' => gzip = true, + 'v' => verbose = true, + 'f' => { + let rest: String = chars.collect(); + if !rest.is_empty() { + archive_file = Some(rest); + } else { + i += 1; + if i < str_args.len() { + archive_file = Some(str_args[i].clone()); + } + } + break; + } + 'C' => { + let rest: String = chars.collect(); + if !rest.is_empty() { + directory = Some(rest); + } else { + i += 1; + if i < str_args.len() { + directory = Some(str_args[i].clone()); + } + } + break; + } + _ => { + eprintln!("tar: unknown option: {}", ch); + return 1; + } + } + } + first_arg = false; + } else { + paths.push(arg.clone()); + first_arg = false; + } + + i += 1; + } + + // Auto-detect gzip from filename + if !gzip { + if let Some(ref f) = archive_file { + if f.ends_with(".tar.gz") || f.ends_with(".tgz") { + gzip = true; + } + } + } + + let result = match mode { + Mode::Create => do_create( + archive_file.as_deref(), + gzip, + verbose, + directory.as_deref(), + &paths, + ), + Mode::Extract => do_extract( + archive_file.as_deref(), + gzip, + verbose, + directory.as_deref(), + strip_components, + ), + Mode::List => do_list(archive_file.as_deref(), gzip, verbose), + Mode::None => { + eprintln!("tar: must specify one of -c, -x, -t"); + return 1; + } + }; + + match result { + Ok(()) => 0, + Err(e) => { + eprintln!("tar: {}", e); + 1 + } + } +} + +fn do_create( + archive_file: Option<&str>, + gzip: bool, + verbose: bool, + directory: Option<&str>, + paths: &[String], +) -> io::Result<()> { + if paths.is_empty() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "cowardly refusing to create an empty archive", + )); + } + + match archive_file { + Some("-") | None => create_to_writer(io::stdout(), gzip, verbose, directory, paths), + Some(path) => { + let mut archive = BoundedVecWriter::new(512 * 1024 * 1024); + create_to_writer(&mut archive, gzip, verbose, directory, paths)?; + fs::write(path, archive.into_inner()) + } + } +} + +struct BoundedVecWriter { + bytes: Vec, + capacity: usize, +} + +impl BoundedVecWriter { + fn new(capacity: usize) -> Self { + Self { + bytes: Vec::new(), + capacity, + } + } + + fn into_inner(self) -> Vec { + self.bytes + } +} + +impl Write for BoundedVecWriter { + fn write(&mut self, buf: &[u8]) -> io::Result { + let Some(new_len) = self.bytes.len().checked_add(buf.len()) else { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "archive is too large to buffer", + )); + }; + if new_len > self.capacity { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "archive is too large to buffer", + )); + } + self.bytes.extend_from_slice(buf); + Ok(buf.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +fn create_to_writer( + writer: W, + gzip: bool, + verbose: bool, + directory: Option<&str>, + paths: &[String], +) -> io::Result<()> { + if gzip { + let encoder = GzEncoder::new(writer, Compression::default()); + let mut builder = tar::Builder::new(encoder); + append_paths(&mut builder, directory, paths, verbose)?; + let encoder = builder.into_inner()?; + let mut writer = encoder.finish()?; + writer.flush() + } else { + let mut builder = tar::Builder::new(writer); + append_paths(&mut builder, directory, paths, verbose)?; + let mut writer = builder.into_inner()?; + writer.flush() + } +} + +fn append_paths( + builder: &mut tar::Builder, + directory: Option<&str>, + paths: &[String], + verbose: bool, +) -> io::Result<()> { + let mut entry_count = 0; + for path in paths { + append_path( + builder, + resolve_disk_path(directory, Path::new(path)), + Path::new(path), + verbose, + 0, + &mut entry_count, + )?; + } + Ok(()) +} + +fn append_path( + builder: &mut tar::Builder, + disk_path: PathBuf, + archive_path: &Path, + verbose: bool, + depth: usize, + entry_count: &mut usize, +) -> io::Result<()> { + if depth > MAX_CREATE_DEPTH { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!( + "maximum directory depth exceeded at {}", + disk_path.display() + ), + )); + } + increment_entry_count(entry_count)?; + + let meta = fs::symlink_metadata(&disk_path).map_err(|e| { + io::Error::new(e.kind(), format!("metadata {}: {}", disk_path.display(), e)) + })?; + + if meta.is_dir() { + append_dir( + builder, + &disk_path, + archive_path, + verbose, + depth, + entry_count, + )?; + } else if meta.is_file() { + if verbose { + eprintln!("{}", archive_path.display()); + } + let mut header = tar::Header::new_gnu(); + header.set_entry_type(tar::EntryType::Regular); + header.set_size(meta.len()); + header.set_mode(0o644); + header.set_cksum(); + let mut file = File::open(&disk_path)?; + builder.append_data(&mut header, archive_path, &mut file)?; + } else if meta.file_type().is_symlink() { + if verbose { + eprintln!("{}", archive_path.display()); + } + let target = fs::read_link(&disk_path)?; + let mut header = tar::Header::new_gnu(); + header.set_entry_type(tar::EntryType::Symlink); + header.set_size(0); + header.set_mode(0o777); + header.set_cksum(); + builder.append_link(&mut header, archive_path, &target)?; + } + + Ok(()) +} + +fn append_dir( + builder: &mut tar::Builder, + disk_dir: &Path, + archive_dir: &Path, + verbose: bool, + depth: usize, + entry_count: &mut usize, +) -> io::Result<()> { + if verbose { + eprintln!("{}/", archive_dir.display()); + } + + let mut header = tar::Header::new_gnu(); + header.set_entry_type(tar::EntryType::Directory); + header.set_size(0); + header.set_mode(0o755); + header.set_cksum(); + builder.append_data(&mut header, archive_dir, io::empty())?; + + let mut dir_entries = 0; + for entry_result in fs::read_dir(disk_dir)? { + let entry = entry_result?; + dir_entries += 1; + if dir_entries > MAX_DIRECTORY_ENTRIES { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("too many entries in {}", disk_dir.display()), + )); + } + let archive_child = archive_dir.join(entry.file_name()); + append_path( + builder, + entry.path(), + &archive_child, + verbose, + depth + 1, + entry_count, + )?; + } + + Ok(()) +} + +fn do_extract( + archive_file: Option<&str>, + gzip: bool, + verbose: bool, + directory: Option<&str>, + strip_components: usize, +) -> io::Result<()> { + let reader = open_read(archive_file, gzip)?; + let mut archive = tar::Archive::new(reader); + let mut known_dirs = HashSet::new(); + if let Some(base) = directory { + validate_extract_base(Path::new(base))?; + known_dirs.insert(PathBuf::from(base)); + } + + let mut entry_count = 0; + for entry_result in archive.entries()? { + increment_entry_count(&mut entry_count)?; + let mut entry = entry_result?; + let orig_path = entry.path()?.into_owned(); + validate_archive_input_path(&orig_path)?; + + let relative_dest = match strip_path_components(&orig_path, strip_components) { + Some(p) if !p.as_os_str().is_empty() => p, + _ => continue, + }; + validate_relative_output_path(&relative_dest)?; + validate_extract_depth(&relative_dest)?; + let dest = resolve_output_path(directory, &relative_dest); + + if verbose { + eprintln!("{}", orig_path.display()); + } + + match entry.header().entry_type() { + tar::EntryType::Directory => { + ensure_relative_dir_exists(directory, &relative_dest, &mut known_dirs)?; + } + tar::EntryType::Regular | tar::EntryType::GNUSparse => { + if let Some(parent) = dest.parent() { + if !parent.as_os_str().is_empty() { + let relative_parent = + relative_dest.parent().unwrap_or_else(|| Path::new("")); + ensure_relative_dir_exists(directory, relative_parent, &mut known_dirs)?; + } + } + reject_existing_symlink(&dest)?; + let mut output = OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(&dest) + .map_err(|e| { + io::Error::new(e.kind(), format!("open {}: {}", dest.display(), e)) + })?; + io::copy(&mut entry, &mut output).map_err(|e| { + io::Error::new(e.kind(), format!("write {}: {}", dest.display(), e)) + })?; + output.flush().map_err(|e| { + io::Error::new(e.kind(), format!("write {}: {}", dest.display(), e)) + })?; + } + tar::EntryType::Symlink => { + if let Some(target) = entry.link_name()? { + validate_symlink_target(target.as_ref())?; + if let Some(parent) = dest.parent() { + if !parent.as_os_str().is_empty() { + let relative_parent = + relative_dest.parent().unwrap_or_else(|| Path::new("")); + ensure_relative_dir_exists( + directory, + relative_parent, + &mut known_dirs, + )?; + } + } + reject_existing_symlink(&dest)?; + #[allow(deprecated)] + std::fs::soft_link(target.as_ref(), &dest).map_err(|e| { + io::Error::new(e.kind(), format!("symlink {}: {}", dest.display(), e)) + })?; + } + } + _ => { + // Skip hard links, char/block devices, etc. + } + } + } + + Ok(()) +} + +fn resolve_disk_path(directory: Option<&str>, path: &Path) -> PathBuf { + match directory { + Some(base) if !path.is_absolute() => Path::new(base).join(path), + _ => path.to_path_buf(), + } +} + +fn resolve_output_path(directory: Option<&str>, path: &Path) -> PathBuf { + match directory { + Some(base) if path.is_relative() => Path::new(base).join(path), + _ => path.to_path_buf(), + } +} + +fn ensure_relative_dir_exists( + directory: Option<&str>, + relative_path: &Path, + known_dirs: &mut HashSet, +) -> io::Result<()> { + let mut current = match directory { + Some(base) => PathBuf::from(base), + None => PathBuf::new(), + }; + + for component in relative_path.components() { + match component { + Component::CurDir => {} + Component::Normal(part) => { + current.push(part); + if known_dirs.contains(¤t) { + continue; + } + match fs::create_dir(¤t) { + Ok(()) => { + known_dirs.insert(current.clone()); + } + Err(err) if err.kind() == io::ErrorKind::AlreadyExists => { + let metadata = fs::symlink_metadata(¤t).map_err(|metadata_err| { + io::Error::new( + metadata_err.kind(), + format!("metadata {}: {}", current.display(), metadata_err), + ) + })?; + if metadata.file_type().is_symlink() || !metadata.is_dir() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("refusing to extract through {}", current.display()), + )); + } + known_dirs.insert(current.clone()); + } + Err(err) => { + return Err(io::Error::new( + err.kind(), + format!("create_dir {}: {}", current.display(), err), + )); + } + } + } + Component::Prefix(_) | Component::RootDir | Component::ParentDir => { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("unsupported path component in {}", relative_path.display()), + )); + } + } + } + + Ok(()) +} + +fn do_list(archive_file: Option<&str>, gzip: bool, verbose: bool) -> io::Result<()> { + let reader = open_read(archive_file, gzip)?; + let mut archive = tar::Archive::new(reader); + let stdout = io::stdout(); + let mut out = stdout.lock(); + + let mut entry_count = 0; + for entry_result in archive.entries()? { + increment_entry_count(&mut entry_count)?; + let entry = entry_result?; + let path = entry.path()?; + + let entry_type = entry.header().entry_type(); + if verbose { + let h = entry.header(); + let size = h.size().unwrap_or(0); + let mode = h.mode().unwrap_or(0o644); + let type_ch = match entry_type { + tar::EntryType::Directory => 'd', + tar::EntryType::Symlink => 'l', + _ => '-', + }; + writeln!( + out, + "{}{} {:>8} {}", + type_ch, + format_mode(mode), + size, + path.display() + )?; + } else if entry_type == tar::EntryType::Directory { + writeln!(out, "{}/", path.display())?; + } else { + writeln!(out, "{}", path.display())?; + } + } + + out.flush() +} + +fn open_read(archive_file: Option<&str>, gzip: bool) -> io::Result> { + let reader: Box = match archive_file { + Some("-") | None => Box::new(io::stdin()), + Some(path) => Box::new(File::open(path)?), + }; + + if gzip { + Ok(Box::new(GzDecoder::new(reader))) + } else { + Ok(reader) + } +} + +fn strip_path_components(path: &Path, n: usize) -> Option { + let mut remaining = n; + let mut stripped = PathBuf::new(); + + for component in path.components() { + match component { + Component::Prefix(_) | Component::RootDir | Component::CurDir => {} + Component::ParentDir => { + if remaining == 0 { + stripped.push(component.as_os_str()); + } + } + Component::Normal(part) => { + if remaining > 0 { + remaining -= 1; + } else { + stripped.push(part); + } + } + } + } + + if stripped.as_os_str().is_empty() { + None + } else { + Some(stripped) + } +} + +fn increment_entry_count(count: &mut usize) -> io::Result<()> { + *count += 1; + if *count > MAX_ARCHIVE_ENTRIES { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "too many archive entries", + )); + } + Ok(()) +} + +fn validate_relative_output_path(path: &Path) -> io::Result<()> { + for component in path.components() { + match component { + Component::Normal(_) | Component::CurDir => {} + Component::Prefix(_) | Component::RootDir | Component::ParentDir => { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("refusing to extract unsafe path {}", path.display()), + )); + } + } + } + Ok(()) +} + +fn validate_archive_input_path(path: &Path) -> io::Result<()> { + for component in path.components() { + match component { + Component::Normal(_) | Component::CurDir => {} + Component::Prefix(_) | Component::RootDir | Component::ParentDir => { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("refusing to extract unsafe path {}", path.display()), + )); + } + } + } + Ok(()) +} + +fn validate_extract_depth(path: &Path) -> io::Result<()> { + let depth = path + .components() + .filter(|component| matches!(component, Component::Normal(_))) + .count(); + if depth > MAX_CREATE_DEPTH { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("maximum extraction depth exceeded at {}", path.display()), + )); + } + Ok(()) +} + +fn validate_extract_base(path: &Path) -> io::Result<()> { + let metadata = fs::symlink_metadata(path).map_err(|err| { + io::Error::new(err.kind(), format!("metadata {}: {}", path.display(), err)) + })?; + if metadata.file_type().is_symlink() || !metadata.is_dir() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("refusing to extract through {}", path.display()), + )); + } + Ok(()) +} + +fn validate_symlink_target(target: &Path) -> io::Result<()> { + for component in target.components() { + match component { + Component::Normal(_) | Component::CurDir => {} + Component::Prefix(_) | Component::RootDir | Component::ParentDir => { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!( + "refusing to extract unsafe symlink target {}", + target.display() + ), + )); + } + } + } + Ok(()) +} + +fn reject_existing_symlink(path: &Path) -> io::Result<()> { + match fs::symlink_metadata(path) { + Ok(metadata) if metadata.file_type().is_symlink() => Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("refusing to overwrite symlink {}", path.display()), + )), + Ok(_) => Ok(()), + Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(()), + Err(err) => Err(io::Error::new( + err.kind(), + format!("metadata {}: {}", path.display(), err), + )), + } +} + +fn format_mode(mode: u32) -> String { + let mut s = String::with_capacity(9); + for &(bit, ch) in &[ + (0o400, 'r'), + (0o200, 'w'), + (0o100, 'x'), + (0o040, 'r'), + (0o020, 'w'), + (0o010, 'x'), + (0o004, 'r'), + (0o002, 'w'), + (0o001, 'x'), + ] { + s.push(if mode & bit != 0 { ch } else { '-' }); + } + s +} + +fn print_usage() { + eprintln!("Usage: tar [options] [files...]"); + eprintln!(" -c create archive"); + eprintln!(" -x extract archive"); + eprintln!(" -t list archive contents"); + eprintln!(" -f FILE archive filename (- for stdin/stdout)"); + eprintln!(" -z gzip compress/decompress"); + eprintln!(" -v verbose"); + eprintln!(" -C DIR change directory"); + eprintln!(" --strip-components=N"); + eprintln!(" strip N leading components on extract"); +} diff --git a/software/tar/package.json b/software/tar/package.json new file mode 100644 index 0000000000..97f1e92da1 --- /dev/null +++ b/software/tar/package.json @@ -0,0 +1,33 @@ +{ + "name": "@agentos-software/tar", + "version": "0.3.3", + "type": "module", + "license": "Apache-2.0", + "description": "GNU tar archiver for secure-exec VMs", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "files": [ + "dist", + "!dist/package", + "!dist/package.tar" + ], + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "scripts": { + "build": "agentos-toolchain stage --commands-dir ../../toolchain/target/wasm32-wasip1/release/commands --if-missing skip && tsc && agentos-toolchain build", + "check-types": "tsc --noEmit", + "test": "vitest run test/ --passWithNoTests" + }, + "devDependencies": { + "@agentos-software/manifest": "workspace:*", + "@rivet-dev/agentos-toolchain": "workspace:*", + "@types/node": "^22.10.2", + "typescript": "^5.9.2", + "@agentos/test-harness": "workspace:*", + "vitest": "^2.1.9" + } +} diff --git a/registry/software/tar/src/index.ts b/software/tar/src/index.ts similarity index 100% rename from registry/software/tar/src/index.ts rename to software/tar/src/index.ts diff --git a/software/tar/test/manifest.test.ts b/software/tar/test/manifest.test.ts new file mode 100644 index 0000000000..89139d33b8 --- /dev/null +++ b/software/tar/test/manifest.test.ts @@ -0,0 +1,19 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +const packageDir = new URL("..", import.meta.url).pathname; + +describe("package manifest", () => { + it("declares command binaries", () => { + const manifest = JSON.parse( + readFileSync(join(packageDir, "agentos-package.json"), "utf8"), + ); + + expect(manifest.commands?.length ?? 0).toBeGreaterThan(0); + for (const command of manifest.commands) { + expect(typeof command).toBe("string"); + expect(command.length).toBeGreaterThan(0); + } + }); +}); diff --git a/software/tar/test/tar.test.ts b/software/tar/test/tar.test.ts new file mode 100644 index 0000000000..adebae14c4 --- /dev/null +++ b/software/tar/test/tar.test.ts @@ -0,0 +1,165 @@ +import { existsSync } from "node:fs"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { + NodeFileSystem, + createKernel, + createWasmVmRuntime, + describeIf, +} from "@agentos/test-harness"; +import type { Kernel } from "@agentos/test-harness"; +import { afterEach, expect, it } from "vitest"; + +const TAR_COMMAND_DIR = fileURLToPath(new URL("../bin", import.meta.url)); +const hasTarPackageBinary = existsSync(join(TAR_COMMAND_DIR, "tar")); + +let tempRoot: string | undefined; + +function hostPath(path: string): string { + if (!tempRoot) throw new Error("fixture root not initialized"); + return join(tempRoot, path.replace(/^\/+/, "")); +} + +async function writeFixture(path: string, contents: string): Promise { + const target = hostPath(path); + await mkdir(dirname(target), { recursive: true }); + await writeFile(target, contents); +} + +async function createTestVFS(): Promise { + tempRoot = await mkdtemp(join(tmpdir(), "agentos-tar-")); + await writeFixture("/project/src/docs/readme.txt", "hello from docs\n"); + await writeFixture("/project/src/data/values.csv", "name,score\nAda,7\nLinus,3\n"); + await mkdir(hostPath("/project/out"), { recursive: true }); + await mkdir(hostPath("/project/strip"), { recursive: true }); + await mkdir(hostPath("/project/gzip-out"), { recursive: true }); + return new NodeFileSystem({ root: tempRoot }); +} + +function lines(stdout: string): string[] { + return stdout.split("\n").filter((line) => line.length > 0); +} + +const textDecoder = new TextDecoder(); + +describeIf(hasTarPackageBinary, "tar command", { timeout: 10_000 }, () => { + let kernel: Kernel | undefined; + + afterEach(async () => { + await kernel?.dispose(); + kernel = undefined; + if (tempRoot) { + await rm(tempRoot, { recursive: true, force: true }); + tempRoot = undefined; + } + }); + + async function mountFixture(): Promise { + const vfs = await createTestVFS(); + kernel = createKernel({ filesystem: vfs }); + await kernel.mount(createWasmVmRuntime({ commandDirs: [TAR_COMMAND_DIR] })); + } + + async function runTar(args: string[]) { + if (!kernel) throw new Error("kernel not mounted"); + let stdout = ""; + let stderr = ""; + const proc = kernel.spawn("tar", args, { + onStdout: (chunk) => { + stdout += Buffer.from(chunk).toString("utf8"); + }, + onStderr: (chunk) => { + stderr += Buffer.from(chunk).toString("utf8"); + }, + }); + const exitCode = await proc.wait(); + await new Promise((resolve) => setTimeout(resolve, 0)); + return { stdout, stderr, exitCode }; + } + + async function createArchive(path = "/project/archive.tar") { + const result = await runTar(["-cf", path, "-C", "/project", "src"]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + } + + async function readGuestText(path: string): Promise { + if (!kernel) throw new Error("kernel not mounted"); + return textDecoder.decode(await kernel.readFile(path)); + } + + it("creates and lists file-backed archives", async () => { + await mountFixture(); + await createArchive(); + + const result = await runTar(["-tf", "/project/archive.tar"]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(lines(result.stdout)).toEqual( + expect.arrayContaining([ + "src/", + "src/docs/", + "src/docs/readme.txt", + "src/data/", + "src/data/values.csv", + ]), + ); + }); + + it("extracts archives into target directories", async () => { + await mountFixture(); + await createArchive(); + + const result = await runTar(["-xf", "/project/archive.tar", "-C", "/project/out"]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + await expect(readGuestText("/project/out/src/docs/readme.txt")).resolves.toBe( + "hello from docs\n", + ); + await expect(readGuestText("/project/out/src/data/values.csv")).resolves.toBe( + "name,score\nAda,7\nLinus,3\n", + ); + }); + + it("auto-detects gzip archives by extension", async () => { + await mountFixture(); + + const create = await runTar(["-czf", "/project/archive.tgz", "-C", "/project", "src"]); + expect(create.exitCode, create.stderr || create.stdout).toBe(0); + + const list = await runTar(["-tf", "/project/archive.tgz"]); + expect(list.exitCode, list.stderr || list.stdout).toBe(0); + expect(lines(list.stdout)).toContain("src/docs/readme.txt"); + + const extract = await runTar(["-xf", "/project/archive.tgz", "-C", "/project/gzip-out"]); + expect(extract.exitCode, extract.stderr || extract.stdout).toBe(0); + await expect(readGuestText("/project/gzip-out/src/docs/readme.txt")).resolves.toBe( + "hello from docs\n", + ); + }); + + it("strips path components on extraction", async () => { + await mountFixture(); + await createArchive(); + + const result = await runTar([ + "-xf", + "/project/archive.tar", + "-C", + "/project/strip", + "--strip-components=1", + ]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + await expect(readGuestText("/project/strip/docs/readme.txt")).resolves.toBe( + "hello from docs\n", + ); + await expect(readGuestText("/project/strip/src/docs/readme.txt")).rejects.toThrow(); + }); + + it("fails when a create input is missing", async () => { + await mountFixture(); + + const result = await runTar(["-cf", "/project/missing.tar", "-C", "/project", "nope"]); + expect(result.exitCode).not.toBe(0); + expect(result.stderr).toContain("nope"); + }); +}); diff --git a/software/tar/tsconfig.json b/software/tar/tsconfig.json new file mode 100644 index 0000000000..03ce790ab7 --- /dev/null +++ b/software/tar/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*"] +} diff --git a/software/tree/agentos-package.json b/software/tree/agentos-package.json new file mode 100644 index 0000000000..9a3df6fcee --- /dev/null +++ b/software/tree/agentos-package.json @@ -0,0 +1,11 @@ +{ + "commands": [ + "tree" + ], + "registry": { + "title": "tree", + "description": "Display directory structure as a tree.", + "priority": 25, + "category": "core" + } +} diff --git a/software/tree/package.json b/software/tree/package.json new file mode 100644 index 0000000000..45d2aee6d2 --- /dev/null +++ b/software/tree/package.json @@ -0,0 +1,33 @@ +{ + "name": "@agentos-software/tree", + "version": "0.3.3", + "type": "module", + "license": "GPL-2.0-or-later", + "description": "tree directory listing for secure-exec VMs", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "files": [ + "dist", + "!dist/package", + "!dist/package.tar" + ], + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "scripts": { + "build": "agentos-toolchain stage --commands-dir ../../toolchain/target/wasm32-wasip1/release/commands --if-missing skip && tsc && agentos-toolchain build", + "check-types": "tsc --noEmit", + "test": "vitest run test/ --passWithNoTests" + }, + "devDependencies": { + "@agentos-software/manifest": "workspace:*", + "@rivet-dev/agentos-toolchain": "workspace:*", + "@types/node": "^22.10.2", + "typescript": "^5.9.2", + "@agentos/test-harness": "workspace:*", + "vitest": "^2.1.9" + } +} diff --git a/registry/software/tree/src/index.ts b/software/tree/src/index.ts similarity index 100% rename from registry/software/tree/src/index.ts rename to software/tree/src/index.ts diff --git a/registry/tests/kernel/tree-test.test.ts b/software/tree/test/tree-test.test.ts similarity index 96% rename from registry/tests/kernel/tree-test.test.ts rename to software/tree/test/tree-test.test.ts index 850b5e371a..0c505f5ff0 100644 --- a/registry/tests/kernel/tree-test.test.ts +++ b/software/tree/test/tree-test.test.ts @@ -10,8 +10,8 @@ import { describeIf, createIntegrationKernel, skipUnlessWasmBuilt, -} from './helpers.ts'; -import type { IntegrationKernelResult } from './helpers.ts'; +} from '@agentos/test-harness'; +import type { IntegrationKernelResult } from '@agentos/test-harness'; const wasmSkip = skipUnlessWasmBuilt(); const TREE_COMMAND_TIMEOUT_MS = 20_000; @@ -65,8 +65,8 @@ describeIf(!wasmSkip, 'tree command behavior', () => { expect(result.stdout).toContain('types.ts'); expect(result.stdout).toContain('index.ts'); expect(result.stdout).toContain('README.md'); - // Should show 2 directories (src, lib) and 4 files - expect(result.stdout).toMatch(/2 director/); + // Upstream tree reports the displayed root plus src/lib. + expect(result.stdout).toMatch(/3 director/); expect(result.stdout).toMatch(/4 file/); }, TREE_TEST_TIMEOUT_MS); diff --git a/software/tree/tsconfig.json b/software/tree/tsconfig.json new file mode 100644 index 0000000000..03ce790ab7 --- /dev/null +++ b/software/tree/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*"] +} diff --git a/registry/tsconfig.base.json b/software/tsconfig.base.json similarity index 100% rename from registry/tsconfig.base.json rename to software/tsconfig.base.json diff --git a/software/unzip/agentos-package.json b/software/unzip/agentos-package.json new file mode 100644 index 0000000000..2260eedf19 --- /dev/null +++ b/software/unzip/agentos-package.json @@ -0,0 +1,11 @@ +{ + "commands": [ + "unzip" + ], + "registry": { + "title": "unzip", + "description": "Extract zip archives.", + "priority": 10, + "category": "core" + } +} diff --git a/software/unzip/package.json b/software/unzip/package.json new file mode 100644 index 0000000000..9fcf15667a --- /dev/null +++ b/software/unzip/package.json @@ -0,0 +1,33 @@ +{ + "name": "@agentos-software/unzip", + "version": "0.3.3", + "type": "module", + "license": "Apache-2.0", + "description": "unzip archive extraction for secure-exec VMs", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "files": [ + "dist", + "!dist/package", + "!dist/package.tar" + ], + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "scripts": { + "build": "agentos-toolchain stage --commands-dir ../../toolchain/target/wasm32-wasip1/release/commands --if-missing skip && tsc && agentos-toolchain build", + "check-types": "tsc --noEmit", + "test": "vitest run test/ --passWithNoTests" + }, + "devDependencies": { + "@agentos-software/manifest": "workspace:*", + "@rivet-dev/agentos-toolchain": "workspace:*", + "@types/node": "^22.10.2", + "typescript": "^5.9.2", + "@agentos/test-harness": "workspace:*", + "vitest": "^2.1.9" + } +} diff --git a/registry/software/unzip/src/index.ts b/software/unzip/src/index.ts similarity index 100% rename from registry/software/unzip/src/index.ts rename to software/unzip/src/index.ts diff --git a/software/unzip/test/unzip.test.ts b/software/unzip/test/unzip.test.ts new file mode 100644 index 0000000000..b6081dc2e2 --- /dev/null +++ b/software/unzip/test/unzip.test.ts @@ -0,0 +1,218 @@ +/** + * Package-owned integration tests for the unzip command. + */ + +import { afterEach, expect, it } from "vitest"; +import { + C_BUILD_DIR, + COMMANDS_DIR, + createInMemoryFileSystem, + createKernel, + createWasmVmRuntime, + describeIf, + hasCWasmBinaries, + type Kernel, +} from "@agentos/test-harness"; + +interface HostileEntry { + name: string; + method: number; + compressedSize: number; + uncompressedSize: number; + localOffset: number; +} + +/** Builds a ZIP whose EOCD cd-size field is corrupt so unzip's raw central-directory fallback parser is exercised. */ +function buildFallbackArchive( + prefix: Uint8Array, + entries: HostileEntry[], +): Uint8Array { + const enc = new TextEncoder(); + const cdParts: Uint8Array[] = []; + for (const entry of entries) { + const nameBytes = enc.encode(entry.name); + const cd = new Uint8Array(46 + nameBytes.length); + const dv = new DataView(cd.buffer); + dv.setUint32(0, 0x02014b50, true); + dv.setUint16(4, 20, true); + dv.setUint16(6, 20, true); + dv.setUint16(10, entry.method, true); + dv.setUint32(20, entry.compressedSize, true); + dv.setUint32(24, entry.uncompressedSize, true); + dv.setUint16(28, nameBytes.length, true); + dv.setUint32(42, entry.localOffset, true); + cd.set(nameBytes, 46); + cdParts.push(cd); + } + const cdOffset = prefix.length; + const cdLen = cdParts.reduce((n, part) => n + part.length, 0); + const eocd = new Uint8Array(22); + const dv = new DataView(eocd.buffer); + dv.setUint32(0, 0x06054b50, true); + dv.setUint16(8, entries.length, true); + dv.setUint16(10, entries.length, true); + dv.setUint32(12, 0xffffffff, true); + dv.setUint32(16, cdOffset, true); + const out = new Uint8Array(prefix.length + cdLen + 22); + out.set(prefix, 0); + let offset = cdOffset; + for (const part of cdParts) { + out.set(part, offset); + offset += part.length; + } + out.set(eocd, offset); + return out; +} + +describeIf( + hasCWasmBinaries("zip", "unzip"), + "unzip command", + { timeout: 10_000 }, + () => { + let kernel: Kernel; + + afterEach(async () => { + await kernel?.dispose(); + }); + + it("extracts a zip archive into a target directory", async () => { + const vfs = createInMemoryFileSystem(); + await vfs.writeFile("/hello.txt", "Hello, World!\n"); + + kernel = createKernel({ filesystem: vfs }); + await kernel.mount( + createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] }), + ); + + const zipResult = await kernel.exec("zip /archive.zip /hello.txt"); + expect(zipResult.exitCode, zipResult.stderr).toBe(0); + + const unzipResult = await kernel.exec("unzip -d /extracted /archive.zip"); + expect(unzipResult.exitCode, unzipResult.stderr).toBe(0); + expect(await vfs.readTextFile("/extracted/hello.txt")).toBe( + "Hello, World!\n", + ); + }); + + it("lists archive contents with sizes", async () => { + const vfs = createInMemoryFileSystem(); + await vfs.writeFile("/data.txt", "some data content\n"); + + kernel = createKernel({ filesystem: vfs }); + await kernel.mount( + createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] }), + ); + + const zipResult = await kernel.exec("zip /list-test.zip /data.txt"); + expect(zipResult.exitCode, zipResult.stderr).toBe(0); + + const listResult = await kernel.exec("unzip -l /list-test.zip"); + expect(listResult.exitCode, listResult.stderr).toBe(0); + expect(listResult.stdout).toContain("data.txt"); + expect(listResult.stdout).toContain("18"); + expect(listResult.stdout).toMatch(/1 file/); + }); + + it("extracts binary file contents exactly", async () => { + const vfs = createInMemoryFileSystem(); + const content = new Uint8Array(256); + for (let i = 0; i < 256; i++) content[i] = i; + await vfs.writeFile("/binary.bin", content); + + kernel = createKernel({ filesystem: vfs }); + await kernel.mount( + createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] }), + ); + + const zipResult = await kernel.exec("zip /roundtrip.zip /binary.bin"); + expect(zipResult.exitCode, zipResult.stderr).toBe(0); + + const unzipResult = await kernel.exec("unzip -d /rt-out /roundtrip.zip"); + expect(unzipResult.exitCode, unzipResult.stderr).toBe(0); + + const extracted = await vfs.readFile("/rt-out/binary.bin"); + expect(extracted.length).toBe(256); + for (let i = 0; i < 256; i++) { + expect(extracted[i]).toBe(i); + } + }); + + it("rejects an entry with a wrapping local offset", async () => { + const vfs = createInMemoryFileSystem(); + const bytes = buildFallbackArchive(new Uint8Array(0), [ + { + name: "evil.txt", + method: 0, + compressedSize: 4, + uncompressedSize: 4, + localOffset: 0xfffffff0, + }, + ]); + await vfs.writeFile("/evil.zip", bytes); + + kernel = createKernel({ filesystem: vfs }); + await kernel.mount( + createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] }), + ); + + const result = await kernel.exec("unzip -d /out /evil.zip"); + expect(result.exitCode, result.stderr).not.toBe(0); + expect(result.stderr).toMatch(/error/); + expect(await vfs.exists("/out/evil.txt")).toBe(false); + }); + + it("rejects an entry whose normalized name is empty", async () => { + const vfs = createInMemoryFileSystem(); + const bytes = buildFallbackArchive(new Uint8Array(0), [ + { + name: "/", + method: 0, + compressedSize: 0, + uncompressedSize: 0, + localOffset: 0, + }, + ]); + await vfs.writeFile("/empty-name.zip", bytes); + + kernel = createKernel({ filesystem: vfs }); + await kernel.mount( + createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] }), + ); + + const result = await kernel.exec("unzip /empty-name.zip"); + expect(result.exitCode, result.stderr).not.toBe(0); + expect(result.stderr).toMatch(/error/); + }); + + it("rejects hostile uncompressed sizes before extracting", async () => { + const vfs = createInMemoryFileSystem(); + const prefix = new Uint8Array(31); + const pdv = new DataView(prefix.buffer); + pdv.setUint32(0, 0x04034b50, true); + pdv.setUint16(4, 20, true); + pdv.setUint16(26, 0, true); + pdv.setUint16(28, 0, true); + prefix[30] = 0x41; + const bytes = buildFallbackArchive(prefix, [ + { + name: "big.bin", + method: 0, + compressedSize: 1, + uncompressedSize: 0xffffffff, + localOffset: 0, + }, + ]); + await vfs.writeFile("/big.zip", bytes); + + kernel = createKernel({ filesystem: vfs }); + await kernel.mount( + createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] }), + ); + + const result = await kernel.exec("unzip -d /cap-out /big.zip"); + expect(result.exitCode, result.stderr).not.toBe(0); + expect(result.stderr).toMatch(/error/); + expect(await vfs.exists("/cap-out/big.bin")).toBe(false); + }); + }, +); diff --git a/software/unzip/tsconfig.json b/software/unzip/tsconfig.json new file mode 100644 index 0000000000..03ce790ab7 --- /dev/null +++ b/software/unzip/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*"] +} diff --git a/software/vim/agentos-package.json b/software/vim/agentos-package.json new file mode 100644 index 0000000000..801b9d8a53 --- /dev/null +++ b/software/vim/agentos-package.json @@ -0,0 +1,25 @@ +{ + "name": "vim", + "commands": [ + "vim" + ], + "provides": { + "env": { + "VIM": "/usr/local/share/vim", + "VIMRUNTIME": "/usr/local/share/vim/vim92" + }, + "files": [ + { + "source": "share/vim/vim92", + "target": "/usr/local/share/vim/vim92" + } + ] + }, + "registry": { + "title": "vim", + "description": "Vim text editor with bundled runtime.", + "priority": 65, + "image": "/images/registry/vim.svg", + "category": "editors" + } +} diff --git a/registry/native/c/vim/compat.h b/software/vim/native/c/vim-bridge/compat.h similarity index 100% rename from registry/native/c/vim/compat.h rename to software/vim/native/c/vim-bridge/compat.h diff --git a/registry/native/c/vim/grp.h b/software/vim/native/c/vim-bridge/grp.h similarity index 100% rename from registry/native/c/vim/grp.h rename to software/vim/native/c/vim-bridge/grp.h diff --git a/software/vim/native/c/vim-bridge/posix_stubs.c b/software/vim/native/c/vim-bridge/posix_stubs.c new file mode 100644 index 0000000000..2428b7beca --- /dev/null +++ b/software/vim/native/c/vim-bridge/posix_stubs.c @@ -0,0 +1,11 @@ +/* posix_stubs.c — stubs for full-OS libc gaps vim references but that the VM + * does not implement yet. Keep this narrow; the patched sysroot owns libc. */ +#include +#include +struct group *getgrent(void) { return NULL; } +void setgrent(void) {} +void endgrent(void) {} + +/* --- process/signal stubs (not used by core editing) --- */ +void __SIG_DFL(int s) { (void)s; } +int sigpending(void *set) { (void)set; return 0; } diff --git a/registry/native/c/vim/termcap_stub.c b/software/vim/native/c/vim-bridge/termcap_stub.c similarity index 100% rename from registry/native/c/vim/termcap_stub.c rename to software/vim/native/c/vim-bridge/termcap_stub.c diff --git a/registry/native/c/vim/termios.h b/software/vim/native/c/vim-bridge/termios.h similarity index 100% rename from registry/native/c/vim/termios.h rename to software/vim/native/c/vim-bridge/termios.h diff --git a/registry/native/c/vim/termios_bridge.c b/software/vim/native/c/vim-bridge/termios_bridge.c similarity index 100% rename from registry/native/c/vim/termios_bridge.c rename to software/vim/native/c/vim-bridge/termios_bridge.c diff --git a/software/vim/package.json b/software/vim/package.json new file mode 100644 index 0000000000..ae33f6fc96 --- /dev/null +++ b/software/vim/package.json @@ -0,0 +1,34 @@ +{ + "name": "@agentos-software/vim", + "version": "0.3.3", + "type": "module", + "license": "Apache-2.0", + "description": "vim for secure-exec VMs (wasm build + bundled runtime via provides)", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "files": [ + "dist", + "!dist/package", + "!dist/package.tar" + ], + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js", + "default": "./dist/index.js" + } + }, + "scripts": { + "build": "agentos-toolchain stage --commands-dir ../../toolchain/target/wasm32-wasip1/release/commands --if-missing skip && node ./scripts/stage-runtime.mjs && tsc && agentos-toolchain build", + "check-types": "tsc --noEmit", + "test": "vitest run test/ --passWithNoTests" + }, + "devDependencies": { + "@agentos-software/manifest": "workspace:*", + "@rivet-dev/agentos-toolchain": "workspace:*", + "@types/node": "^22.10.2", + "typescript": "^5.9.2", + "@agentos/test-harness": "workspace:*", + "vitest": "^2.1.9" + } +} diff --git a/registry/software/vim/scripts/stage-runtime.mjs b/software/vim/scripts/stage-runtime.mjs similarity index 100% rename from registry/software/vim/scripts/stage-runtime.mjs rename to software/vim/scripts/stage-runtime.mjs diff --git a/registry/software/vim/share/vim/vim92/autoload/RstFold.vim b/software/vim/share/vim/vim92/autoload/RstFold.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/autoload/RstFold.vim rename to software/vim/share/vim/vim92/autoload/RstFold.vim diff --git a/registry/software/vim/share/vim/vim92/autoload/ada.vim b/software/vim/share/vim/vim92/autoload/ada.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/autoload/ada.vim rename to software/vim/share/vim/vim92/autoload/ada.vim diff --git a/registry/software/vim/share/vim/vim92/autoload/adacomplete.vim b/software/vim/share/vim/vim92/autoload/adacomplete.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/autoload/adacomplete.vim rename to software/vim/share/vim/vim92/autoload/adacomplete.vim diff --git a/registry/software/vim/share/vim/vim92/autoload/bitbake.vim b/software/vim/share/vim/vim92/autoload/bitbake.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/autoload/bitbake.vim rename to software/vim/share/vim/vim92/autoload/bitbake.vim diff --git a/registry/software/vim/share/vim/vim92/autoload/ccomplete.vim b/software/vim/share/vim/vim92/autoload/ccomplete.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/autoload/ccomplete.vim rename to software/vim/share/vim/vim92/autoload/ccomplete.vim diff --git a/registry/software/vim/share/vim/vim92/autoload/clojurecomplete.vim b/software/vim/share/vim/vim92/autoload/clojurecomplete.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/autoload/clojurecomplete.vim rename to software/vim/share/vim/vim92/autoload/clojurecomplete.vim diff --git a/registry/software/vim/share/vim/vim92/autoload/context.vim b/software/vim/share/vim/vim92/autoload/context.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/autoload/context.vim rename to software/vim/share/vim/vim92/autoload/context.vim diff --git a/registry/software/vim/share/vim/vim92/autoload/contextcomplete.vim b/software/vim/share/vim/vim92/autoload/contextcomplete.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/autoload/contextcomplete.vim rename to software/vim/share/vim/vim92/autoload/contextcomplete.vim diff --git a/registry/software/vim/share/vim/vim92/autoload/csscomplete.vim b/software/vim/share/vim/vim92/autoload/csscomplete.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/autoload/csscomplete.vim rename to software/vim/share/vim/vim92/autoload/csscomplete.vim diff --git a/registry/software/vim/share/vim/vim92/autoload/decada.vim b/software/vim/share/vim/vim92/autoload/decada.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/autoload/decada.vim rename to software/vim/share/vim/vim92/autoload/decada.vim diff --git a/registry/software/vim/share/vim/vim92/autoload/freebasic.vim b/software/vim/share/vim/vim92/autoload/freebasic.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/autoload/freebasic.vim rename to software/vim/share/vim/vim92/autoload/freebasic.vim diff --git a/registry/software/vim/share/vim/vim92/autoload/getscript.vim b/software/vim/share/vim/vim92/autoload/getscript.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/autoload/getscript.vim rename to software/vim/share/vim/vim92/autoload/getscript.vim diff --git a/registry/software/vim/share/vim/vim92/autoload/gnat.vim b/software/vim/share/vim/vim92/autoload/gnat.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/autoload/gnat.vim rename to software/vim/share/vim/vim92/autoload/gnat.vim diff --git a/registry/software/vim/share/vim/vim92/autoload/gzip.vim b/software/vim/share/vim/vim92/autoload/gzip.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/autoload/gzip.vim rename to software/vim/share/vim/vim92/autoload/gzip.vim diff --git a/registry/software/vim/share/vim/vim92/autoload/haskellcomplete.vim b/software/vim/share/vim/vim92/autoload/haskellcomplete.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/autoload/haskellcomplete.vim rename to software/vim/share/vim/vim92/autoload/haskellcomplete.vim diff --git a/registry/software/vim/share/vim/vim92/autoload/htmlcomplete.vim b/software/vim/share/vim/vim92/autoload/htmlcomplete.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/autoload/htmlcomplete.vim rename to software/vim/share/vim/vim92/autoload/htmlcomplete.vim diff --git a/registry/software/vim/share/vim/vim92/autoload/javascriptcomplete.vim b/software/vim/share/vim/vim92/autoload/javascriptcomplete.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/autoload/javascriptcomplete.vim rename to software/vim/share/vim/vim92/autoload/javascriptcomplete.vim diff --git a/registry/software/vim/share/vim/vim92/autoload/netrw.vim b/software/vim/share/vim/vim92/autoload/netrw.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/autoload/netrw.vim rename to software/vim/share/vim/vim92/autoload/netrw.vim diff --git a/registry/software/vim/share/vim/vim92/autoload/netrwFileHandlers.vim b/software/vim/share/vim/vim92/autoload/netrwFileHandlers.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/autoload/netrwFileHandlers.vim rename to software/vim/share/vim/vim92/autoload/netrwFileHandlers.vim diff --git a/registry/software/vim/share/vim/vim92/autoload/netrwSettings.vim b/software/vim/share/vim/vim92/autoload/netrwSettings.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/autoload/netrwSettings.vim rename to software/vim/share/vim/vim92/autoload/netrwSettings.vim diff --git a/registry/software/vim/share/vim/vim92/autoload/netrw_gitignore.vim b/software/vim/share/vim/vim92/autoload/netrw_gitignore.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/autoload/netrw_gitignore.vim rename to software/vim/share/vim/vim92/autoload/netrw_gitignore.vim diff --git a/registry/software/vim/share/vim/vim92/autoload/paste.vim b/software/vim/share/vim/vim92/autoload/paste.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/autoload/paste.vim rename to software/vim/share/vim/vim92/autoload/paste.vim diff --git a/registry/software/vim/share/vim/vim92/autoload/phpcomplete.vim b/software/vim/share/vim/vim92/autoload/phpcomplete.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/autoload/phpcomplete.vim rename to software/vim/share/vim/vim92/autoload/phpcomplete.vim diff --git a/registry/software/vim/share/vim/vim92/autoload/python.vim b/software/vim/share/vim/vim92/autoload/python.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/autoload/python.vim rename to software/vim/share/vim/vim92/autoload/python.vim diff --git a/registry/software/vim/share/vim/vim92/autoload/python3complete.vim b/software/vim/share/vim/vim92/autoload/python3complete.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/autoload/python3complete.vim rename to software/vim/share/vim/vim92/autoload/python3complete.vim diff --git a/registry/software/vim/share/vim/vim92/autoload/pythoncomplete.vim b/software/vim/share/vim/vim92/autoload/pythoncomplete.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/autoload/pythoncomplete.vim rename to software/vim/share/vim/vim92/autoload/pythoncomplete.vim diff --git a/registry/software/vim/share/vim/vim92/autoload/rubycomplete.vim b/software/vim/share/vim/vim92/autoload/rubycomplete.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/autoload/rubycomplete.vim rename to software/vim/share/vim/vim92/autoload/rubycomplete.vim diff --git a/registry/software/vim/share/vim/vim92/autoload/rust.vim b/software/vim/share/vim/vim92/autoload/rust.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/autoload/rust.vim rename to software/vim/share/vim/vim92/autoload/rust.vim diff --git a/registry/software/vim/share/vim/vim92/autoload/rustfmt.vim b/software/vim/share/vim/vim92/autoload/rustfmt.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/autoload/rustfmt.vim rename to software/vim/share/vim/vim92/autoload/rustfmt.vim diff --git a/registry/software/vim/share/vim/vim92/autoload/spellfile.vim b/software/vim/share/vim/vim92/autoload/spellfile.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/autoload/spellfile.vim rename to software/vim/share/vim/vim92/autoload/spellfile.vim diff --git a/registry/software/vim/share/vim/vim92/autoload/sqlcomplete.vim b/software/vim/share/vim/vim92/autoload/sqlcomplete.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/autoload/sqlcomplete.vim rename to software/vim/share/vim/vim92/autoload/sqlcomplete.vim diff --git a/registry/software/vim/share/vim/vim92/autoload/syntaxcomplete.vim b/software/vim/share/vim/vim92/autoload/syntaxcomplete.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/autoload/syntaxcomplete.vim rename to software/vim/share/vim/vim92/autoload/syntaxcomplete.vim diff --git a/registry/software/vim/share/vim/vim92/autoload/tar.vim b/software/vim/share/vim/vim92/autoload/tar.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/autoload/tar.vim rename to software/vim/share/vim/vim92/autoload/tar.vim diff --git a/registry/software/vim/share/vim/vim92/autoload/tohtml.vim b/software/vim/share/vim/vim92/autoload/tohtml.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/autoload/tohtml.vim rename to software/vim/share/vim/vim92/autoload/tohtml.vim diff --git a/registry/software/vim/share/vim/vim92/autoload/typeset.vim b/software/vim/share/vim/vim92/autoload/typeset.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/autoload/typeset.vim rename to software/vim/share/vim/vim92/autoload/typeset.vim diff --git a/registry/software/vim/share/vim/vim92/autoload/vimball.vim b/software/vim/share/vim/vim92/autoload/vimball.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/autoload/vimball.vim rename to software/vim/share/vim/vim92/autoload/vimball.vim diff --git a/registry/software/vim/share/vim/vim92/autoload/xml/html32.vim b/software/vim/share/vim/vim92/autoload/xml/html32.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/autoload/xml/html32.vim rename to software/vim/share/vim/vim92/autoload/xml/html32.vim diff --git a/registry/software/vim/share/vim/vim92/autoload/xml/html401f.vim b/software/vim/share/vim/vim92/autoload/xml/html401f.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/autoload/xml/html401f.vim rename to software/vim/share/vim/vim92/autoload/xml/html401f.vim diff --git a/registry/software/vim/share/vim/vim92/autoload/xml/html401s.vim b/software/vim/share/vim/vim92/autoload/xml/html401s.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/autoload/xml/html401s.vim rename to software/vim/share/vim/vim92/autoload/xml/html401s.vim diff --git a/registry/software/vim/share/vim/vim92/autoload/xml/html401t.vim b/software/vim/share/vim/vim92/autoload/xml/html401t.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/autoload/xml/html401t.vim rename to software/vim/share/vim/vim92/autoload/xml/html401t.vim diff --git a/registry/software/vim/share/vim/vim92/autoload/xml/html40f.vim b/software/vim/share/vim/vim92/autoload/xml/html40f.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/autoload/xml/html40f.vim rename to software/vim/share/vim/vim92/autoload/xml/html40f.vim diff --git a/registry/software/vim/share/vim/vim92/autoload/xml/html40s.vim b/software/vim/share/vim/vim92/autoload/xml/html40s.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/autoload/xml/html40s.vim rename to software/vim/share/vim/vim92/autoload/xml/html40s.vim diff --git a/registry/software/vim/share/vim/vim92/autoload/xml/html40t.vim b/software/vim/share/vim/vim92/autoload/xml/html40t.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/autoload/xml/html40t.vim rename to software/vim/share/vim/vim92/autoload/xml/html40t.vim diff --git a/registry/software/vim/share/vim/vim92/autoload/xml/xhtml10f.vim b/software/vim/share/vim/vim92/autoload/xml/xhtml10f.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/autoload/xml/xhtml10f.vim rename to software/vim/share/vim/vim92/autoload/xml/xhtml10f.vim diff --git a/registry/software/vim/share/vim/vim92/autoload/xml/xhtml10s.vim b/software/vim/share/vim/vim92/autoload/xml/xhtml10s.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/autoload/xml/xhtml10s.vim rename to software/vim/share/vim/vim92/autoload/xml/xhtml10s.vim diff --git a/registry/software/vim/share/vim/vim92/autoload/xml/xhtml10t.vim b/software/vim/share/vim/vim92/autoload/xml/xhtml10t.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/autoload/xml/xhtml10t.vim rename to software/vim/share/vim/vim92/autoload/xml/xhtml10t.vim diff --git a/registry/software/vim/share/vim/vim92/autoload/xml/xhtml11.vim b/software/vim/share/vim/vim92/autoload/xml/xhtml11.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/autoload/xml/xhtml11.vim rename to software/vim/share/vim/vim92/autoload/xml/xhtml11.vim diff --git a/registry/software/vim/share/vim/vim92/autoload/xml/xsd.vim b/software/vim/share/vim/vim92/autoload/xml/xsd.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/autoload/xml/xsd.vim rename to software/vim/share/vim/vim92/autoload/xml/xsd.vim diff --git a/registry/software/vim/share/vim/vim92/autoload/xml/xsl.vim b/software/vim/share/vim/vim92/autoload/xml/xsl.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/autoload/xml/xsl.vim rename to software/vim/share/vim/vim92/autoload/xml/xsl.vim diff --git a/registry/software/vim/share/vim/vim92/autoload/xmlcomplete.vim b/software/vim/share/vim/vim92/autoload/xmlcomplete.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/autoload/xmlcomplete.vim rename to software/vim/share/vim/vim92/autoload/xmlcomplete.vim diff --git a/registry/software/vim/share/vim/vim92/autoload/xmlformat.vim b/software/vim/share/vim/vim92/autoload/xmlformat.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/autoload/xmlformat.vim rename to software/vim/share/vim/vim92/autoload/xmlformat.vim diff --git a/registry/software/vim/share/vim/vim92/autoload/zip.vim b/software/vim/share/vim/vim92/autoload/zip.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/autoload/zip.vim rename to software/vim/share/vim/vim92/autoload/zip.vim diff --git a/registry/software/vim/share/vim/vim92/bugreport.vim b/software/vim/share/vim/vim92/bugreport.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/bugreport.vim rename to software/vim/share/vim/vim92/bugreport.vim diff --git a/registry/software/vim/share/vim/vim92/colors/blue.vim b/software/vim/share/vim/vim92/colors/blue.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/colors/blue.vim rename to software/vim/share/vim/vim92/colors/blue.vim diff --git a/registry/software/vim/share/vim/vim92/colors/darkblue.vim b/software/vim/share/vim/vim92/colors/darkblue.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/colors/darkblue.vim rename to software/vim/share/vim/vim92/colors/darkblue.vim diff --git a/registry/software/vim/share/vim/vim92/colors/default.vim b/software/vim/share/vim/vim92/colors/default.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/colors/default.vim rename to software/vim/share/vim/vim92/colors/default.vim diff --git a/registry/software/vim/share/vim/vim92/colors/delek.vim b/software/vim/share/vim/vim92/colors/delek.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/colors/delek.vim rename to software/vim/share/vim/vim92/colors/delek.vim diff --git a/registry/software/vim/share/vim/vim92/colors/desert.vim b/software/vim/share/vim/vim92/colors/desert.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/colors/desert.vim rename to software/vim/share/vim/vim92/colors/desert.vim diff --git a/registry/software/vim/share/vim/vim92/colors/elflord.vim b/software/vim/share/vim/vim92/colors/elflord.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/colors/elflord.vim rename to software/vim/share/vim/vim92/colors/elflord.vim diff --git a/registry/software/vim/share/vim/vim92/colors/evening.vim b/software/vim/share/vim/vim92/colors/evening.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/colors/evening.vim rename to software/vim/share/vim/vim92/colors/evening.vim diff --git a/registry/software/vim/share/vim/vim92/colors/habamax.vim b/software/vim/share/vim/vim92/colors/habamax.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/colors/habamax.vim rename to software/vim/share/vim/vim92/colors/habamax.vim diff --git a/registry/software/vim/share/vim/vim92/colors/industry.vim b/software/vim/share/vim/vim92/colors/industry.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/colors/industry.vim rename to software/vim/share/vim/vim92/colors/industry.vim diff --git a/registry/software/vim/share/vim/vim92/colors/koehler.vim b/software/vim/share/vim/vim92/colors/koehler.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/colors/koehler.vim rename to software/vim/share/vim/vim92/colors/koehler.vim diff --git a/registry/software/vim/share/vim/vim92/colors/lists/csscolors.vim b/software/vim/share/vim/vim92/colors/lists/csscolors.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/colors/lists/csscolors.vim rename to software/vim/share/vim/vim92/colors/lists/csscolors.vim diff --git a/registry/software/vim/share/vim/vim92/colors/lists/default.vim b/software/vim/share/vim/vim92/colors/lists/default.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/colors/lists/default.vim rename to software/vim/share/vim/vim92/colors/lists/default.vim diff --git a/registry/software/vim/share/vim/vim92/colors/lunaperche.vim b/software/vim/share/vim/vim92/colors/lunaperche.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/colors/lunaperche.vim rename to software/vim/share/vim/vim92/colors/lunaperche.vim diff --git a/registry/software/vim/share/vim/vim92/colors/morning.vim b/software/vim/share/vim/vim92/colors/morning.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/colors/morning.vim rename to software/vim/share/vim/vim92/colors/morning.vim diff --git a/registry/software/vim/share/vim/vim92/colors/murphy.vim b/software/vim/share/vim/vim92/colors/murphy.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/colors/murphy.vim rename to software/vim/share/vim/vim92/colors/murphy.vim diff --git a/registry/software/vim/share/vim/vim92/colors/pablo.vim b/software/vim/share/vim/vim92/colors/pablo.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/colors/pablo.vim rename to software/vim/share/vim/vim92/colors/pablo.vim diff --git a/registry/software/vim/share/vim/vim92/colors/peachpuff.vim b/software/vim/share/vim/vim92/colors/peachpuff.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/colors/peachpuff.vim rename to software/vim/share/vim/vim92/colors/peachpuff.vim diff --git a/registry/software/vim/share/vim/vim92/colors/quiet.vim b/software/vim/share/vim/vim92/colors/quiet.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/colors/quiet.vim rename to software/vim/share/vim/vim92/colors/quiet.vim diff --git a/registry/software/vim/share/vim/vim92/colors/ron.vim b/software/vim/share/vim/vim92/colors/ron.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/colors/ron.vim rename to software/vim/share/vim/vim92/colors/ron.vim diff --git a/registry/software/vim/share/vim/vim92/colors/shine.vim b/software/vim/share/vim/vim92/colors/shine.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/colors/shine.vim rename to software/vim/share/vim/vim92/colors/shine.vim diff --git a/registry/software/vim/share/vim/vim92/colors/slate.vim b/software/vim/share/vim/vim92/colors/slate.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/colors/slate.vim rename to software/vim/share/vim/vim92/colors/slate.vim diff --git a/registry/software/vim/share/vim/vim92/colors/tools/check_colors.vim b/software/vim/share/vim/vim92/colors/tools/check_colors.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/colors/tools/check_colors.vim rename to software/vim/share/vim/vim92/colors/tools/check_colors.vim diff --git a/registry/software/vim/share/vim/vim92/colors/torte.vim b/software/vim/share/vim/vim92/colors/torte.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/colors/torte.vim rename to software/vim/share/vim/vim92/colors/torte.vim diff --git a/registry/software/vim/share/vim/vim92/colors/zellner.vim b/software/vim/share/vim/vim92/colors/zellner.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/colors/zellner.vim rename to software/vim/share/vim/vim92/colors/zellner.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/ant.vim b/software/vim/share/vim/vim92/compiler/ant.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/ant.vim rename to software/vim/share/vim/vim92/compiler/ant.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/bcc.vim b/software/vim/share/vim/vim92/compiler/bcc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/bcc.vim rename to software/vim/share/vim/vim92/compiler/bcc.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/bdf.vim b/software/vim/share/vim/vim92/compiler/bdf.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/bdf.vim rename to software/vim/share/vim/vim92/compiler/bdf.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/cargo.vim b/software/vim/share/vim/vim92/compiler/cargo.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/cargo.vim rename to software/vim/share/vim/vim92/compiler/cargo.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/checkstyle.vim b/software/vim/share/vim/vim92/compiler/checkstyle.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/checkstyle.vim rename to software/vim/share/vim/vim92/compiler/checkstyle.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/cm3.vim b/software/vim/share/vim/vim92/compiler/cm3.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/cm3.vim rename to software/vim/share/vim/vim92/compiler/cm3.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/context.vim b/software/vim/share/vim/vim92/compiler/context.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/context.vim rename to software/vim/share/vim/vim92/compiler/context.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/cs.vim b/software/vim/share/vim/vim92/compiler/cs.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/cs.vim rename to software/vim/share/vim/vim92/compiler/cs.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/csslint.vim b/software/vim/share/vim/vim92/compiler/csslint.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/csslint.vim rename to software/vim/share/vim/vim92/compiler/csslint.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/cucumber.vim b/software/vim/share/vim/vim92/compiler/cucumber.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/cucumber.vim rename to software/vim/share/vim/vim92/compiler/cucumber.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/dart.vim b/software/vim/share/vim/vim92/compiler/dart.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/dart.vim rename to software/vim/share/vim/vim92/compiler/dart.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/dart2js.vim b/software/vim/share/vim/vim92/compiler/dart2js.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/dart2js.vim rename to software/vim/share/vim/vim92/compiler/dart2js.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/dart2native.vim b/software/vim/share/vim/vim92/compiler/dart2native.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/dart2native.vim rename to software/vim/share/vim/vim92/compiler/dart2native.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/dartanalyser.vim b/software/vim/share/vim/vim92/compiler/dartanalyser.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/dartanalyser.vim rename to software/vim/share/vim/vim92/compiler/dartanalyser.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/dartdevc.vim b/software/vim/share/vim/vim92/compiler/dartdevc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/dartdevc.vim rename to software/vim/share/vim/vim92/compiler/dartdevc.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/dartdoc.vim b/software/vim/share/vim/vim92/compiler/dartdoc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/dartdoc.vim rename to software/vim/share/vim/vim92/compiler/dartdoc.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/dartfmt.vim b/software/vim/share/vim/vim92/compiler/dartfmt.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/dartfmt.vim rename to software/vim/share/vim/vim92/compiler/dartfmt.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/decada.vim b/software/vim/share/vim/vim92/compiler/decada.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/decada.vim rename to software/vim/share/vim/vim92/compiler/decada.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/dot.vim b/software/vim/share/vim/vim92/compiler/dot.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/dot.vim rename to software/vim/share/vim/vim92/compiler/dot.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/dotnet.vim b/software/vim/share/vim/vim92/compiler/dotnet.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/dotnet.vim rename to software/vim/share/vim/vim92/compiler/dotnet.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/erlang.vim b/software/vim/share/vim/vim92/compiler/erlang.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/erlang.vim rename to software/vim/share/vim/vim92/compiler/erlang.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/eruby.vim b/software/vim/share/vim/vim92/compiler/eruby.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/eruby.vim rename to software/vim/share/vim/vim92/compiler/eruby.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/eslint.vim b/software/vim/share/vim/vim92/compiler/eslint.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/eslint.vim rename to software/vim/share/vim/vim92/compiler/eslint.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/fbc.vim b/software/vim/share/vim/vim92/compiler/fbc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/fbc.vim rename to software/vim/share/vim/vim92/compiler/fbc.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/fortran_F.vim b/software/vim/share/vim/vim92/compiler/fortran_F.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/fortran_F.vim rename to software/vim/share/vim/vim92/compiler/fortran_F.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/fortran_cv.vim b/software/vim/share/vim/vim92/compiler/fortran_cv.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/fortran_cv.vim rename to software/vim/share/vim/vim92/compiler/fortran_cv.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/fortran_elf90.vim b/software/vim/share/vim/vim92/compiler/fortran_elf90.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/fortran_elf90.vim rename to software/vim/share/vim/vim92/compiler/fortran_elf90.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/fortran_g77.vim b/software/vim/share/vim/vim92/compiler/fortran_g77.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/fortran_g77.vim rename to software/vim/share/vim/vim92/compiler/fortran_g77.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/fortran_lf95.vim b/software/vim/share/vim/vim92/compiler/fortran_lf95.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/fortran_lf95.vim rename to software/vim/share/vim/vim92/compiler/fortran_lf95.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/fpc.vim b/software/vim/share/vim/vim92/compiler/fpc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/fpc.vim rename to software/vim/share/vim/vim92/compiler/fpc.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/g95.vim b/software/vim/share/vim/vim92/compiler/g95.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/g95.vim rename to software/vim/share/vim/vim92/compiler/g95.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/gawk.vim b/software/vim/share/vim/vim92/compiler/gawk.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/gawk.vim rename to software/vim/share/vim/vim92/compiler/gawk.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/gcc.vim b/software/vim/share/vim/vim92/compiler/gcc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/gcc.vim rename to software/vim/share/vim/vim92/compiler/gcc.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/gfortran.vim b/software/vim/share/vim/vim92/compiler/gfortran.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/gfortran.vim rename to software/vim/share/vim/vim92/compiler/gfortran.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/ghc.vim b/software/vim/share/vim/vim92/compiler/ghc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/ghc.vim rename to software/vim/share/vim/vim92/compiler/ghc.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/gjs.vim b/software/vim/share/vim/vim92/compiler/gjs.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/gjs.vim rename to software/vim/share/vim/vim92/compiler/gjs.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/gnat.vim b/software/vim/share/vim/vim92/compiler/gnat.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/gnat.vim rename to software/vim/share/vim/vim92/compiler/gnat.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/go.vim b/software/vim/share/vim/vim92/compiler/go.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/go.vim rename to software/vim/share/vim/vim92/compiler/go.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/haml.vim b/software/vim/share/vim/vim92/compiler/haml.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/haml.vim rename to software/vim/share/vim/vim92/compiler/haml.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/hare.vim b/software/vim/share/vim/vim92/compiler/hare.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/hare.vim rename to software/vim/share/vim/vim92/compiler/hare.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/hp_acc.vim b/software/vim/share/vim/vim92/compiler/hp_acc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/hp_acc.vim rename to software/vim/share/vim/vim92/compiler/hp_acc.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/icc.vim b/software/vim/share/vim/vim92/compiler/icc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/icc.vim rename to software/vim/share/vim/vim92/compiler/icc.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/icon.vim b/software/vim/share/vim/vim92/compiler/icon.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/icon.vim rename to software/vim/share/vim/vim92/compiler/icon.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/ifort.vim b/software/vim/share/vim/vim92/compiler/ifort.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/ifort.vim rename to software/vim/share/vim/vim92/compiler/ifort.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/intel.vim b/software/vim/share/vim/vim92/compiler/intel.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/intel.vim rename to software/vim/share/vim/vim92/compiler/intel.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/irix5_c.vim b/software/vim/share/vim/vim92/compiler/irix5_c.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/irix5_c.vim rename to software/vim/share/vim/vim92/compiler/irix5_c.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/irix5_cpp.vim b/software/vim/share/vim/vim92/compiler/irix5_cpp.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/irix5_cpp.vim rename to software/vim/share/vim/vim92/compiler/irix5_cpp.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/javac.vim b/software/vim/share/vim/vim92/compiler/javac.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/javac.vim rename to software/vim/share/vim/vim92/compiler/javac.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/jest.vim b/software/vim/share/vim/vim92/compiler/jest.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/jest.vim rename to software/vim/share/vim/vim92/compiler/jest.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/jikes.vim b/software/vim/share/vim/vim92/compiler/jikes.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/jikes.vim rename to software/vim/share/vim/vim92/compiler/jikes.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/jjs.vim b/software/vim/share/vim/vim92/compiler/jjs.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/jjs.vim rename to software/vim/share/vim/vim92/compiler/jjs.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/jshint.vim b/software/vim/share/vim/vim92/compiler/jshint.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/jshint.vim rename to software/vim/share/vim/vim92/compiler/jshint.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/jsonlint.vim b/software/vim/share/vim/vim92/compiler/jsonlint.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/jsonlint.vim rename to software/vim/share/vim/vim92/compiler/jsonlint.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/mcs.vim b/software/vim/share/vim/vim92/compiler/mcs.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/mcs.vim rename to software/vim/share/vim/vim92/compiler/mcs.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/mips_c.vim b/software/vim/share/vim/vim92/compiler/mips_c.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/mips_c.vim rename to software/vim/share/vim/vim92/compiler/mips_c.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/mipspro_c89.vim b/software/vim/share/vim/vim92/compiler/mipspro_c89.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/mipspro_c89.vim rename to software/vim/share/vim/vim92/compiler/mipspro_c89.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/mipspro_cpp.vim b/software/vim/share/vim/vim92/compiler/mipspro_cpp.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/mipspro_cpp.vim rename to software/vim/share/vim/vim92/compiler/mipspro_cpp.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/modelsim_vcom.vim b/software/vim/share/vim/vim92/compiler/modelsim_vcom.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/modelsim_vcom.vim rename to software/vim/share/vim/vim92/compiler/modelsim_vcom.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/msbuild.vim b/software/vim/share/vim/vim92/compiler/msbuild.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/msbuild.vim rename to software/vim/share/vim/vim92/compiler/msbuild.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/msvc.vim b/software/vim/share/vim/vim92/compiler/msvc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/msvc.vim rename to software/vim/share/vim/vim92/compiler/msvc.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/neato.vim b/software/vim/share/vim/vim92/compiler/neato.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/neato.vim rename to software/vim/share/vim/vim92/compiler/neato.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/ocaml.vim b/software/vim/share/vim/vim92/compiler/ocaml.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/ocaml.vim rename to software/vim/share/vim/vim92/compiler/ocaml.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/onsgmls.vim b/software/vim/share/vim/vim92/compiler/onsgmls.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/onsgmls.vim rename to software/vim/share/vim/vim92/compiler/onsgmls.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/pbx.vim b/software/vim/share/vim/vim92/compiler/pbx.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/pbx.vim rename to software/vim/share/vim/vim92/compiler/pbx.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/perl.vim b/software/vim/share/vim/vim92/compiler/perl.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/perl.vim rename to software/vim/share/vim/vim92/compiler/perl.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/perlcritic.vim b/software/vim/share/vim/vim92/compiler/perlcritic.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/perlcritic.vim rename to software/vim/share/vim/vim92/compiler/perlcritic.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/php.vim b/software/vim/share/vim/vim92/compiler/php.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/php.vim rename to software/vim/share/vim/vim92/compiler/php.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/podchecker.vim b/software/vim/share/vim/vim92/compiler/podchecker.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/podchecker.vim rename to software/vim/share/vim/vim92/compiler/podchecker.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/powershell.vim b/software/vim/share/vim/vim92/compiler/powershell.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/powershell.vim rename to software/vim/share/vim/vim92/compiler/powershell.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/pylint.vim b/software/vim/share/vim/vim92/compiler/pylint.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/pylint.vim rename to software/vim/share/vim/vim92/compiler/pylint.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/pyunit.vim b/software/vim/share/vim/vim92/compiler/pyunit.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/pyunit.vim rename to software/vim/share/vim/vim92/compiler/pyunit.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/raco.vim b/software/vim/share/vim/vim92/compiler/raco.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/raco.vim rename to software/vim/share/vim/vim92/compiler/raco.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/racomake.vim b/software/vim/share/vim/vim92/compiler/racomake.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/racomake.vim rename to software/vim/share/vim/vim92/compiler/racomake.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/racosetup.vim b/software/vim/share/vim/vim92/compiler/racosetup.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/racosetup.vim rename to software/vim/share/vim/vim92/compiler/racosetup.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/racotest.vim b/software/vim/share/vim/vim92/compiler/racotest.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/racotest.vim rename to software/vim/share/vim/vim92/compiler/racotest.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/rake.vim b/software/vim/share/vim/vim92/compiler/rake.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/rake.vim rename to software/vim/share/vim/vim92/compiler/rake.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/rhino.vim b/software/vim/share/vim/vim92/compiler/rhino.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/rhino.vim rename to software/vim/share/vim/vim92/compiler/rhino.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/rspec.vim b/software/vim/share/vim/vim92/compiler/rspec.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/rspec.vim rename to software/vim/share/vim/vim92/compiler/rspec.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/rst.vim b/software/vim/share/vim/vim92/compiler/rst.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/rst.vim rename to software/vim/share/vim/vim92/compiler/rst.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/rubocop.vim b/software/vim/share/vim/vim92/compiler/rubocop.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/rubocop.vim rename to software/vim/share/vim/vim92/compiler/rubocop.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/ruby.vim b/software/vim/share/vim/vim92/compiler/ruby.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/ruby.vim rename to software/vim/share/vim/vim92/compiler/ruby.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/rubyunit.vim b/software/vim/share/vim/vim92/compiler/rubyunit.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/rubyunit.vim rename to software/vim/share/vim/vim92/compiler/rubyunit.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/rustc.vim b/software/vim/share/vim/vim92/compiler/rustc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/rustc.vim rename to software/vim/share/vim/vim92/compiler/rustc.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/sass.vim b/software/vim/share/vim/vim92/compiler/sass.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/sass.vim rename to software/vim/share/vim/vim92/compiler/sass.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/scdoc.vim b/software/vim/share/vim/vim92/compiler/scdoc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/scdoc.vim rename to software/vim/share/vim/vim92/compiler/scdoc.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/se.vim b/software/vim/share/vim/vim92/compiler/se.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/se.vim rename to software/vim/share/vim/vim92/compiler/se.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/shellcheck.vim b/software/vim/share/vim/vim92/compiler/shellcheck.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/shellcheck.vim rename to software/vim/share/vim/vim92/compiler/shellcheck.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/sml.vim b/software/vim/share/vim/vim92/compiler/sml.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/sml.vim rename to software/vim/share/vim/vim92/compiler/sml.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/spectral.vim b/software/vim/share/vim/vim92/compiler/spectral.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/spectral.vim rename to software/vim/share/vim/vim92/compiler/spectral.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/splint.vim b/software/vim/share/vim/vim92/compiler/splint.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/splint.vim rename to software/vim/share/vim/vim92/compiler/splint.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/stack.vim b/software/vim/share/vim/vim92/compiler/stack.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/stack.vim rename to software/vim/share/vim/vim92/compiler/stack.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/standard.vim b/software/vim/share/vim/vim92/compiler/standard.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/standard.vim rename to software/vim/share/vim/vim92/compiler/standard.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/stylelint.vim b/software/vim/share/vim/vim92/compiler/stylelint.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/stylelint.vim rename to software/vim/share/vim/vim92/compiler/stylelint.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/tcl.vim b/software/vim/share/vim/vim92/compiler/tcl.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/tcl.vim rename to software/vim/share/vim/vim92/compiler/tcl.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/tex.vim b/software/vim/share/vim/vim92/compiler/tex.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/tex.vim rename to software/vim/share/vim/vim92/compiler/tex.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/tidy.vim b/software/vim/share/vim/vim92/compiler/tidy.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/tidy.vim rename to software/vim/share/vim/vim92/compiler/tidy.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/ts-node.vim b/software/vim/share/vim/vim92/compiler/ts-node.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/ts-node.vim rename to software/vim/share/vim/vim92/compiler/ts-node.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/tsc.vim b/software/vim/share/vim/vim92/compiler/tsc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/tsc.vim rename to software/vim/share/vim/vim92/compiler/tsc.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/typedoc.vim b/software/vim/share/vim/vim92/compiler/typedoc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/typedoc.vim rename to software/vim/share/vim/vim92/compiler/typedoc.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/xbuild.vim b/software/vim/share/vim/vim92/compiler/xbuild.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/xbuild.vim rename to software/vim/share/vim/vim92/compiler/xbuild.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/xmllint.vim b/software/vim/share/vim/vim92/compiler/xmllint.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/xmllint.vim rename to software/vim/share/vim/vim92/compiler/xmllint.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/xmlwf.vim b/software/vim/share/vim/vim92/compiler/xmlwf.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/xmlwf.vim rename to software/vim/share/vim/vim92/compiler/xmlwf.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/xo.vim b/software/vim/share/vim/vim92/compiler/xo.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/xo.vim rename to software/vim/share/vim/vim92/compiler/xo.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/yamllint.vim b/software/vim/share/vim/vim92/compiler/yamllint.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/yamllint.vim rename to software/vim/share/vim/vim92/compiler/yamllint.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/zig.vim b/software/vim/share/vim/vim92/compiler/zig.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/zig.vim rename to software/vim/share/vim/vim92/compiler/zig.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/zig_build.vim b/software/vim/share/vim/vim92/compiler/zig_build.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/zig_build.vim rename to software/vim/share/vim/vim92/compiler/zig_build.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/zig_build_exe.vim b/software/vim/share/vim/vim92/compiler/zig_build_exe.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/zig_build_exe.vim rename to software/vim/share/vim/vim92/compiler/zig_build_exe.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/zig_test.vim b/software/vim/share/vim/vim92/compiler/zig_test.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/zig_test.vim rename to software/vim/share/vim/vim92/compiler/zig_test.vim diff --git a/registry/software/vim/share/vim/vim92/compiler/zsh.vim b/software/vim/share/vim/vim92/compiler/zsh.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/compiler/zsh.vim rename to software/vim/share/vim/vim92/compiler/zsh.vim diff --git a/registry/software/vim/share/vim/vim92/debian.vim b/software/vim/share/vim/vim92/debian.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/debian.vim rename to software/vim/share/vim/vim92/debian.vim diff --git a/registry/software/vim/share/vim/vim92/defaults.vim b/software/vim/share/vim/vim92/defaults.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/defaults.vim rename to software/vim/share/vim/vim92/defaults.vim diff --git a/registry/software/vim/share/vim/vim92/delmenu.vim b/software/vim/share/vim/vim92/delmenu.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/delmenu.vim rename to software/vim/share/vim/vim92/delmenu.vim diff --git a/registry/software/vim/share/vim/vim92/evim.vim b/software/vim/share/vim/vim92/evim.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/evim.vim rename to software/vim/share/vim/vim92/evim.vim diff --git a/registry/software/vim/share/vim/vim92/filetype.vim b/software/vim/share/vim/vim92/filetype.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/filetype.vim rename to software/vim/share/vim/vim92/filetype.vim diff --git a/registry/software/vim/share/vim/vim92/ftoff.vim b/software/vim/share/vim/vim92/ftoff.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftoff.vim rename to software/vim/share/vim/vim92/ftoff.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin.vim b/software/vim/share/vim/vim92/ftplugin.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin.vim rename to software/vim/share/vim/vim92/ftplugin.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/8th.vim b/software/vim/share/vim/vim92/ftplugin/8th.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/8th.vim rename to software/vim/share/vim/vim92/ftplugin/8th.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/a2ps.vim b/software/vim/share/vim/vim92/ftplugin/a2ps.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/a2ps.vim rename to software/vim/share/vim/vim92/ftplugin/a2ps.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/aap.vim b/software/vim/share/vim/vim92/ftplugin/aap.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/aap.vim rename to software/vim/share/vim/vim92/ftplugin/aap.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/abap.vim b/software/vim/share/vim/vim92/ftplugin/abap.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/abap.vim rename to software/vim/share/vim/vim92/ftplugin/abap.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/abaqus.vim b/software/vim/share/vim/vim92/ftplugin/abaqus.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/abaqus.vim rename to software/vim/share/vim/vim92/ftplugin/abaqus.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/ada.vim b/software/vim/share/vim/vim92/ftplugin/ada.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/ada.vim rename to software/vim/share/vim/vim92/ftplugin/ada.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/alsaconf.vim b/software/vim/share/vim/vim92/ftplugin/alsaconf.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/alsaconf.vim rename to software/vim/share/vim/vim92/ftplugin/alsaconf.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/ant.vim b/software/vim/share/vim/vim92/ftplugin/ant.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/ant.vim rename to software/vim/share/vim/vim92/ftplugin/ant.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/apache.vim b/software/vim/share/vim/vim92/ftplugin/apache.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/apache.vim rename to software/vim/share/vim/vim92/ftplugin/apache.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/arch.vim b/software/vim/share/vim/vim92/ftplugin/arch.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/arch.vim rename to software/vim/share/vim/vim92/ftplugin/arch.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/art.vim b/software/vim/share/vim/vim92/ftplugin/art.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/art.vim rename to software/vim/share/vim/vim92/ftplugin/art.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/asm.vim b/software/vim/share/vim/vim92/ftplugin/asm.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/asm.vim rename to software/vim/share/vim/vim92/ftplugin/asm.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/aspvbs.vim b/software/vim/share/vim/vim92/ftplugin/aspvbs.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/aspvbs.vim rename to software/vim/share/vim/vim92/ftplugin/aspvbs.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/automake.vim b/software/vim/share/vim/vim92/ftplugin/automake.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/automake.vim rename to software/vim/share/vim/vim92/ftplugin/automake.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/awk.vim b/software/vim/share/vim/vim92/ftplugin/awk.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/awk.vim rename to software/vim/share/vim/vim92/ftplugin/awk.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/bash.vim b/software/vim/share/vim/vim92/ftplugin/bash.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/bash.vim rename to software/vim/share/vim/vim92/ftplugin/bash.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/basic.vim b/software/vim/share/vim/vim92/ftplugin/basic.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/basic.vim rename to software/vim/share/vim/vim92/ftplugin/basic.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/bdf.vim b/software/vim/share/vim/vim92/ftplugin/bdf.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/bdf.vim rename to software/vim/share/vim/vim92/ftplugin/bdf.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/bitbake.vim b/software/vim/share/vim/vim92/ftplugin/bitbake.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/bitbake.vim rename to software/vim/share/vim/vim92/ftplugin/bitbake.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/bst.vim b/software/vim/share/vim/vim92/ftplugin/bst.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/bst.vim rename to software/vim/share/vim/vim92/ftplugin/bst.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/btm.vim b/software/vim/share/vim/vim92/ftplugin/btm.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/btm.vim rename to software/vim/share/vim/vim92/ftplugin/btm.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/bzl.vim b/software/vim/share/vim/vim92/ftplugin/bzl.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/bzl.vim rename to software/vim/share/vim/vim92/ftplugin/bzl.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/c.vim b/software/vim/share/vim/vim92/ftplugin/c.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/c.vim rename to software/vim/share/vim/vim92/ftplugin/c.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/calendar.vim b/software/vim/share/vim/vim92/ftplugin/calendar.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/calendar.vim rename to software/vim/share/vim/vim92/ftplugin/calendar.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/cdrdaoconf.vim b/software/vim/share/vim/vim92/ftplugin/cdrdaoconf.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/cdrdaoconf.vim rename to software/vim/share/vim/vim92/ftplugin/cdrdaoconf.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/cfg.vim b/software/vim/share/vim/vim92/ftplugin/cfg.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/cfg.vim rename to software/vim/share/vim/vim92/ftplugin/cfg.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/ch.vim b/software/vim/share/vim/vim92/ftplugin/ch.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/ch.vim rename to software/vim/share/vim/vim92/ftplugin/ch.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/changelog.vim b/software/vim/share/vim/vim92/ftplugin/changelog.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/changelog.vim rename to software/vim/share/vim/vim92/ftplugin/changelog.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/chatito.vim b/software/vim/share/vim/vim92/ftplugin/chatito.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/chatito.vim rename to software/vim/share/vim/vim92/ftplugin/chatito.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/chicken.vim b/software/vim/share/vim/vim92/ftplugin/chicken.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/chicken.vim rename to software/vim/share/vim/vim92/ftplugin/chicken.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/clojure.vim b/software/vim/share/vim/vim92/ftplugin/clojure.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/clojure.vim rename to software/vim/share/vim/vim92/ftplugin/clojure.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/cmake.vim b/software/vim/share/vim/vim92/ftplugin/cmake.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/cmake.vim rename to software/vim/share/vim/vim92/ftplugin/cmake.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/cobol.vim b/software/vim/share/vim/vim92/ftplugin/cobol.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/cobol.vim rename to software/vim/share/vim/vim92/ftplugin/cobol.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/conf.vim b/software/vim/share/vim/vim92/ftplugin/conf.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/conf.vim rename to software/vim/share/vim/vim92/ftplugin/conf.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/config.vim b/software/vim/share/vim/vim92/ftplugin/config.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/config.vim rename to software/vim/share/vim/vim92/ftplugin/config.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/confini.vim b/software/vim/share/vim/vim92/ftplugin/confini.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/confini.vim rename to software/vim/share/vim/vim92/ftplugin/confini.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/context.vim b/software/vim/share/vim/vim92/ftplugin/context.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/context.vim rename to software/vim/share/vim/vim92/ftplugin/context.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/cpp.vim b/software/vim/share/vim/vim92/ftplugin/cpp.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/cpp.vim rename to software/vim/share/vim/vim92/ftplugin/cpp.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/crm.vim b/software/vim/share/vim/vim92/ftplugin/crm.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/crm.vim rename to software/vim/share/vim/vim92/ftplugin/crm.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/crontab.vim b/software/vim/share/vim/vim92/ftplugin/crontab.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/crontab.vim rename to software/vim/share/vim/vim92/ftplugin/crontab.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/cs.vim b/software/vim/share/vim/vim92/ftplugin/cs.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/cs.vim rename to software/vim/share/vim/vim92/ftplugin/cs.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/csc.vim b/software/vim/share/vim/vim92/ftplugin/csc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/csc.vim rename to software/vim/share/vim/vim92/ftplugin/csc.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/csh.vim b/software/vim/share/vim/vim92/ftplugin/csh.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/csh.vim rename to software/vim/share/vim/vim92/ftplugin/csh.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/css.vim b/software/vim/share/vim/vim92/ftplugin/css.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/css.vim rename to software/vim/share/vim/vim92/ftplugin/css.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/cucumber.vim b/software/vim/share/vim/vim92/ftplugin/cucumber.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/cucumber.vim rename to software/vim/share/vim/vim92/ftplugin/cucumber.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/cvsrc.vim b/software/vim/share/vim/vim92/ftplugin/cvsrc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/cvsrc.vim rename to software/vim/share/vim/vim92/ftplugin/cvsrc.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/debchangelog.vim b/software/vim/share/vim/vim92/ftplugin/debchangelog.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/debchangelog.vim rename to software/vim/share/vim/vim92/ftplugin/debchangelog.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/debcontrol.vim b/software/vim/share/vim/vim92/ftplugin/debcontrol.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/debcontrol.vim rename to software/vim/share/vim/vim92/ftplugin/debcontrol.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/denyhosts.vim b/software/vim/share/vim/vim92/ftplugin/denyhosts.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/denyhosts.vim rename to software/vim/share/vim/vim92/ftplugin/denyhosts.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/desktop.vim b/software/vim/share/vim/vim92/ftplugin/desktop.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/desktop.vim rename to software/vim/share/vim/vim92/ftplugin/desktop.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/dictconf.vim b/software/vim/share/vim/vim92/ftplugin/dictconf.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/dictconf.vim rename to software/vim/share/vim/vim92/ftplugin/dictconf.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/dictdconf.vim b/software/vim/share/vim/vim92/ftplugin/dictdconf.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/dictdconf.vim rename to software/vim/share/vim/vim92/ftplugin/dictdconf.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/diff.vim b/software/vim/share/vim/vim92/ftplugin/diff.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/diff.vim rename to software/vim/share/vim/vim92/ftplugin/diff.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/dircolors.vim b/software/vim/share/vim/vim92/ftplugin/dircolors.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/dircolors.vim rename to software/vim/share/vim/vim92/ftplugin/dircolors.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/docbk.vim b/software/vim/share/vim/vim92/ftplugin/docbk.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/docbk.vim rename to software/vim/share/vim/vim92/ftplugin/docbk.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/dockerfile.vim b/software/vim/share/vim/vim92/ftplugin/dockerfile.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/dockerfile.vim rename to software/vim/share/vim/vim92/ftplugin/dockerfile.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/dosbatch.vim b/software/vim/share/vim/vim92/ftplugin/dosbatch.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/dosbatch.vim rename to software/vim/share/vim/vim92/ftplugin/dosbatch.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/dosini.vim b/software/vim/share/vim/vim92/ftplugin/dosini.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/dosini.vim rename to software/vim/share/vim/vim92/ftplugin/dosini.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/dtd.vim b/software/vim/share/vim/vim92/ftplugin/dtd.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/dtd.vim rename to software/vim/share/vim/vim92/ftplugin/dtd.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/dtrace.vim b/software/vim/share/vim/vim92/ftplugin/dtrace.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/dtrace.vim rename to software/vim/share/vim/vim92/ftplugin/dtrace.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/dune.vim b/software/vim/share/vim/vim92/ftplugin/dune.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/dune.vim rename to software/vim/share/vim/vim92/ftplugin/dune.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/eiffel.vim b/software/vim/share/vim/vim92/ftplugin/eiffel.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/eiffel.vim rename to software/vim/share/vim/vim92/ftplugin/eiffel.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/elinks.vim b/software/vim/share/vim/vim92/ftplugin/elinks.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/elinks.vim rename to software/vim/share/vim/vim92/ftplugin/elinks.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/elixir.vim b/software/vim/share/vim/vim92/ftplugin/elixir.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/elixir.vim rename to software/vim/share/vim/vim92/ftplugin/elixir.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/elm.vim b/software/vim/share/vim/vim92/ftplugin/elm.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/elm.vim rename to software/vim/share/vim/vim92/ftplugin/elm.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/erlang.vim b/software/vim/share/vim/vim92/ftplugin/erlang.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/erlang.vim rename to software/vim/share/vim/vim92/ftplugin/erlang.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/eruby.vim b/software/vim/share/vim/vim92/ftplugin/eruby.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/eruby.vim rename to software/vim/share/vim/vim92/ftplugin/eruby.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/eterm.vim b/software/vim/share/vim/vim92/ftplugin/eterm.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/eterm.vim rename to software/vim/share/vim/vim92/ftplugin/eterm.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/expect.vim b/software/vim/share/vim/vim92/ftplugin/expect.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/expect.vim rename to software/vim/share/vim/vim92/ftplugin/expect.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/falcon.vim b/software/vim/share/vim/vim92/ftplugin/falcon.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/falcon.vim rename to software/vim/share/vim/vim92/ftplugin/falcon.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/fennel.vim b/software/vim/share/vim/vim92/ftplugin/fennel.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/fennel.vim rename to software/vim/share/vim/vim92/ftplugin/fennel.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/fetchmail.vim b/software/vim/share/vim/vim92/ftplugin/fetchmail.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/fetchmail.vim rename to software/vim/share/vim/vim92/ftplugin/fetchmail.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/fish.vim b/software/vim/share/vim/vim92/ftplugin/fish.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/fish.vim rename to software/vim/share/vim/vim92/ftplugin/fish.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/flexwiki.vim b/software/vim/share/vim/vim92/ftplugin/flexwiki.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/flexwiki.vim rename to software/vim/share/vim/vim92/ftplugin/flexwiki.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/fortran.vim b/software/vim/share/vim/vim92/ftplugin/fortran.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/fortran.vim rename to software/vim/share/vim/vim92/ftplugin/fortran.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/fpcmake.vim b/software/vim/share/vim/vim92/ftplugin/fpcmake.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/fpcmake.vim rename to software/vim/share/vim/vim92/ftplugin/fpcmake.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/framescript.vim b/software/vim/share/vim/vim92/ftplugin/framescript.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/framescript.vim rename to software/vim/share/vim/vim92/ftplugin/framescript.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/freebasic.vim b/software/vim/share/vim/vim92/ftplugin/freebasic.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/freebasic.vim rename to software/vim/share/vim/vim92/ftplugin/freebasic.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/fstab.vim b/software/vim/share/vim/vim92/ftplugin/fstab.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/fstab.vim rename to software/vim/share/vim/vim92/ftplugin/fstab.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/fvwm.vim b/software/vim/share/vim/vim92/ftplugin/fvwm.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/fvwm.vim rename to software/vim/share/vim/vim92/ftplugin/fvwm.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/gdb.vim b/software/vim/share/vim/vim92/ftplugin/gdb.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/gdb.vim rename to software/vim/share/vim/vim92/ftplugin/gdb.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/gdscript.vim b/software/vim/share/vim/vim92/ftplugin/gdscript.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/gdscript.vim rename to software/vim/share/vim/vim92/ftplugin/gdscript.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/gdshader.vim b/software/vim/share/vim/vim92/ftplugin/gdshader.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/gdshader.vim rename to software/vim/share/vim/vim92/ftplugin/gdshader.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/gitattributes.vim b/software/vim/share/vim/vim92/ftplugin/gitattributes.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/gitattributes.vim rename to software/vim/share/vim/vim92/ftplugin/gitattributes.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/gitcommit.vim b/software/vim/share/vim/vim92/ftplugin/gitcommit.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/gitcommit.vim rename to software/vim/share/vim/vim92/ftplugin/gitcommit.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/gitconfig.vim b/software/vim/share/vim/vim92/ftplugin/gitconfig.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/gitconfig.vim rename to software/vim/share/vim/vim92/ftplugin/gitconfig.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/gitignore.vim b/software/vim/share/vim/vim92/ftplugin/gitignore.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/gitignore.vim rename to software/vim/share/vim/vim92/ftplugin/gitignore.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/gitrebase.vim b/software/vim/share/vim/vim92/ftplugin/gitrebase.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/gitrebase.vim rename to software/vim/share/vim/vim92/ftplugin/gitrebase.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/gitsendemail.vim b/software/vim/share/vim/vim92/ftplugin/gitsendemail.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/gitsendemail.vim rename to software/vim/share/vim/vim92/ftplugin/gitsendemail.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/go.vim b/software/vim/share/vim/vim92/ftplugin/go.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/go.vim rename to software/vim/share/vim/vim92/ftplugin/go.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/gpg.vim b/software/vim/share/vim/vim92/ftplugin/gpg.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/gpg.vim rename to software/vim/share/vim/vim92/ftplugin/gpg.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/gprof.vim b/software/vim/share/vim/vim92/ftplugin/gprof.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/gprof.vim rename to software/vim/share/vim/vim92/ftplugin/gprof.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/groovy.vim b/software/vim/share/vim/vim92/ftplugin/groovy.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/groovy.vim rename to software/vim/share/vim/vim92/ftplugin/groovy.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/group.vim b/software/vim/share/vim/vim92/ftplugin/group.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/group.vim rename to software/vim/share/vim/vim92/ftplugin/group.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/grub.vim b/software/vim/share/vim/vim92/ftplugin/grub.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/grub.vim rename to software/vim/share/vim/vim92/ftplugin/grub.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/gyp.vim b/software/vim/share/vim/vim92/ftplugin/gyp.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/gyp.vim rename to software/vim/share/vim/vim92/ftplugin/gyp.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/haml.vim b/software/vim/share/vim/vim92/ftplugin/haml.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/haml.vim rename to software/vim/share/vim/vim92/ftplugin/haml.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/hamster.vim b/software/vim/share/vim/vim92/ftplugin/hamster.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/hamster.vim rename to software/vim/share/vim/vim92/ftplugin/hamster.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/hare.vim b/software/vim/share/vim/vim92/ftplugin/hare.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/hare.vim rename to software/vim/share/vim/vim92/ftplugin/hare.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/haskell.vim b/software/vim/share/vim/vim92/ftplugin/haskell.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/haskell.vim rename to software/vim/share/vim/vim92/ftplugin/haskell.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/heex.vim b/software/vim/share/vim/vim92/ftplugin/heex.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/heex.vim rename to software/vim/share/vim/vim92/ftplugin/heex.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/help.vim b/software/vim/share/vim/vim92/ftplugin/help.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/help.vim rename to software/vim/share/vim/vim92/ftplugin/help.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/hgcommit.vim b/software/vim/share/vim/vim92/ftplugin/hgcommit.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/hgcommit.vim rename to software/vim/share/vim/vim92/ftplugin/hgcommit.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/hog.vim b/software/vim/share/vim/vim92/ftplugin/hog.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/hog.vim rename to software/vim/share/vim/vim92/ftplugin/hog.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/hostconf.vim b/software/vim/share/vim/vim92/ftplugin/hostconf.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/hostconf.vim rename to software/vim/share/vim/vim92/ftplugin/hostconf.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/hostsaccess.vim b/software/vim/share/vim/vim92/ftplugin/hostsaccess.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/hostsaccess.vim rename to software/vim/share/vim/vim92/ftplugin/hostsaccess.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/html.vim b/software/vim/share/vim/vim92/ftplugin/html.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/html.vim rename to software/vim/share/vim/vim92/ftplugin/html.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/htmldjango.vim b/software/vim/share/vim/vim92/ftplugin/htmldjango.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/htmldjango.vim rename to software/vim/share/vim/vim92/ftplugin/htmldjango.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/i3config.vim b/software/vim/share/vim/vim92/ftplugin/i3config.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/i3config.vim rename to software/vim/share/vim/vim92/ftplugin/i3config.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/icon.vim b/software/vim/share/vim/vim92/ftplugin/icon.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/icon.vim rename to software/vim/share/vim/vim92/ftplugin/icon.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/indent.vim b/software/vim/share/vim/vim92/ftplugin/indent.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/indent.vim rename to software/vim/share/vim/vim92/ftplugin/indent.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/initex.vim b/software/vim/share/vim/vim92/ftplugin/initex.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/initex.vim rename to software/vim/share/vim/vim92/ftplugin/initex.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/ishd.vim b/software/vim/share/vim/vim92/ftplugin/ishd.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/ishd.vim rename to software/vim/share/vim/vim92/ftplugin/ishd.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/j.vim b/software/vim/share/vim/vim92/ftplugin/j.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/j.vim rename to software/vim/share/vim/vim92/ftplugin/j.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/java.vim b/software/vim/share/vim/vim92/ftplugin/java.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/java.vim rename to software/vim/share/vim/vim92/ftplugin/java.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/javascript.vim b/software/vim/share/vim/vim92/ftplugin/javascript.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/javascript.vim rename to software/vim/share/vim/vim92/ftplugin/javascript.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/javascriptreact.vim b/software/vim/share/vim/vim92/ftplugin/javascriptreact.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/javascriptreact.vim rename to software/vim/share/vim/vim92/ftplugin/javascriptreact.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/jproperties.vim b/software/vim/share/vim/vim92/ftplugin/jproperties.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/jproperties.vim rename to software/vim/share/vim/vim92/ftplugin/jproperties.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/json.vim b/software/vim/share/vim/vim92/ftplugin/json.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/json.vim rename to software/vim/share/vim/vim92/ftplugin/json.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/jsonc.vim b/software/vim/share/vim/vim92/ftplugin/jsonc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/jsonc.vim rename to software/vim/share/vim/vim92/ftplugin/jsonc.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/jsonnet.vim b/software/vim/share/vim/vim92/ftplugin/jsonnet.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/jsonnet.vim rename to software/vim/share/vim/vim92/ftplugin/jsonnet.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/jsp.vim b/software/vim/share/vim/vim92/ftplugin/jsp.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/jsp.vim rename to software/vim/share/vim/vim92/ftplugin/jsp.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/julia.vim b/software/vim/share/vim/vim92/ftplugin/julia.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/julia.vim rename to software/vim/share/vim/vim92/ftplugin/julia.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/kconfig.vim b/software/vim/share/vim/vim92/ftplugin/kconfig.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/kconfig.vim rename to software/vim/share/vim/vim92/ftplugin/kconfig.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/kwt.vim b/software/vim/share/vim/vim92/ftplugin/kwt.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/kwt.vim rename to software/vim/share/vim/vim92/ftplugin/kwt.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/ld.vim b/software/vim/share/vim/vim92/ftplugin/ld.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/ld.vim rename to software/vim/share/vim/vim92/ftplugin/ld.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/less.vim b/software/vim/share/vim/vim92/ftplugin/less.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/less.vim rename to software/vim/share/vim/vim92/ftplugin/less.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/lftp.vim b/software/vim/share/vim/vim92/ftplugin/lftp.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/lftp.vim rename to software/vim/share/vim/vim92/ftplugin/lftp.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/libao.vim b/software/vim/share/vim/vim92/ftplugin/libao.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/libao.vim rename to software/vim/share/vim/vim92/ftplugin/libao.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/limits.vim b/software/vim/share/vim/vim92/ftplugin/limits.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/limits.vim rename to software/vim/share/vim/vim92/ftplugin/limits.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/liquid.vim b/software/vim/share/vim/vim92/ftplugin/liquid.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/liquid.vim rename to software/vim/share/vim/vim92/ftplugin/liquid.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/lisp.vim b/software/vim/share/vim/vim92/ftplugin/lisp.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/lisp.vim rename to software/vim/share/vim/vim92/ftplugin/lisp.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/logcheck.vim b/software/vim/share/vim/vim92/ftplugin/logcheck.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/logcheck.vim rename to software/vim/share/vim/vim92/ftplugin/logcheck.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/loginaccess.vim b/software/vim/share/vim/vim92/ftplugin/loginaccess.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/loginaccess.vim rename to software/vim/share/vim/vim92/ftplugin/loginaccess.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/logindefs.vim b/software/vim/share/vim/vim92/ftplugin/logindefs.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/logindefs.vim rename to software/vim/share/vim/vim92/ftplugin/logindefs.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/logtalk.dict b/software/vim/share/vim/vim92/ftplugin/logtalk.dict similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/logtalk.dict rename to software/vim/share/vim/vim92/ftplugin/logtalk.dict diff --git a/registry/software/vim/share/vim/vim92/ftplugin/logtalk.vim b/software/vim/share/vim/vim92/ftplugin/logtalk.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/logtalk.vim rename to software/vim/share/vim/vim92/ftplugin/logtalk.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/lprolog.vim b/software/vim/share/vim/vim92/ftplugin/lprolog.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/lprolog.vim rename to software/vim/share/vim/vim92/ftplugin/lprolog.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/lua.vim b/software/vim/share/vim/vim92/ftplugin/lua.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/lua.vim rename to software/vim/share/vim/vim92/ftplugin/lua.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/lynx.vim b/software/vim/share/vim/vim92/ftplugin/lynx.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/lynx.vim rename to software/vim/share/vim/vim92/ftplugin/lynx.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/m3build.vim b/software/vim/share/vim/vim92/ftplugin/m3build.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/m3build.vim rename to software/vim/share/vim/vim92/ftplugin/m3build.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/m3quake.vim b/software/vim/share/vim/vim92/ftplugin/m3quake.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/m3quake.vim rename to software/vim/share/vim/vim92/ftplugin/m3quake.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/m4.vim b/software/vim/share/vim/vim92/ftplugin/m4.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/m4.vim rename to software/vim/share/vim/vim92/ftplugin/m4.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/mail.vim b/software/vim/share/vim/vim92/ftplugin/mail.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/mail.vim rename to software/vim/share/vim/vim92/ftplugin/mail.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/mailaliases.vim b/software/vim/share/vim/vim92/ftplugin/mailaliases.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/mailaliases.vim rename to software/vim/share/vim/vim92/ftplugin/mailaliases.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/mailcap.vim b/software/vim/share/vim/vim92/ftplugin/mailcap.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/mailcap.vim rename to software/vim/share/vim/vim92/ftplugin/mailcap.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/make.vim b/software/vim/share/vim/vim92/ftplugin/make.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/make.vim rename to software/vim/share/vim/vim92/ftplugin/make.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/man.vim b/software/vim/share/vim/vim92/ftplugin/man.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/man.vim rename to software/vim/share/vim/vim92/ftplugin/man.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/manconf.vim b/software/vim/share/vim/vim92/ftplugin/manconf.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/manconf.vim rename to software/vim/share/vim/vim92/ftplugin/manconf.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/markdown.vim b/software/vim/share/vim/vim92/ftplugin/markdown.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/markdown.vim rename to software/vim/share/vim/vim92/ftplugin/markdown.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/masm.vim b/software/vim/share/vim/vim92/ftplugin/masm.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/masm.vim rename to software/vim/share/vim/vim92/ftplugin/masm.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/matlab.vim b/software/vim/share/vim/vim92/ftplugin/matlab.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/matlab.vim rename to software/vim/share/vim/vim92/ftplugin/matlab.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/mermaid.vim b/software/vim/share/vim/vim92/ftplugin/mermaid.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/mermaid.vim rename to software/vim/share/vim/vim92/ftplugin/mermaid.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/meson.vim b/software/vim/share/vim/vim92/ftplugin/meson.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/meson.vim rename to software/vim/share/vim/vim92/ftplugin/meson.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/mf.vim b/software/vim/share/vim/vim92/ftplugin/mf.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/mf.vim rename to software/vim/share/vim/vim92/ftplugin/mf.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/mma.vim b/software/vim/share/vim/vim92/ftplugin/mma.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/mma.vim rename to software/vim/share/vim/vim92/ftplugin/mma.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/modconf.vim b/software/vim/share/vim/vim92/ftplugin/modconf.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/modconf.vim rename to software/vim/share/vim/vim92/ftplugin/modconf.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/modula2.vim b/software/vim/share/vim/vim92/ftplugin/modula2.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/modula2.vim rename to software/vim/share/vim/vim92/ftplugin/modula2.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/modula3.vim b/software/vim/share/vim/vim92/ftplugin/modula3.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/modula3.vim rename to software/vim/share/vim/vim92/ftplugin/modula3.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/mp.vim b/software/vim/share/vim/vim92/ftplugin/mp.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/mp.vim rename to software/vim/share/vim/vim92/ftplugin/mp.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/mplayerconf.vim b/software/vim/share/vim/vim92/ftplugin/mplayerconf.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/mplayerconf.vim rename to software/vim/share/vim/vim92/ftplugin/mplayerconf.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/mrxvtrc.vim b/software/vim/share/vim/vim92/ftplugin/mrxvtrc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/mrxvtrc.vim rename to software/vim/share/vim/vim92/ftplugin/mrxvtrc.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/msmessages.vim b/software/vim/share/vim/vim92/ftplugin/msmessages.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/msmessages.vim rename to software/vim/share/vim/vim92/ftplugin/msmessages.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/muttrc.vim b/software/vim/share/vim/vim92/ftplugin/muttrc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/muttrc.vim rename to software/vim/share/vim/vim92/ftplugin/muttrc.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/nanorc.vim b/software/vim/share/vim/vim92/ftplugin/nanorc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/nanorc.vim rename to software/vim/share/vim/vim92/ftplugin/nanorc.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/neomuttrc.vim b/software/vim/share/vim/vim92/ftplugin/neomuttrc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/neomuttrc.vim rename to software/vim/share/vim/vim92/ftplugin/neomuttrc.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/netrc.vim b/software/vim/share/vim/vim92/ftplugin/netrc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/netrc.vim rename to software/vim/share/vim/vim92/ftplugin/netrc.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/nginx.vim b/software/vim/share/vim/vim92/ftplugin/nginx.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/nginx.vim rename to software/vim/share/vim/vim92/ftplugin/nginx.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/nroff.vim b/software/vim/share/vim/vim92/ftplugin/nroff.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/nroff.vim rename to software/vim/share/vim/vim92/ftplugin/nroff.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/nsis.vim b/software/vim/share/vim/vim92/ftplugin/nsis.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/nsis.vim rename to software/vim/share/vim/vim92/ftplugin/nsis.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/objc.vim b/software/vim/share/vim/vim92/ftplugin/objc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/objc.vim rename to software/vim/share/vim/vim92/ftplugin/objc.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/obse.vim b/software/vim/share/vim/vim92/ftplugin/obse.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/obse.vim rename to software/vim/share/vim/vim92/ftplugin/obse.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/ocaml.vim b/software/vim/share/vim/vim92/ftplugin/ocaml.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/ocaml.vim rename to software/vim/share/vim/vim92/ftplugin/ocaml.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/occam.vim b/software/vim/share/vim/vim92/ftplugin/occam.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/occam.vim rename to software/vim/share/vim/vim92/ftplugin/occam.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/octave.vim b/software/vim/share/vim/vim92/ftplugin/octave.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/octave.vim rename to software/vim/share/vim/vim92/ftplugin/octave.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/openvpn.vim b/software/vim/share/vim/vim92/ftplugin/openvpn.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/openvpn.vim rename to software/vim/share/vim/vim92/ftplugin/openvpn.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/pamconf.vim b/software/vim/share/vim/vim92/ftplugin/pamconf.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/pamconf.vim rename to software/vim/share/vim/vim92/ftplugin/pamconf.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/pascal.vim b/software/vim/share/vim/vim92/ftplugin/pascal.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/pascal.vim rename to software/vim/share/vim/vim92/ftplugin/pascal.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/passwd.vim b/software/vim/share/vim/vim92/ftplugin/passwd.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/passwd.vim rename to software/vim/share/vim/vim92/ftplugin/passwd.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/pbtxt.vim b/software/vim/share/vim/vim92/ftplugin/pbtxt.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/pbtxt.vim rename to software/vim/share/vim/vim92/ftplugin/pbtxt.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/pdf.vim b/software/vim/share/vim/vim92/ftplugin/pdf.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/pdf.vim rename to software/vim/share/vim/vim92/ftplugin/pdf.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/perl.vim b/software/vim/share/vim/vim92/ftplugin/perl.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/perl.vim rename to software/vim/share/vim/vim92/ftplugin/perl.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/php.vim b/software/vim/share/vim/vim92/ftplugin/php.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/php.vim rename to software/vim/share/vim/vim92/ftplugin/php.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/pinfo.vim b/software/vim/share/vim/vim92/ftplugin/pinfo.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/pinfo.vim rename to software/vim/share/vim/vim92/ftplugin/pinfo.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/plaintex.vim b/software/vim/share/vim/vim92/ftplugin/plaintex.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/plaintex.vim rename to software/vim/share/vim/vim92/ftplugin/plaintex.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/pod.vim b/software/vim/share/vim/vim92/ftplugin/pod.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/pod.vim rename to software/vim/share/vim/vim92/ftplugin/pod.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/poefilter.vim b/software/vim/share/vim/vim92/ftplugin/poefilter.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/poefilter.vim rename to software/vim/share/vim/vim92/ftplugin/poefilter.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/poke.vim b/software/vim/share/vim/vim92/ftplugin/poke.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/poke.vim rename to software/vim/share/vim/vim92/ftplugin/poke.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/postscr.vim b/software/vim/share/vim/vim92/ftplugin/postscr.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/postscr.vim rename to software/vim/share/vim/vim92/ftplugin/postscr.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/procmail.vim b/software/vim/share/vim/vim92/ftplugin/procmail.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/procmail.vim rename to software/vim/share/vim/vim92/ftplugin/procmail.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/prolog.vim b/software/vim/share/vim/vim92/ftplugin/prolog.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/prolog.vim rename to software/vim/share/vim/vim92/ftplugin/prolog.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/protocols.vim b/software/vim/share/vim/vim92/ftplugin/protocols.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/protocols.vim rename to software/vim/share/vim/vim92/ftplugin/protocols.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/ps1.vim b/software/vim/share/vim/vim92/ftplugin/ps1.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/ps1.vim rename to software/vim/share/vim/vim92/ftplugin/ps1.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/ps1xml.vim b/software/vim/share/vim/vim92/ftplugin/ps1xml.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/ps1xml.vim rename to software/vim/share/vim/vim92/ftplugin/ps1xml.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/pyrex.vim b/software/vim/share/vim/vim92/ftplugin/pyrex.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/pyrex.vim rename to software/vim/share/vim/vim92/ftplugin/pyrex.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/python.vim b/software/vim/share/vim/vim92/ftplugin/python.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/python.vim rename to software/vim/share/vim/vim92/ftplugin/python.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/qb64.vim b/software/vim/share/vim/vim92/ftplugin/qb64.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/qb64.vim rename to software/vim/share/vim/vim92/ftplugin/qb64.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/qf.vim b/software/vim/share/vim/vim92/ftplugin/qf.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/qf.vim rename to software/vim/share/vim/vim92/ftplugin/qf.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/quake.vim b/software/vim/share/vim/vim92/ftplugin/quake.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/quake.vim rename to software/vim/share/vim/vim92/ftplugin/quake.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/quarto.vim b/software/vim/share/vim/vim92/ftplugin/quarto.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/quarto.vim rename to software/vim/share/vim/vim92/ftplugin/quarto.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/r.vim b/software/vim/share/vim/vim92/ftplugin/r.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/r.vim rename to software/vim/share/vim/vim92/ftplugin/r.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/racc.vim b/software/vim/share/vim/vim92/ftplugin/racc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/racc.vim rename to software/vim/share/vim/vim92/ftplugin/racc.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/racket.vim b/software/vim/share/vim/vim92/ftplugin/racket.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/racket.vim rename to software/vim/share/vim/vim92/ftplugin/racket.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/raku.vim b/software/vim/share/vim/vim92/ftplugin/raku.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/raku.vim rename to software/vim/share/vim/vim92/ftplugin/raku.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/readline.vim b/software/vim/share/vim/vim92/ftplugin/readline.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/readline.vim rename to software/vim/share/vim/vim92/ftplugin/readline.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/registry.vim b/software/vim/share/vim/vim92/ftplugin/registry.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/registry.vim rename to software/vim/share/vim/vim92/ftplugin/registry.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/reva.vim b/software/vim/share/vim/vim92/ftplugin/reva.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/reva.vim rename to software/vim/share/vim/vim92/ftplugin/reva.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/rhelp.vim b/software/vim/share/vim/vim92/ftplugin/rhelp.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/rhelp.vim rename to software/vim/share/vim/vim92/ftplugin/rhelp.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/rmd.vim b/software/vim/share/vim/vim92/ftplugin/rmd.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/rmd.vim rename to software/vim/share/vim/vim92/ftplugin/rmd.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/rnc.vim b/software/vim/share/vim/vim92/ftplugin/rnc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/rnc.vim rename to software/vim/share/vim/vim92/ftplugin/rnc.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/rnoweb.vim b/software/vim/share/vim/vim92/ftplugin/rnoweb.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/rnoweb.vim rename to software/vim/share/vim/vim92/ftplugin/rnoweb.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/routeros.vim b/software/vim/share/vim/vim92/ftplugin/routeros.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/routeros.vim rename to software/vim/share/vim/vim92/ftplugin/routeros.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/rpl.vim b/software/vim/share/vim/vim92/ftplugin/rpl.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/rpl.vim rename to software/vim/share/vim/vim92/ftplugin/rpl.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/rrst.vim b/software/vim/share/vim/vim92/ftplugin/rrst.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/rrst.vim rename to software/vim/share/vim/vim92/ftplugin/rrst.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/rst.vim b/software/vim/share/vim/vim92/ftplugin/rst.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/rst.vim rename to software/vim/share/vim/vim92/ftplugin/rst.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/ruby.vim b/software/vim/share/vim/vim92/ftplugin/ruby.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/ruby.vim rename to software/vim/share/vim/vim92/ftplugin/ruby.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/rust.vim b/software/vim/share/vim/vim92/ftplugin/rust.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/rust.vim rename to software/vim/share/vim/vim92/ftplugin/rust.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/sass.vim b/software/vim/share/vim/vim92/ftplugin/sass.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/sass.vim rename to software/vim/share/vim/vim92/ftplugin/sass.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/sbt.vim b/software/vim/share/vim/vim92/ftplugin/sbt.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/sbt.vim rename to software/vim/share/vim/vim92/ftplugin/sbt.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/scala.vim b/software/vim/share/vim/vim92/ftplugin/scala.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/scala.vim rename to software/vim/share/vim/vim92/ftplugin/scala.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/scdoc.vim b/software/vim/share/vim/vim92/ftplugin/scdoc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/scdoc.vim rename to software/vim/share/vim/vim92/ftplugin/scdoc.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/scheme.vim b/software/vim/share/vim/vim92/ftplugin/scheme.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/scheme.vim rename to software/vim/share/vim/vim92/ftplugin/scheme.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/screen.vim b/software/vim/share/vim/vim92/ftplugin/screen.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/screen.vim rename to software/vim/share/vim/vim92/ftplugin/screen.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/scss.vim b/software/vim/share/vim/vim92/ftplugin/scss.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/scss.vim rename to software/vim/share/vim/vim92/ftplugin/scss.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/sensors.vim b/software/vim/share/vim/vim92/ftplugin/sensors.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/sensors.vim rename to software/vim/share/vim/vim92/ftplugin/sensors.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/services.vim b/software/vim/share/vim/vim92/ftplugin/services.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/services.vim rename to software/vim/share/vim/vim92/ftplugin/services.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/setserial.vim b/software/vim/share/vim/vim92/ftplugin/setserial.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/setserial.vim rename to software/vim/share/vim/vim92/ftplugin/setserial.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/sexplib.vim b/software/vim/share/vim/vim92/ftplugin/sexplib.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/sexplib.vim rename to software/vim/share/vim/vim92/ftplugin/sexplib.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/sgml.vim b/software/vim/share/vim/vim92/ftplugin/sgml.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/sgml.vim rename to software/vim/share/vim/vim92/ftplugin/sgml.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/sh.vim b/software/vim/share/vim/vim92/ftplugin/sh.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/sh.vim rename to software/vim/share/vim/vim92/ftplugin/sh.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/sieve.vim b/software/vim/share/vim/vim92/ftplugin/sieve.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/sieve.vim rename to software/vim/share/vim/vim92/ftplugin/sieve.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/slpconf.vim b/software/vim/share/vim/vim92/ftplugin/slpconf.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/slpconf.vim rename to software/vim/share/vim/vim92/ftplugin/slpconf.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/slpreg.vim b/software/vim/share/vim/vim92/ftplugin/slpreg.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/slpreg.vim rename to software/vim/share/vim/vim92/ftplugin/slpreg.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/slpspi.vim b/software/vim/share/vim/vim92/ftplugin/slpspi.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/slpspi.vim rename to software/vim/share/vim/vim92/ftplugin/slpspi.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/solution.vim b/software/vim/share/vim/vim92/ftplugin/solution.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/solution.vim rename to software/vim/share/vim/vim92/ftplugin/solution.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/spec.vim b/software/vim/share/vim/vim92/ftplugin/spec.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/spec.vim rename to software/vim/share/vim/vim92/ftplugin/spec.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/sql.vim b/software/vim/share/vim/vim92/ftplugin/sql.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/sql.vim rename to software/vim/share/vim/vim92/ftplugin/sql.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/ssa.vim b/software/vim/share/vim/vim92/ftplugin/ssa.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/ssa.vim rename to software/vim/share/vim/vim92/ftplugin/ssa.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/sshconfig.vim b/software/vim/share/vim/vim92/ftplugin/sshconfig.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/sshconfig.vim rename to software/vim/share/vim/vim92/ftplugin/sshconfig.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/sudoers.vim b/software/vim/share/vim/vim92/ftplugin/sudoers.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/sudoers.vim rename to software/vim/share/vim/vim92/ftplugin/sudoers.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/svg.vim b/software/vim/share/vim/vim92/ftplugin/svg.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/svg.vim rename to software/vim/share/vim/vim92/ftplugin/svg.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/swayconfig.vim b/software/vim/share/vim/vim92/ftplugin/swayconfig.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/swayconfig.vim rename to software/vim/share/vim/vim92/ftplugin/swayconfig.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/swift.vim b/software/vim/share/vim/vim92/ftplugin/swift.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/swift.vim rename to software/vim/share/vim/vim92/ftplugin/swift.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/swiftgyb.vim b/software/vim/share/vim/vim92/ftplugin/swiftgyb.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/swiftgyb.vim rename to software/vim/share/vim/vim92/ftplugin/swiftgyb.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/sysctl.vim b/software/vim/share/vim/vim92/ftplugin/sysctl.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/sysctl.vim rename to software/vim/share/vim/vim92/ftplugin/sysctl.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/systemd.vim b/software/vim/share/vim/vim92/ftplugin/systemd.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/systemd.vim rename to software/vim/share/vim/vim92/ftplugin/systemd.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/systemverilog.vim b/software/vim/share/vim/vim92/ftplugin/systemverilog.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/systemverilog.vim rename to software/vim/share/vim/vim92/ftplugin/systemverilog.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/tap.vim b/software/vim/share/vim/vim92/ftplugin/tap.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/tap.vim rename to software/vim/share/vim/vim92/ftplugin/tap.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/tcl.vim b/software/vim/share/vim/vim92/ftplugin/tcl.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/tcl.vim rename to software/vim/share/vim/vim92/ftplugin/tcl.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/tcsh.vim b/software/vim/share/vim/vim92/ftplugin/tcsh.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/tcsh.vim rename to software/vim/share/vim/vim92/ftplugin/tcsh.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/terminfo.vim b/software/vim/share/vim/vim92/ftplugin/terminfo.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/terminfo.vim rename to software/vim/share/vim/vim92/ftplugin/terminfo.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/tex.vim b/software/vim/share/vim/vim92/ftplugin/tex.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/tex.vim rename to software/vim/share/vim/vim92/ftplugin/tex.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/text.vim b/software/vim/share/vim/vim92/ftplugin/text.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/text.vim rename to software/vim/share/vim/vim92/ftplugin/text.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/tidy.vim b/software/vim/share/vim/vim92/ftplugin/tidy.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/tidy.vim rename to software/vim/share/vim/vim92/ftplugin/tidy.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/tmux.vim b/software/vim/share/vim/vim92/ftplugin/tmux.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/tmux.vim rename to software/vim/share/vim/vim92/ftplugin/tmux.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/toml.vim b/software/vim/share/vim/vim92/ftplugin/toml.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/toml.vim rename to software/vim/share/vim/vim92/ftplugin/toml.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/treetop.vim b/software/vim/share/vim/vim92/ftplugin/treetop.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/treetop.vim rename to software/vim/share/vim/vim92/ftplugin/treetop.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/tt2html.vim b/software/vim/share/vim/vim92/ftplugin/tt2html.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/tt2html.vim rename to software/vim/share/vim/vim92/ftplugin/tt2html.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/typescript.vim b/software/vim/share/vim/vim92/ftplugin/typescript.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/typescript.vim rename to software/vim/share/vim/vim92/ftplugin/typescript.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/typescriptreact.vim b/software/vim/share/vim/vim92/ftplugin/typescriptreact.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/typescriptreact.vim rename to software/vim/share/vim/vim92/ftplugin/typescriptreact.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/udevconf.vim b/software/vim/share/vim/vim92/ftplugin/udevconf.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/udevconf.vim rename to software/vim/share/vim/vim92/ftplugin/udevconf.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/udevperm.vim b/software/vim/share/vim/vim92/ftplugin/udevperm.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/udevperm.vim rename to software/vim/share/vim/vim92/ftplugin/udevperm.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/udevrules.vim b/software/vim/share/vim/vim92/ftplugin/udevrules.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/udevrules.vim rename to software/vim/share/vim/vim92/ftplugin/udevrules.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/updatedb.vim b/software/vim/share/vim/vim92/ftplugin/updatedb.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/updatedb.vim rename to software/vim/share/vim/vim92/ftplugin/updatedb.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/vb.vim b/software/vim/share/vim/vim92/ftplugin/vb.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/vb.vim rename to software/vim/share/vim/vim92/ftplugin/vb.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/vdf.vim b/software/vim/share/vim/vim92/ftplugin/vdf.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/vdf.vim rename to software/vim/share/vim/vim92/ftplugin/vdf.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/verilog.vim b/software/vim/share/vim/vim92/ftplugin/verilog.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/verilog.vim rename to software/vim/share/vim/vim92/ftplugin/verilog.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/vhdl.vim b/software/vim/share/vim/vim92/ftplugin/vhdl.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/vhdl.vim rename to software/vim/share/vim/vim92/ftplugin/vhdl.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/vim.vim b/software/vim/share/vim/vim92/ftplugin/vim.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/vim.vim rename to software/vim/share/vim/vim92/ftplugin/vim.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/vroom.vim b/software/vim/share/vim/vim92/ftplugin/vroom.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/vroom.vim rename to software/vim/share/vim/vim92/ftplugin/vroom.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/vue.vim b/software/vim/share/vim/vim92/ftplugin/vue.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/vue.vim rename to software/vim/share/vim/vim92/ftplugin/vue.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/wast.vim b/software/vim/share/vim/vim92/ftplugin/wast.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/wast.vim rename to software/vim/share/vim/vim92/ftplugin/wast.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/wget.vim b/software/vim/share/vim/vim92/ftplugin/wget.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/wget.vim rename to software/vim/share/vim/vim92/ftplugin/wget.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/wget2.vim b/software/vim/share/vim/vim92/ftplugin/wget2.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/wget2.vim rename to software/vim/share/vim/vim92/ftplugin/wget2.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/xdefaults.vim b/software/vim/share/vim/vim92/ftplugin/xdefaults.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/xdefaults.vim rename to software/vim/share/vim/vim92/ftplugin/xdefaults.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/xf86conf.vim b/software/vim/share/vim/vim92/ftplugin/xf86conf.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/xf86conf.vim rename to software/vim/share/vim/vim92/ftplugin/xf86conf.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/xhtml.vim b/software/vim/share/vim/vim92/ftplugin/xhtml.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/xhtml.vim rename to software/vim/share/vim/vim92/ftplugin/xhtml.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/xinetd.vim b/software/vim/share/vim/vim92/ftplugin/xinetd.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/xinetd.vim rename to software/vim/share/vim/vim92/ftplugin/xinetd.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/xml.vim b/software/vim/share/vim/vim92/ftplugin/xml.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/xml.vim rename to software/vim/share/vim/vim92/ftplugin/xml.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/xmodmap.vim b/software/vim/share/vim/vim92/ftplugin/xmodmap.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/xmodmap.vim rename to software/vim/share/vim/vim92/ftplugin/xmodmap.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/xs.vim b/software/vim/share/vim/vim92/ftplugin/xs.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/xs.vim rename to software/vim/share/vim/vim92/ftplugin/xs.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/xsd.vim b/software/vim/share/vim/vim92/ftplugin/xsd.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/xsd.vim rename to software/vim/share/vim/vim92/ftplugin/xsd.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/xslt.vim b/software/vim/share/vim/vim92/ftplugin/xslt.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/xslt.vim rename to software/vim/share/vim/vim92/ftplugin/xslt.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/yaml.vim b/software/vim/share/vim/vim92/ftplugin/yaml.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/yaml.vim rename to software/vim/share/vim/vim92/ftplugin/yaml.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/zig.vim b/software/vim/share/vim/vim92/ftplugin/zig.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/zig.vim rename to software/vim/share/vim/vim92/ftplugin/zig.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/zimbu.vim b/software/vim/share/vim/vim92/ftplugin/zimbu.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/zimbu.vim rename to software/vim/share/vim/vim92/ftplugin/zimbu.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugin/zsh.vim b/software/vim/share/vim/vim92/ftplugin/zsh.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugin/zsh.vim rename to software/vim/share/vim/vim92/ftplugin/zsh.vim diff --git a/registry/software/vim/share/vim/vim92/ftplugof.vim b/software/vim/share/vim/vim92/ftplugof.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/ftplugof.vim rename to software/vim/share/vim/vim92/ftplugof.vim diff --git a/registry/software/vim/share/vim/vim92/gvimrc_example.vim b/software/vim/share/vim/vim92/gvimrc_example.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/gvimrc_example.vim rename to software/vim/share/vim/vim92/gvimrc_example.vim diff --git a/registry/software/vim/share/vim/vim92/indent.vim b/software/vim/share/vim/vim92/indent.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent.vim rename to software/vim/share/vim/vim92/indent.vim diff --git a/registry/software/vim/share/vim/vim92/indent/aap.vim b/software/vim/share/vim/vim92/indent/aap.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/aap.vim rename to software/vim/share/vim/vim92/indent/aap.vim diff --git a/registry/software/vim/share/vim/vim92/indent/ada.vim b/software/vim/share/vim/vim92/indent/ada.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/ada.vim rename to software/vim/share/vim/vim92/indent/ada.vim diff --git a/registry/software/vim/share/vim/vim92/indent/ant.vim b/software/vim/share/vim/vim92/indent/ant.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/ant.vim rename to software/vim/share/vim/vim92/indent/ant.vim diff --git a/registry/software/vim/share/vim/vim92/indent/automake.vim b/software/vim/share/vim/vim92/indent/automake.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/automake.vim rename to software/vim/share/vim/vim92/indent/automake.vim diff --git a/registry/software/vim/share/vim/vim92/indent/awk.vim b/software/vim/share/vim/vim92/indent/awk.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/awk.vim rename to software/vim/share/vim/vim92/indent/awk.vim diff --git a/registry/software/vim/share/vim/vim92/indent/bash.vim b/software/vim/share/vim/vim92/indent/bash.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/bash.vim rename to software/vim/share/vim/vim92/indent/bash.vim diff --git a/registry/software/vim/share/vim/vim92/indent/basic.vim b/software/vim/share/vim/vim92/indent/basic.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/basic.vim rename to software/vim/share/vim/vim92/indent/basic.vim diff --git a/registry/software/vim/share/vim/vim92/indent/bib.vim b/software/vim/share/vim/vim92/indent/bib.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/bib.vim rename to software/vim/share/vim/vim92/indent/bib.vim diff --git a/registry/software/vim/share/vim/vim92/indent/bitbake.vim b/software/vim/share/vim/vim92/indent/bitbake.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/bitbake.vim rename to software/vim/share/vim/vim92/indent/bitbake.vim diff --git a/registry/software/vim/share/vim/vim92/indent/bst.vim b/software/vim/share/vim/vim92/indent/bst.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/bst.vim rename to software/vim/share/vim/vim92/indent/bst.vim diff --git a/registry/software/vim/share/vim/vim92/indent/bzl.vim b/software/vim/share/vim/vim92/indent/bzl.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/bzl.vim rename to software/vim/share/vim/vim92/indent/bzl.vim diff --git a/registry/software/vim/share/vim/vim92/indent/c.vim b/software/vim/share/vim/vim92/indent/c.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/c.vim rename to software/vim/share/vim/vim92/indent/c.vim diff --git a/registry/software/vim/share/vim/vim92/indent/cdl.vim b/software/vim/share/vim/vim92/indent/cdl.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/cdl.vim rename to software/vim/share/vim/vim92/indent/cdl.vim diff --git a/registry/software/vim/share/vim/vim92/indent/ch.vim b/software/vim/share/vim/vim92/indent/ch.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/ch.vim rename to software/vim/share/vim/vim92/indent/ch.vim diff --git a/registry/software/vim/share/vim/vim92/indent/chaiscript.vim b/software/vim/share/vim/vim92/indent/chaiscript.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/chaiscript.vim rename to software/vim/share/vim/vim92/indent/chaiscript.vim diff --git a/registry/software/vim/share/vim/vim92/indent/changelog.vim b/software/vim/share/vim/vim92/indent/changelog.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/changelog.vim rename to software/vim/share/vim/vim92/indent/changelog.vim diff --git a/registry/software/vim/share/vim/vim92/indent/chatito.vim b/software/vim/share/vim/vim92/indent/chatito.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/chatito.vim rename to software/vim/share/vim/vim92/indent/chatito.vim diff --git a/registry/software/vim/share/vim/vim92/indent/clojure.vim b/software/vim/share/vim/vim92/indent/clojure.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/clojure.vim rename to software/vim/share/vim/vim92/indent/clojure.vim diff --git a/registry/software/vim/share/vim/vim92/indent/cmake.vim b/software/vim/share/vim/vim92/indent/cmake.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/cmake.vim rename to software/vim/share/vim/vim92/indent/cmake.vim diff --git a/registry/software/vim/share/vim/vim92/indent/cobol.vim b/software/vim/share/vim/vim92/indent/cobol.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/cobol.vim rename to software/vim/share/vim/vim92/indent/cobol.vim diff --git a/registry/software/vim/share/vim/vim92/indent/config.vim b/software/vim/share/vim/vim92/indent/config.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/config.vim rename to software/vim/share/vim/vim92/indent/config.vim diff --git a/registry/software/vim/share/vim/vim92/indent/context.vim b/software/vim/share/vim/vim92/indent/context.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/context.vim rename to software/vim/share/vim/vim92/indent/context.vim diff --git a/registry/software/vim/share/vim/vim92/indent/cpp.vim b/software/vim/share/vim/vim92/indent/cpp.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/cpp.vim rename to software/vim/share/vim/vim92/indent/cpp.vim diff --git a/registry/software/vim/share/vim/vim92/indent/cs.vim b/software/vim/share/vim/vim92/indent/cs.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/cs.vim rename to software/vim/share/vim/vim92/indent/cs.vim diff --git a/registry/software/vim/share/vim/vim92/indent/css.vim b/software/vim/share/vim/vim92/indent/css.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/css.vim rename to software/vim/share/vim/vim92/indent/css.vim diff --git a/registry/software/vim/share/vim/vim92/indent/cucumber.vim b/software/vim/share/vim/vim92/indent/cucumber.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/cucumber.vim rename to software/vim/share/vim/vim92/indent/cucumber.vim diff --git a/registry/software/vim/share/vim/vim92/indent/cuda.vim b/software/vim/share/vim/vim92/indent/cuda.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/cuda.vim rename to software/vim/share/vim/vim92/indent/cuda.vim diff --git a/registry/software/vim/share/vim/vim92/indent/d.vim b/software/vim/share/vim/vim92/indent/d.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/d.vim rename to software/vim/share/vim/vim92/indent/d.vim diff --git a/registry/software/vim/share/vim/vim92/indent/dictconf.vim b/software/vim/share/vim/vim92/indent/dictconf.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/dictconf.vim rename to software/vim/share/vim/vim92/indent/dictconf.vim diff --git a/registry/software/vim/share/vim/vim92/indent/dictdconf.vim b/software/vim/share/vim/vim92/indent/dictdconf.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/dictdconf.vim rename to software/vim/share/vim/vim92/indent/dictdconf.vim diff --git a/registry/software/vim/share/vim/vim92/indent/docbk.vim b/software/vim/share/vim/vim92/indent/docbk.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/docbk.vim rename to software/vim/share/vim/vim92/indent/docbk.vim diff --git a/registry/software/vim/share/vim/vim92/indent/dosbatch.vim b/software/vim/share/vim/vim92/indent/dosbatch.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/dosbatch.vim rename to software/vim/share/vim/vim92/indent/dosbatch.vim diff --git a/registry/software/vim/share/vim/vim92/indent/dtd.vim b/software/vim/share/vim/vim92/indent/dtd.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/dtd.vim rename to software/vim/share/vim/vim92/indent/dtd.vim diff --git a/registry/software/vim/share/vim/vim92/indent/dtrace.vim b/software/vim/share/vim/vim92/indent/dtrace.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/dtrace.vim rename to software/vim/share/vim/vim92/indent/dtrace.vim diff --git a/registry/software/vim/share/vim/vim92/indent/dune.vim b/software/vim/share/vim/vim92/indent/dune.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/dune.vim rename to software/vim/share/vim/vim92/indent/dune.vim diff --git a/registry/software/vim/share/vim/vim92/indent/dylan.vim b/software/vim/share/vim/vim92/indent/dylan.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/dylan.vim rename to software/vim/share/vim/vim92/indent/dylan.vim diff --git a/registry/software/vim/share/vim/vim92/indent/eiffel.vim b/software/vim/share/vim/vim92/indent/eiffel.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/eiffel.vim rename to software/vim/share/vim/vim92/indent/eiffel.vim diff --git a/registry/software/vim/share/vim/vim92/indent/elm.vim b/software/vim/share/vim/vim92/indent/elm.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/elm.vim rename to software/vim/share/vim/vim92/indent/elm.vim diff --git a/registry/software/vim/share/vim/vim92/indent/erlang.vim b/software/vim/share/vim/vim92/indent/erlang.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/erlang.vim rename to software/vim/share/vim/vim92/indent/erlang.vim diff --git a/registry/software/vim/share/vim/vim92/indent/eruby.vim b/software/vim/share/vim/vim92/indent/eruby.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/eruby.vim rename to software/vim/share/vim/vim92/indent/eruby.vim diff --git a/registry/software/vim/share/vim/vim92/indent/eterm.vim b/software/vim/share/vim/vim92/indent/eterm.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/eterm.vim rename to software/vim/share/vim/vim92/indent/eterm.vim diff --git a/registry/software/vim/share/vim/vim92/indent/expect.vim b/software/vim/share/vim/vim92/indent/expect.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/expect.vim rename to software/vim/share/vim/vim92/indent/expect.vim diff --git a/registry/software/vim/share/vim/vim92/indent/falcon.vim b/software/vim/share/vim/vim92/indent/falcon.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/falcon.vim rename to software/vim/share/vim/vim92/indent/falcon.vim diff --git a/registry/software/vim/share/vim/vim92/indent/fennel.vim b/software/vim/share/vim/vim92/indent/fennel.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/fennel.vim rename to software/vim/share/vim/vim92/indent/fennel.vim diff --git a/registry/software/vim/share/vim/vim92/indent/fish.vim b/software/vim/share/vim/vim92/indent/fish.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/fish.vim rename to software/vim/share/vim/vim92/indent/fish.vim diff --git a/registry/software/vim/share/vim/vim92/indent/fortran.vim b/software/vim/share/vim/vim92/indent/fortran.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/fortran.vim rename to software/vim/share/vim/vim92/indent/fortran.vim diff --git a/registry/software/vim/share/vim/vim92/indent/framescript.vim b/software/vim/share/vim/vim92/indent/framescript.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/framescript.vim rename to software/vim/share/vim/vim92/indent/framescript.vim diff --git a/registry/software/vim/share/vim/vim92/indent/freebasic.vim b/software/vim/share/vim/vim92/indent/freebasic.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/freebasic.vim rename to software/vim/share/vim/vim92/indent/freebasic.vim diff --git a/registry/software/vim/share/vim/vim92/indent/gdscript.vim b/software/vim/share/vim/vim92/indent/gdscript.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/gdscript.vim rename to software/vim/share/vim/vim92/indent/gdscript.vim diff --git a/registry/software/vim/share/vim/vim92/indent/gitconfig.vim b/software/vim/share/vim/vim92/indent/gitconfig.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/gitconfig.vim rename to software/vim/share/vim/vim92/indent/gitconfig.vim diff --git a/registry/software/vim/share/vim/vim92/indent/gitolite.vim b/software/vim/share/vim/vim92/indent/gitolite.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/gitolite.vim rename to software/vim/share/vim/vim92/indent/gitolite.vim diff --git a/registry/software/vim/share/vim/vim92/indent/go.vim b/software/vim/share/vim/vim92/indent/go.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/go.vim rename to software/vim/share/vim/vim92/indent/go.vim diff --git a/registry/software/vim/share/vim/vim92/indent/gyp.vim b/software/vim/share/vim/vim92/indent/gyp.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/gyp.vim rename to software/vim/share/vim/vim92/indent/gyp.vim diff --git a/registry/software/vim/share/vim/vim92/indent/haml.vim b/software/vim/share/vim/vim92/indent/haml.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/haml.vim rename to software/vim/share/vim/vim92/indent/haml.vim diff --git a/registry/software/vim/share/vim/vim92/indent/hamster.vim b/software/vim/share/vim/vim92/indent/hamster.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/hamster.vim rename to software/vim/share/vim/vim92/indent/hamster.vim diff --git a/registry/software/vim/share/vim/vim92/indent/hare.vim b/software/vim/share/vim/vim92/indent/hare.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/hare.vim rename to software/vim/share/vim/vim92/indent/hare.vim diff --git a/registry/software/vim/share/vim/vim92/indent/hog.vim b/software/vim/share/vim/vim92/indent/hog.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/hog.vim rename to software/vim/share/vim/vim92/indent/hog.vim diff --git a/registry/software/vim/share/vim/vim92/indent/html.vim b/software/vim/share/vim/vim92/indent/html.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/html.vim rename to software/vim/share/vim/vim92/indent/html.vim diff --git a/registry/software/vim/share/vim/vim92/indent/htmldjango.vim b/software/vim/share/vim/vim92/indent/htmldjango.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/htmldjango.vim rename to software/vim/share/vim/vim92/indent/htmldjango.vim diff --git a/registry/software/vim/share/vim/vim92/indent/idlang.vim b/software/vim/share/vim/vim92/indent/idlang.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/idlang.vim rename to software/vim/share/vim/vim92/indent/idlang.vim diff --git a/registry/software/vim/share/vim/vim92/indent/ishd.vim b/software/vim/share/vim/vim92/indent/ishd.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/ishd.vim rename to software/vim/share/vim/vim92/indent/ishd.vim diff --git a/registry/software/vim/share/vim/vim92/indent/j.vim b/software/vim/share/vim/vim92/indent/j.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/j.vim rename to software/vim/share/vim/vim92/indent/j.vim diff --git a/registry/software/vim/share/vim/vim92/indent/java.vim b/software/vim/share/vim/vim92/indent/java.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/java.vim rename to software/vim/share/vim/vim92/indent/java.vim diff --git a/registry/software/vim/share/vim/vim92/indent/javascript.vim b/software/vim/share/vim/vim92/indent/javascript.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/javascript.vim rename to software/vim/share/vim/vim92/indent/javascript.vim diff --git a/registry/software/vim/share/vim/vim92/indent/javascriptreact.vim b/software/vim/share/vim/vim92/indent/javascriptreact.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/javascriptreact.vim rename to software/vim/share/vim/vim92/indent/javascriptreact.vim diff --git a/registry/software/vim/share/vim/vim92/indent/json.vim b/software/vim/share/vim/vim92/indent/json.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/json.vim rename to software/vim/share/vim/vim92/indent/json.vim diff --git a/registry/software/vim/share/vim/vim92/indent/jsonc.vim b/software/vim/share/vim/vim92/indent/jsonc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/jsonc.vim rename to software/vim/share/vim/vim92/indent/jsonc.vim diff --git a/registry/software/vim/share/vim/vim92/indent/jsp.vim b/software/vim/share/vim/vim92/indent/jsp.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/jsp.vim rename to software/vim/share/vim/vim92/indent/jsp.vim diff --git a/registry/software/vim/share/vim/vim92/indent/julia.vim b/software/vim/share/vim/vim92/indent/julia.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/julia.vim rename to software/vim/share/vim/vim92/indent/julia.vim diff --git a/registry/software/vim/share/vim/vim92/indent/krl.vim b/software/vim/share/vim/vim92/indent/krl.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/krl.vim rename to software/vim/share/vim/vim92/indent/krl.vim diff --git a/registry/software/vim/share/vim/vim92/indent/ld.vim b/software/vim/share/vim/vim92/indent/ld.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/ld.vim rename to software/vim/share/vim/vim92/indent/ld.vim diff --git a/registry/software/vim/share/vim/vim92/indent/less.vim b/software/vim/share/vim/vim92/indent/less.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/less.vim rename to software/vim/share/vim/vim92/indent/less.vim diff --git a/registry/software/vim/share/vim/vim92/indent/lifelines.vim b/software/vim/share/vim/vim92/indent/lifelines.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/lifelines.vim rename to software/vim/share/vim/vim92/indent/lifelines.vim diff --git a/registry/software/vim/share/vim/vim92/indent/liquid.vim b/software/vim/share/vim/vim92/indent/liquid.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/liquid.vim rename to software/vim/share/vim/vim92/indent/liquid.vim diff --git a/registry/software/vim/share/vim/vim92/indent/lisp.vim b/software/vim/share/vim/vim92/indent/lisp.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/lisp.vim rename to software/vim/share/vim/vim92/indent/lisp.vim diff --git a/registry/software/vim/share/vim/vim92/indent/logtalk.vim b/software/vim/share/vim/vim92/indent/logtalk.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/logtalk.vim rename to software/vim/share/vim/vim92/indent/logtalk.vim diff --git a/registry/software/vim/share/vim/vim92/indent/lua.vim b/software/vim/share/vim/vim92/indent/lua.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/lua.vim rename to software/vim/share/vim/vim92/indent/lua.vim diff --git a/registry/software/vim/share/vim/vim92/indent/mail.vim b/software/vim/share/vim/vim92/indent/mail.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/mail.vim rename to software/vim/share/vim/vim92/indent/mail.vim diff --git a/registry/software/vim/share/vim/vim92/indent/make.vim b/software/vim/share/vim/vim92/indent/make.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/make.vim rename to software/vim/share/vim/vim92/indent/make.vim diff --git a/registry/software/vim/share/vim/vim92/indent/matlab.vim b/software/vim/share/vim/vim92/indent/matlab.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/matlab.vim rename to software/vim/share/vim/vim92/indent/matlab.vim diff --git a/registry/software/vim/share/vim/vim92/indent/meson.vim b/software/vim/share/vim/vim92/indent/meson.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/meson.vim rename to software/vim/share/vim/vim92/indent/meson.vim diff --git a/registry/software/vim/share/vim/vim92/indent/mf.vim b/software/vim/share/vim/vim92/indent/mf.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/mf.vim rename to software/vim/share/vim/vim92/indent/mf.vim diff --git a/registry/software/vim/share/vim/vim92/indent/mma.vim b/software/vim/share/vim/vim92/indent/mma.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/mma.vim rename to software/vim/share/vim/vim92/indent/mma.vim diff --git a/registry/software/vim/share/vim/vim92/indent/mp.vim b/software/vim/share/vim/vim92/indent/mp.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/mp.vim rename to software/vim/share/vim/vim92/indent/mp.vim diff --git a/registry/software/vim/share/vim/vim92/indent/nginx.vim b/software/vim/share/vim/vim92/indent/nginx.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/nginx.vim rename to software/vim/share/vim/vim92/indent/nginx.vim diff --git a/registry/software/vim/share/vim/vim92/indent/nsis.vim b/software/vim/share/vim/vim92/indent/nsis.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/nsis.vim rename to software/vim/share/vim/vim92/indent/nsis.vim diff --git a/registry/software/vim/share/vim/vim92/indent/objc.vim b/software/vim/share/vim/vim92/indent/objc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/objc.vim rename to software/vim/share/vim/vim92/indent/objc.vim diff --git a/registry/software/vim/share/vim/vim92/indent/obse.vim b/software/vim/share/vim/vim92/indent/obse.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/obse.vim rename to software/vim/share/vim/vim92/indent/obse.vim diff --git a/registry/software/vim/share/vim/vim92/indent/ocaml.vim b/software/vim/share/vim/vim92/indent/ocaml.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/ocaml.vim rename to software/vim/share/vim/vim92/indent/ocaml.vim diff --git a/registry/software/vim/share/vim/vim92/indent/occam.vim b/software/vim/share/vim/vim92/indent/occam.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/occam.vim rename to software/vim/share/vim/vim92/indent/occam.vim diff --git a/registry/software/vim/share/vim/vim92/indent/pascal.vim b/software/vim/share/vim/vim92/indent/pascal.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/pascal.vim rename to software/vim/share/vim/vim92/indent/pascal.vim diff --git a/registry/software/vim/share/vim/vim92/indent/perl.vim b/software/vim/share/vim/vim92/indent/perl.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/perl.vim rename to software/vim/share/vim/vim92/indent/perl.vim diff --git a/registry/software/vim/share/vim/vim92/indent/php.vim b/software/vim/share/vim/vim92/indent/php.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/php.vim rename to software/vim/share/vim/vim92/indent/php.vim diff --git a/registry/software/vim/share/vim/vim92/indent/postscr.vim b/software/vim/share/vim/vim92/indent/postscr.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/postscr.vim rename to software/vim/share/vim/vim92/indent/postscr.vim diff --git a/registry/software/vim/share/vim/vim92/indent/pov.vim b/software/vim/share/vim/vim92/indent/pov.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/pov.vim rename to software/vim/share/vim/vim92/indent/pov.vim diff --git a/registry/software/vim/share/vim/vim92/indent/prolog.vim b/software/vim/share/vim/vim92/indent/prolog.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/prolog.vim rename to software/vim/share/vim/vim92/indent/prolog.vim diff --git a/registry/software/vim/share/vim/vim92/indent/ps1.vim b/software/vim/share/vim/vim92/indent/ps1.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/ps1.vim rename to software/vim/share/vim/vim92/indent/ps1.vim diff --git a/registry/software/vim/share/vim/vim92/indent/pyrex.vim b/software/vim/share/vim/vim92/indent/pyrex.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/pyrex.vim rename to software/vim/share/vim/vim92/indent/pyrex.vim diff --git a/registry/software/vim/share/vim/vim92/indent/python.vim b/software/vim/share/vim/vim92/indent/python.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/python.vim rename to software/vim/share/vim/vim92/indent/python.vim diff --git a/registry/software/vim/share/vim/vim92/indent/qb64.vim b/software/vim/share/vim/vim92/indent/qb64.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/qb64.vim rename to software/vim/share/vim/vim92/indent/qb64.vim diff --git a/registry/software/vim/share/vim/vim92/indent/quarto.vim b/software/vim/share/vim/vim92/indent/quarto.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/quarto.vim rename to software/vim/share/vim/vim92/indent/quarto.vim diff --git a/registry/software/vim/share/vim/vim92/indent/r.vim b/software/vim/share/vim/vim92/indent/r.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/r.vim rename to software/vim/share/vim/vim92/indent/r.vim diff --git a/registry/software/vim/share/vim/vim92/indent/racket.vim b/software/vim/share/vim/vim92/indent/racket.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/racket.vim rename to software/vim/share/vim/vim92/indent/racket.vim diff --git a/registry/software/vim/share/vim/vim92/indent/raku.vim b/software/vim/share/vim/vim92/indent/raku.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/raku.vim rename to software/vim/share/vim/vim92/indent/raku.vim diff --git a/registry/software/vim/share/vim/vim92/indent/raml.vim b/software/vim/share/vim/vim92/indent/raml.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/raml.vim rename to software/vim/share/vim/vim92/indent/raml.vim diff --git a/registry/software/vim/share/vim/vim92/indent/readline.vim b/software/vim/share/vim/vim92/indent/readline.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/readline.vim rename to software/vim/share/vim/vim92/indent/readline.vim diff --git a/registry/software/vim/share/vim/vim92/indent/rhelp.vim b/software/vim/share/vim/vim92/indent/rhelp.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/rhelp.vim rename to software/vim/share/vim/vim92/indent/rhelp.vim diff --git a/registry/software/vim/share/vim/vim92/indent/rmd.vim b/software/vim/share/vim/vim92/indent/rmd.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/rmd.vim rename to software/vim/share/vim/vim92/indent/rmd.vim diff --git a/registry/software/vim/share/vim/vim92/indent/rnoweb.vim b/software/vim/share/vim/vim92/indent/rnoweb.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/rnoweb.vim rename to software/vim/share/vim/vim92/indent/rnoweb.vim diff --git a/registry/software/vim/share/vim/vim92/indent/rpl.vim b/software/vim/share/vim/vim92/indent/rpl.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/rpl.vim rename to software/vim/share/vim/vim92/indent/rpl.vim diff --git a/registry/software/vim/share/vim/vim92/indent/rrst.vim b/software/vim/share/vim/vim92/indent/rrst.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/rrst.vim rename to software/vim/share/vim/vim92/indent/rrst.vim diff --git a/registry/software/vim/share/vim/vim92/indent/rst.vim b/software/vim/share/vim/vim92/indent/rst.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/rst.vim rename to software/vim/share/vim/vim92/indent/rst.vim diff --git a/registry/software/vim/share/vim/vim92/indent/ruby.vim b/software/vim/share/vim/vim92/indent/ruby.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/ruby.vim rename to software/vim/share/vim/vim92/indent/ruby.vim diff --git a/registry/software/vim/share/vim/vim92/indent/rust.vim b/software/vim/share/vim/vim92/indent/rust.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/rust.vim rename to software/vim/share/vim/vim92/indent/rust.vim diff --git a/registry/software/vim/share/vim/vim92/indent/sas.vim b/software/vim/share/vim/vim92/indent/sas.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/sas.vim rename to software/vim/share/vim/vim92/indent/sas.vim diff --git a/registry/software/vim/share/vim/vim92/indent/sass.vim b/software/vim/share/vim/vim92/indent/sass.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/sass.vim rename to software/vim/share/vim/vim92/indent/sass.vim diff --git a/registry/software/vim/share/vim/vim92/indent/scala.vim b/software/vim/share/vim/vim92/indent/scala.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/scala.vim rename to software/vim/share/vim/vim92/indent/scala.vim diff --git a/registry/software/vim/share/vim/vim92/indent/scheme.vim b/software/vim/share/vim/vim92/indent/scheme.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/scheme.vim rename to software/vim/share/vim/vim92/indent/scheme.vim diff --git a/registry/software/vim/share/vim/vim92/indent/scss.vim b/software/vim/share/vim/vim92/indent/scss.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/scss.vim rename to software/vim/share/vim/vim92/indent/scss.vim diff --git a/registry/software/vim/share/vim/vim92/indent/sdl.vim b/software/vim/share/vim/vim92/indent/sdl.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/sdl.vim rename to software/vim/share/vim/vim92/indent/sdl.vim diff --git a/registry/software/vim/share/vim/vim92/indent/sh.vim b/software/vim/share/vim/vim92/indent/sh.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/sh.vim rename to software/vim/share/vim/vim92/indent/sh.vim diff --git a/registry/software/vim/share/vim/vim92/indent/sml.vim b/software/vim/share/vim/vim92/indent/sml.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/sml.vim rename to software/vim/share/vim/vim92/indent/sml.vim diff --git a/registry/software/vim/share/vim/vim92/indent/solidity.vim b/software/vim/share/vim/vim92/indent/solidity.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/solidity.vim rename to software/vim/share/vim/vim92/indent/solidity.vim diff --git a/registry/software/vim/share/vim/vim92/indent/sql.vim b/software/vim/share/vim/vim92/indent/sql.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/sql.vim rename to software/vim/share/vim/vim92/indent/sql.vim diff --git a/registry/software/vim/share/vim/vim92/indent/sqlanywhere.vim b/software/vim/share/vim/vim92/indent/sqlanywhere.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/sqlanywhere.vim rename to software/vim/share/vim/vim92/indent/sqlanywhere.vim diff --git a/registry/software/vim/share/vim/vim92/indent/sshconfig.vim b/software/vim/share/vim/vim92/indent/sshconfig.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/sshconfig.vim rename to software/vim/share/vim/vim92/indent/sshconfig.vim diff --git a/registry/software/vim/share/vim/vim92/indent/systemverilog.vim b/software/vim/share/vim/vim92/indent/systemverilog.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/systemverilog.vim rename to software/vim/share/vim/vim92/indent/systemverilog.vim diff --git a/registry/software/vim/share/vim/vim92/indent/tcl.vim b/software/vim/share/vim/vim92/indent/tcl.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/tcl.vim rename to software/vim/share/vim/vim92/indent/tcl.vim diff --git a/registry/software/vim/share/vim/vim92/indent/tcsh.vim b/software/vim/share/vim/vim92/indent/tcsh.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/tcsh.vim rename to software/vim/share/vim/vim92/indent/tcsh.vim diff --git a/registry/software/vim/share/vim/vim92/indent/teraterm.vim b/software/vim/share/vim/vim92/indent/teraterm.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/teraterm.vim rename to software/vim/share/vim/vim92/indent/teraterm.vim diff --git a/registry/software/vim/share/vim/vim92/indent/tex.vim b/software/vim/share/vim/vim92/indent/tex.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/tex.vim rename to software/vim/share/vim/vim92/indent/tex.vim diff --git a/registry/software/vim/share/vim/vim92/indent/tf.vim b/software/vim/share/vim/vim92/indent/tf.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/tf.vim rename to software/vim/share/vim/vim92/indent/tf.vim diff --git a/registry/software/vim/share/vim/vim92/indent/tilde.vim b/software/vim/share/vim/vim92/indent/tilde.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/tilde.vim rename to software/vim/share/vim/vim92/indent/tilde.vim diff --git a/registry/software/vim/share/vim/vim92/indent/treetop.vim b/software/vim/share/vim/vim92/indent/treetop.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/treetop.vim rename to software/vim/share/vim/vim92/indent/treetop.vim diff --git a/registry/software/vim/share/vim/vim92/indent/typescript.vim b/software/vim/share/vim/vim92/indent/typescript.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/typescript.vim rename to software/vim/share/vim/vim92/indent/typescript.vim diff --git a/registry/software/vim/share/vim/vim92/indent/vb.vim b/software/vim/share/vim/vim92/indent/vb.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/vb.vim rename to software/vim/share/vim/vim92/indent/vb.vim diff --git a/registry/software/vim/share/vim/vim92/indent/verilog.vim b/software/vim/share/vim/vim92/indent/verilog.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/verilog.vim rename to software/vim/share/vim/vim92/indent/verilog.vim diff --git a/registry/software/vim/share/vim/vim92/indent/vhdl.vim b/software/vim/share/vim/vim92/indent/vhdl.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/vhdl.vim rename to software/vim/share/vim/vim92/indent/vhdl.vim diff --git a/registry/software/vim/share/vim/vim92/indent/vim.vim b/software/vim/share/vim/vim92/indent/vim.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/vim.vim rename to software/vim/share/vim/vim92/indent/vim.vim diff --git a/registry/software/vim/share/vim/vim92/indent/vroom.vim b/software/vim/share/vim/vim92/indent/vroom.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/vroom.vim rename to software/vim/share/vim/vim92/indent/vroom.vim diff --git a/registry/software/vim/share/vim/vim92/indent/vue.vim b/software/vim/share/vim/vim92/indent/vue.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/vue.vim rename to software/vim/share/vim/vim92/indent/vue.vim diff --git a/registry/software/vim/share/vim/vim92/indent/wast.vim b/software/vim/share/vim/vim92/indent/wast.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/wast.vim rename to software/vim/share/vim/vim92/indent/wast.vim diff --git a/registry/software/vim/share/vim/vim92/indent/xf86conf.vim b/software/vim/share/vim/vim92/indent/xf86conf.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/xf86conf.vim rename to software/vim/share/vim/vim92/indent/xf86conf.vim diff --git a/registry/software/vim/share/vim/vim92/indent/xhtml.vim b/software/vim/share/vim/vim92/indent/xhtml.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/xhtml.vim rename to software/vim/share/vim/vim92/indent/xhtml.vim diff --git a/registry/software/vim/share/vim/vim92/indent/xinetd.vim b/software/vim/share/vim/vim92/indent/xinetd.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/xinetd.vim rename to software/vim/share/vim/vim92/indent/xinetd.vim diff --git a/registry/software/vim/share/vim/vim92/indent/xml.vim b/software/vim/share/vim/vim92/indent/xml.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/xml.vim rename to software/vim/share/vim/vim92/indent/xml.vim diff --git a/registry/software/vim/share/vim/vim92/indent/xsd.vim b/software/vim/share/vim/vim92/indent/xsd.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/xsd.vim rename to software/vim/share/vim/vim92/indent/xsd.vim diff --git a/registry/software/vim/share/vim/vim92/indent/xslt.vim b/software/vim/share/vim/vim92/indent/xslt.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/xslt.vim rename to software/vim/share/vim/vim92/indent/xslt.vim diff --git a/registry/software/vim/share/vim/vim92/indent/yacc.vim b/software/vim/share/vim/vim92/indent/yacc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/yacc.vim rename to software/vim/share/vim/vim92/indent/yacc.vim diff --git a/registry/software/vim/share/vim/vim92/indent/yaml.vim b/software/vim/share/vim/vim92/indent/yaml.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/yaml.vim rename to software/vim/share/vim/vim92/indent/yaml.vim diff --git a/registry/software/vim/share/vim/vim92/indent/zig.vim b/software/vim/share/vim/vim92/indent/zig.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/zig.vim rename to software/vim/share/vim/vim92/indent/zig.vim diff --git a/registry/software/vim/share/vim/vim92/indent/zimbu.vim b/software/vim/share/vim/vim92/indent/zimbu.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/zimbu.vim rename to software/vim/share/vim/vim92/indent/zimbu.vim diff --git a/registry/software/vim/share/vim/vim92/indent/zsh.vim b/software/vim/share/vim/vim92/indent/zsh.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indent/zsh.vim rename to software/vim/share/vim/vim92/indent/zsh.vim diff --git a/registry/software/vim/share/vim/vim92/indoff.vim b/software/vim/share/vim/vim92/indoff.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/indoff.vim rename to software/vim/share/vim/vim92/indoff.vim diff --git a/registry/software/vim/share/vim/vim92/macros/editexisting.vim b/software/vim/share/vim/vim92/macros/editexisting.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/macros/editexisting.vim rename to software/vim/share/vim/vim92/macros/editexisting.vim diff --git a/registry/software/vim/share/vim/vim92/macros/hanoi/click.me b/software/vim/share/vim/vim92/macros/hanoi/click.me similarity index 100% rename from registry/software/vim/share/vim/vim92/macros/hanoi/click.me rename to software/vim/share/vim/vim92/macros/hanoi/click.me diff --git a/registry/software/vim/share/vim/vim92/macros/hanoi/hanoi.vim b/software/vim/share/vim/vim92/macros/hanoi/hanoi.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/macros/hanoi/hanoi.vim rename to software/vim/share/vim/vim92/macros/hanoi/hanoi.vim diff --git a/registry/software/vim/share/vim/vim92/macros/hanoi/poster b/software/vim/share/vim/vim92/macros/hanoi/poster similarity index 100% rename from registry/software/vim/share/vim/vim92/macros/hanoi/poster rename to software/vim/share/vim/vim92/macros/hanoi/poster diff --git a/registry/software/vim/share/vim/vim92/macros/justify.vim b/software/vim/share/vim/vim92/macros/justify.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/macros/justify.vim rename to software/vim/share/vim/vim92/macros/justify.vim diff --git a/registry/software/vim/share/vim/vim92/macros/less.bat b/software/vim/share/vim/vim92/macros/less.bat similarity index 100% rename from registry/software/vim/share/vim/vim92/macros/less.bat rename to software/vim/share/vim/vim92/macros/less.bat diff --git a/registry/software/vim/share/vim/vim92/macros/less.sh b/software/vim/share/vim/vim92/macros/less.sh similarity index 100% rename from registry/software/vim/share/vim/vim92/macros/less.sh rename to software/vim/share/vim/vim92/macros/less.sh diff --git a/registry/software/vim/share/vim/vim92/macros/less.vim b/software/vim/share/vim/vim92/macros/less.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/macros/less.vim rename to software/vim/share/vim/vim92/macros/less.vim diff --git a/registry/software/vim/share/vim/vim92/macros/life/click.me b/software/vim/share/vim/vim92/macros/life/click.me similarity index 100% rename from registry/software/vim/share/vim/vim92/macros/life/click.me rename to software/vim/share/vim/vim92/macros/life/click.me diff --git a/registry/software/vim/share/vim/vim92/macros/life/life.vim b/software/vim/share/vim/vim92/macros/life/life.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/macros/life/life.vim rename to software/vim/share/vim/vim92/macros/life/life.vim diff --git a/registry/software/vim/share/vim/vim92/macros/matchit.vim b/software/vim/share/vim/vim92/macros/matchit.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/macros/matchit.vim rename to software/vim/share/vim/vim92/macros/matchit.vim diff --git a/registry/software/vim/share/vim/vim92/macros/maze/Makefile b/software/vim/share/vim/vim92/macros/maze/Makefile similarity index 100% rename from registry/software/vim/share/vim/vim92/macros/maze/Makefile rename to software/vim/share/vim/vim92/macros/maze/Makefile diff --git a/registry/software/vim/share/vim/vim92/macros/maze/maze.c b/software/vim/share/vim/vim92/macros/maze/maze.c similarity index 100% rename from registry/software/vim/share/vim/vim92/macros/maze/maze.c rename to software/vim/share/vim/vim92/macros/maze/maze.c diff --git a/registry/software/vim/share/vim/vim92/macros/maze/maze_5.78 b/software/vim/share/vim/vim92/macros/maze/maze_5.78 similarity index 100% rename from registry/software/vim/share/vim/vim92/macros/maze/maze_5.78 rename to software/vim/share/vim/vim92/macros/maze/maze_5.78 diff --git a/registry/software/vim/share/vim/vim92/macros/maze/maze_mac b/software/vim/share/vim/vim92/macros/maze/maze_mac similarity index 100% rename from registry/software/vim/share/vim/vim92/macros/maze/maze_mac rename to software/vim/share/vim/vim92/macros/maze/maze_mac diff --git a/registry/software/vim/share/vim/vim92/macros/maze/mazeansi.c b/software/vim/share/vim/vim92/macros/maze/mazeansi.c similarity index 100% rename from registry/software/vim/share/vim/vim92/macros/maze/mazeansi.c rename to software/vim/share/vim/vim92/macros/maze/mazeansi.c diff --git a/registry/software/vim/share/vim/vim92/macros/maze/mazeclean.c b/software/vim/share/vim/vim92/macros/maze/mazeclean.c similarity index 100% rename from registry/software/vim/share/vim/vim92/macros/maze/mazeclean.c rename to software/vim/share/vim/vim92/macros/maze/mazeclean.c diff --git a/registry/software/vim/share/vim/vim92/macros/maze/poster b/software/vim/share/vim/vim92/macros/maze/poster similarity index 100% rename from registry/software/vim/share/vim/vim92/macros/maze/poster rename to software/vim/share/vim/vim92/macros/maze/poster diff --git a/registry/software/vim/share/vim/vim92/macros/shellmenu.vim b/software/vim/share/vim/vim92/macros/shellmenu.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/macros/shellmenu.vim rename to software/vim/share/vim/vim92/macros/shellmenu.vim diff --git a/registry/software/vim/share/vim/vim92/macros/swapmous.vim b/software/vim/share/vim/vim92/macros/swapmous.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/macros/swapmous.vim rename to software/vim/share/vim/vim92/macros/swapmous.vim diff --git a/registry/software/vim/share/vim/vim92/macros/urm/examples b/software/vim/share/vim/vim92/macros/urm/examples similarity index 100% rename from registry/software/vim/share/vim/vim92/macros/urm/examples rename to software/vim/share/vim/vim92/macros/urm/examples diff --git a/registry/software/vim/share/vim/vim92/macros/urm/urm b/software/vim/share/vim/vim92/macros/urm/urm similarity index 100% rename from registry/software/vim/share/vim/vim92/macros/urm/urm rename to software/vim/share/vim/vim92/macros/urm/urm diff --git a/registry/software/vim/share/vim/vim92/macros/urm/urm.vim b/software/vim/share/vim/vim92/macros/urm/urm.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/macros/urm/urm.vim rename to software/vim/share/vim/vim92/macros/urm/urm.vim diff --git a/registry/software/vim/share/vim/vim92/menu.vim b/software/vim/share/vim/vim92/menu.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/menu.vim rename to software/vim/share/vim/vim92/menu.vim diff --git a/registry/software/vim/share/vim/vim92/mswin.vim b/software/vim/share/vim/vim92/mswin.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/mswin.vim rename to software/vim/share/vim/vim92/mswin.vim diff --git a/registry/software/vim/share/vim/vim92/optwin.vim b/software/vim/share/vim/vim92/optwin.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/optwin.vim rename to software/vim/share/vim/vim92/optwin.vim diff --git a/registry/software/vim/share/vim/vim92/plugin/getscriptPlugin.vim b/software/vim/share/vim/vim92/plugin/getscriptPlugin.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/plugin/getscriptPlugin.vim rename to software/vim/share/vim/vim92/plugin/getscriptPlugin.vim diff --git a/registry/software/vim/share/vim/vim92/plugin/gzip.vim b/software/vim/share/vim/vim92/plugin/gzip.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/plugin/gzip.vim rename to software/vim/share/vim/vim92/plugin/gzip.vim diff --git a/registry/software/vim/share/vim/vim92/plugin/logiPat.vim b/software/vim/share/vim/vim92/plugin/logiPat.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/plugin/logiPat.vim rename to software/vim/share/vim/vim92/plugin/logiPat.vim diff --git a/registry/software/vim/share/vim/vim92/plugin/manpager.vim b/software/vim/share/vim/vim92/plugin/manpager.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/plugin/manpager.vim rename to software/vim/share/vim/vim92/plugin/manpager.vim diff --git a/registry/software/vim/share/vim/vim92/plugin/matchparen.vim b/software/vim/share/vim/vim92/plugin/matchparen.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/plugin/matchparen.vim rename to software/vim/share/vim/vim92/plugin/matchparen.vim diff --git a/registry/software/vim/share/vim/vim92/plugin/netrwPlugin.vim b/software/vim/share/vim/vim92/plugin/netrwPlugin.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/plugin/netrwPlugin.vim rename to software/vim/share/vim/vim92/plugin/netrwPlugin.vim diff --git a/registry/software/vim/share/vim/vim92/plugin/rrhelper.vim b/software/vim/share/vim/vim92/plugin/rrhelper.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/plugin/rrhelper.vim rename to software/vim/share/vim/vim92/plugin/rrhelper.vim diff --git a/registry/software/vim/share/vim/vim92/plugin/spellfile.vim b/software/vim/share/vim/vim92/plugin/spellfile.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/plugin/spellfile.vim rename to software/vim/share/vim/vim92/plugin/spellfile.vim diff --git a/registry/software/vim/share/vim/vim92/plugin/tarPlugin.vim b/software/vim/share/vim/vim92/plugin/tarPlugin.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/plugin/tarPlugin.vim rename to software/vim/share/vim/vim92/plugin/tarPlugin.vim diff --git a/registry/software/vim/share/vim/vim92/plugin/tohtml.vim b/software/vim/share/vim/vim92/plugin/tohtml.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/plugin/tohtml.vim rename to software/vim/share/vim/vim92/plugin/tohtml.vim diff --git a/registry/software/vim/share/vim/vim92/plugin/vimballPlugin.vim b/software/vim/share/vim/vim92/plugin/vimballPlugin.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/plugin/vimballPlugin.vim rename to software/vim/share/vim/vim92/plugin/vimballPlugin.vim diff --git a/registry/software/vim/share/vim/vim92/plugin/zipPlugin.vim b/software/vim/share/vim/vim92/plugin/zipPlugin.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/plugin/zipPlugin.vim rename to software/vim/share/vim/vim92/plugin/zipPlugin.vim diff --git a/registry/software/vim/share/vim/vim92/scripts.vim b/software/vim/share/vim/vim92/scripts.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/scripts.vim rename to software/vim/share/vim/vim92/scripts.vim diff --git a/registry/software/vim/share/vim/vim92/synmenu.vim b/software/vim/share/vim/vim92/synmenu.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/synmenu.vim rename to software/vim/share/vim/vim92/synmenu.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/2html.vim b/software/vim/share/vim/vim92/syntax/2html.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/2html.vim rename to software/vim/share/vim/vim92/syntax/2html.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/8th.vim b/software/vim/share/vim/vim92/syntax/8th.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/8th.vim rename to software/vim/share/vim/vim92/syntax/8th.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/a2ps.vim b/software/vim/share/vim/vim92/syntax/a2ps.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/a2ps.vim rename to software/vim/share/vim/vim92/syntax/a2ps.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/a65.vim b/software/vim/share/vim/vim92/syntax/a65.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/a65.vim rename to software/vim/share/vim/vim92/syntax/a65.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/aap.vim b/software/vim/share/vim/vim92/syntax/aap.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/aap.vim rename to software/vim/share/vim/vim92/syntax/aap.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/abap.vim b/software/vim/share/vim/vim92/syntax/abap.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/abap.vim rename to software/vim/share/vim/vim92/syntax/abap.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/abaqus.vim b/software/vim/share/vim/vim92/syntax/abaqus.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/abaqus.vim rename to software/vim/share/vim/vim92/syntax/abaqus.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/abc.vim b/software/vim/share/vim/vim92/syntax/abc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/abc.vim rename to software/vim/share/vim/vim92/syntax/abc.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/abel.vim b/software/vim/share/vim/vim92/syntax/abel.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/abel.vim rename to software/vim/share/vim/vim92/syntax/abel.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/acedb.vim b/software/vim/share/vim/vim92/syntax/acedb.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/acedb.vim rename to software/vim/share/vim/vim92/syntax/acedb.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/ada.vim b/software/vim/share/vim/vim92/syntax/ada.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/ada.vim rename to software/vim/share/vim/vim92/syntax/ada.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/aflex.vim b/software/vim/share/vim/vim92/syntax/aflex.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/aflex.vim rename to software/vim/share/vim/vim92/syntax/aflex.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/ahdl.vim b/software/vim/share/vim/vim92/syntax/ahdl.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/ahdl.vim rename to software/vim/share/vim/vim92/syntax/ahdl.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/aidl.vim b/software/vim/share/vim/vim92/syntax/aidl.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/aidl.vim rename to software/vim/share/vim/vim92/syntax/aidl.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/alsaconf.vim b/software/vim/share/vim/vim92/syntax/alsaconf.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/alsaconf.vim rename to software/vim/share/vim/vim92/syntax/alsaconf.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/amiga.vim b/software/vim/share/vim/vim92/syntax/amiga.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/amiga.vim rename to software/vim/share/vim/vim92/syntax/amiga.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/aml.vim b/software/vim/share/vim/vim92/syntax/aml.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/aml.vim rename to software/vim/share/vim/vim92/syntax/aml.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/ampl.vim b/software/vim/share/vim/vim92/syntax/ampl.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/ampl.vim rename to software/vim/share/vim/vim92/syntax/ampl.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/ant.vim b/software/vim/share/vim/vim92/syntax/ant.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/ant.vim rename to software/vim/share/vim/vim92/syntax/ant.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/antlr.vim b/software/vim/share/vim/vim92/syntax/antlr.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/antlr.vim rename to software/vim/share/vim/vim92/syntax/antlr.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/apache.vim b/software/vim/share/vim/vim92/syntax/apache.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/apache.vim rename to software/vim/share/vim/vim92/syntax/apache.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/apachestyle.vim b/software/vim/share/vim/vim92/syntax/apachestyle.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/apachestyle.vim rename to software/vim/share/vim/vim92/syntax/apachestyle.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/aptconf.vim b/software/vim/share/vim/vim92/syntax/aptconf.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/aptconf.vim rename to software/vim/share/vim/vim92/syntax/aptconf.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/arch.vim b/software/vim/share/vim/vim92/syntax/arch.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/arch.vim rename to software/vim/share/vim/vim92/syntax/arch.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/arduino.vim b/software/vim/share/vim/vim92/syntax/arduino.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/arduino.vim rename to software/vim/share/vim/vim92/syntax/arduino.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/art.vim b/software/vim/share/vim/vim92/syntax/art.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/art.vim rename to software/vim/share/vim/vim92/syntax/art.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/asciidoc.vim b/software/vim/share/vim/vim92/syntax/asciidoc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/asciidoc.vim rename to software/vim/share/vim/vim92/syntax/asciidoc.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/asm.vim b/software/vim/share/vim/vim92/syntax/asm.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/asm.vim rename to software/vim/share/vim/vim92/syntax/asm.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/asm68k.vim b/software/vim/share/vim/vim92/syntax/asm68k.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/asm68k.vim rename to software/vim/share/vim/vim92/syntax/asm68k.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/asmh8300.vim b/software/vim/share/vim/vim92/syntax/asmh8300.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/asmh8300.vim rename to software/vim/share/vim/vim92/syntax/asmh8300.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/asn.vim b/software/vim/share/vim/vim92/syntax/asn.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/asn.vim rename to software/vim/share/vim/vim92/syntax/asn.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/aspperl.vim b/software/vim/share/vim/vim92/syntax/aspperl.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/aspperl.vim rename to software/vim/share/vim/vim92/syntax/aspperl.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/aspvbs.vim b/software/vim/share/vim/vim92/syntax/aspvbs.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/aspvbs.vim rename to software/vim/share/vim/vim92/syntax/aspvbs.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/asterisk.vim b/software/vim/share/vim/vim92/syntax/asterisk.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/asterisk.vim rename to software/vim/share/vim/vim92/syntax/asterisk.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/asteriskvm.vim b/software/vim/share/vim/vim92/syntax/asteriskvm.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/asteriskvm.vim rename to software/vim/share/vim/vim92/syntax/asteriskvm.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/atlas.vim b/software/vim/share/vim/vim92/syntax/atlas.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/atlas.vim rename to software/vim/share/vim/vim92/syntax/atlas.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/autodoc.vim b/software/vim/share/vim/vim92/syntax/autodoc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/autodoc.vim rename to software/vim/share/vim/vim92/syntax/autodoc.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/autohotkey.vim b/software/vim/share/vim/vim92/syntax/autohotkey.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/autohotkey.vim rename to software/vim/share/vim/vim92/syntax/autohotkey.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/autoit.vim b/software/vim/share/vim/vim92/syntax/autoit.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/autoit.vim rename to software/vim/share/vim/vim92/syntax/autoit.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/automake.vim b/software/vim/share/vim/vim92/syntax/automake.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/automake.vim rename to software/vim/share/vim/vim92/syntax/automake.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/ave.vim b/software/vim/share/vim/vim92/syntax/ave.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/ave.vim rename to software/vim/share/vim/vim92/syntax/ave.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/avra.vim b/software/vim/share/vim/vim92/syntax/avra.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/avra.vim rename to software/vim/share/vim/vim92/syntax/avra.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/awk.vim b/software/vim/share/vim/vim92/syntax/awk.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/awk.vim rename to software/vim/share/vim/vim92/syntax/awk.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/ayacc.vim b/software/vim/share/vim/vim92/syntax/ayacc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/ayacc.vim rename to software/vim/share/vim/vim92/syntax/ayacc.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/b.vim b/software/vim/share/vim/vim92/syntax/b.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/b.vim rename to software/vim/share/vim/vim92/syntax/b.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/baan.vim b/software/vim/share/vim/vim92/syntax/baan.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/baan.vim rename to software/vim/share/vim/vim92/syntax/baan.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/bash.vim b/software/vim/share/vim/vim92/syntax/bash.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/bash.vim rename to software/vim/share/vim/vim92/syntax/bash.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/basic.vim b/software/vim/share/vim/vim92/syntax/basic.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/basic.vim rename to software/vim/share/vim/vim92/syntax/basic.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/bc.vim b/software/vim/share/vim/vim92/syntax/bc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/bc.vim rename to software/vim/share/vim/vim92/syntax/bc.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/bdf.vim b/software/vim/share/vim/vim92/syntax/bdf.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/bdf.vim rename to software/vim/share/vim/vim92/syntax/bdf.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/bib.vim b/software/vim/share/vim/vim92/syntax/bib.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/bib.vim rename to software/vim/share/vim/vim92/syntax/bib.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/bindzone.vim b/software/vim/share/vim/vim92/syntax/bindzone.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/bindzone.vim rename to software/vim/share/vim/vim92/syntax/bindzone.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/bitbake.vim b/software/vim/share/vim/vim92/syntax/bitbake.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/bitbake.vim rename to software/vim/share/vim/vim92/syntax/bitbake.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/blank.vim b/software/vim/share/vim/vim92/syntax/blank.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/blank.vim rename to software/vim/share/vim/vim92/syntax/blank.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/bsdl.vim b/software/vim/share/vim/vim92/syntax/bsdl.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/bsdl.vim rename to software/vim/share/vim/vim92/syntax/bsdl.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/bst.vim b/software/vim/share/vim/vim92/syntax/bst.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/bst.vim rename to software/vim/share/vim/vim92/syntax/bst.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/btm.vim b/software/vim/share/vim/vim92/syntax/btm.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/btm.vim rename to software/vim/share/vim/vim92/syntax/btm.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/bzl.vim b/software/vim/share/vim/vim92/syntax/bzl.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/bzl.vim rename to software/vim/share/vim/vim92/syntax/bzl.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/bzr.vim b/software/vim/share/vim/vim92/syntax/bzr.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/bzr.vim rename to software/vim/share/vim/vim92/syntax/bzr.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/c.vim b/software/vim/share/vim/vim92/syntax/c.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/c.vim rename to software/vim/share/vim/vim92/syntax/c.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/cabal.vim b/software/vim/share/vim/vim92/syntax/cabal.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/cabal.vim rename to software/vim/share/vim/vim92/syntax/cabal.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/cabalconfig.vim b/software/vim/share/vim/vim92/syntax/cabalconfig.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/cabalconfig.vim rename to software/vim/share/vim/vim92/syntax/cabalconfig.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/cabalproject.vim b/software/vim/share/vim/vim92/syntax/cabalproject.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/cabalproject.vim rename to software/vim/share/vim/vim92/syntax/cabalproject.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/calendar.vim b/software/vim/share/vim/vim92/syntax/calendar.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/calendar.vim rename to software/vim/share/vim/vim92/syntax/calendar.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/catalog.vim b/software/vim/share/vim/vim92/syntax/catalog.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/catalog.vim rename to software/vim/share/vim/vim92/syntax/catalog.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/cdl.vim b/software/vim/share/vim/vim92/syntax/cdl.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/cdl.vim rename to software/vim/share/vim/vim92/syntax/cdl.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/cdrdaoconf.vim b/software/vim/share/vim/vim92/syntax/cdrdaoconf.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/cdrdaoconf.vim rename to software/vim/share/vim/vim92/syntax/cdrdaoconf.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/cdrtoc.vim b/software/vim/share/vim/vim92/syntax/cdrtoc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/cdrtoc.vim rename to software/vim/share/vim/vim92/syntax/cdrtoc.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/cf.vim b/software/vim/share/vim/vim92/syntax/cf.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/cf.vim rename to software/vim/share/vim/vim92/syntax/cf.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/cfg.vim b/software/vim/share/vim/vim92/syntax/cfg.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/cfg.vim rename to software/vim/share/vim/vim92/syntax/cfg.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/ch.vim b/software/vim/share/vim/vim92/syntax/ch.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/ch.vim rename to software/vim/share/vim/vim92/syntax/ch.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/chaiscript.vim b/software/vim/share/vim/vim92/syntax/chaiscript.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/chaiscript.vim rename to software/vim/share/vim/vim92/syntax/chaiscript.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/change.vim b/software/vim/share/vim/vim92/syntax/change.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/change.vim rename to software/vim/share/vim/vim92/syntax/change.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/changelog.vim b/software/vim/share/vim/vim92/syntax/changelog.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/changelog.vim rename to software/vim/share/vim/vim92/syntax/changelog.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/chaskell.vim b/software/vim/share/vim/vim92/syntax/chaskell.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/chaskell.vim rename to software/vim/share/vim/vim92/syntax/chaskell.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/chatito.vim b/software/vim/share/vim/vim92/syntax/chatito.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/chatito.vim rename to software/vim/share/vim/vim92/syntax/chatito.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/cheetah.vim b/software/vim/share/vim/vim92/syntax/cheetah.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/cheetah.vim rename to software/vim/share/vim/vim92/syntax/cheetah.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/chicken.vim b/software/vim/share/vim/vim92/syntax/chicken.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/chicken.vim rename to software/vim/share/vim/vim92/syntax/chicken.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/chill.vim b/software/vim/share/vim/vim92/syntax/chill.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/chill.vim rename to software/vim/share/vim/vim92/syntax/chill.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/chordpro.vim b/software/vim/share/vim/vim92/syntax/chordpro.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/chordpro.vim rename to software/vim/share/vim/vim92/syntax/chordpro.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/cl.vim b/software/vim/share/vim/vim92/syntax/cl.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/cl.vim rename to software/vim/share/vim/vim92/syntax/cl.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/clean.vim b/software/vim/share/vim/vim92/syntax/clean.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/clean.vim rename to software/vim/share/vim/vim92/syntax/clean.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/clipper.vim b/software/vim/share/vim/vim92/syntax/clipper.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/clipper.vim rename to software/vim/share/vim/vim92/syntax/clipper.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/clojure.vim b/software/vim/share/vim/vim92/syntax/clojure.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/clojure.vim rename to software/vim/share/vim/vim92/syntax/clojure.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/cmake.vim b/software/vim/share/vim/vim92/syntax/cmake.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/cmake.vim rename to software/vim/share/vim/vim92/syntax/cmake.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/cmod.vim b/software/vim/share/vim/vim92/syntax/cmod.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/cmod.vim rename to software/vim/share/vim/vim92/syntax/cmod.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/cmusrc.vim b/software/vim/share/vim/vim92/syntax/cmusrc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/cmusrc.vim rename to software/vim/share/vim/vim92/syntax/cmusrc.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/cobol.vim b/software/vim/share/vim/vim92/syntax/cobol.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/cobol.vim rename to software/vim/share/vim/vim92/syntax/cobol.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/coco.vim b/software/vim/share/vim/vim92/syntax/coco.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/coco.vim rename to software/vim/share/vim/vim92/syntax/coco.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/colortest.vim b/software/vim/share/vim/vim92/syntax/colortest.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/colortest.vim rename to software/vim/share/vim/vim92/syntax/colortest.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/conaryrecipe.vim b/software/vim/share/vim/vim92/syntax/conaryrecipe.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/conaryrecipe.vim rename to software/vim/share/vim/vim92/syntax/conaryrecipe.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/conf.vim b/software/vim/share/vim/vim92/syntax/conf.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/conf.vim rename to software/vim/share/vim/vim92/syntax/conf.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/config.vim b/software/vim/share/vim/vim92/syntax/config.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/config.vim rename to software/vim/share/vim/vim92/syntax/config.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/confini.vim b/software/vim/share/vim/vim92/syntax/confini.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/confini.vim rename to software/vim/share/vim/vim92/syntax/confini.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/context.vim b/software/vim/share/vim/vim92/syntax/context.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/context.vim rename to software/vim/share/vim/vim92/syntax/context.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/cpp.vim b/software/vim/share/vim/vim92/syntax/cpp.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/cpp.vim rename to software/vim/share/vim/vim92/syntax/cpp.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/crm.vim b/software/vim/share/vim/vim92/syntax/crm.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/crm.vim rename to software/vim/share/vim/vim92/syntax/crm.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/crontab.vim b/software/vim/share/vim/vim92/syntax/crontab.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/crontab.vim rename to software/vim/share/vim/vim92/syntax/crontab.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/cs.vim b/software/vim/share/vim/vim92/syntax/cs.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/cs.vim rename to software/vim/share/vim/vim92/syntax/cs.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/csc.vim b/software/vim/share/vim/vim92/syntax/csc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/csc.vim rename to software/vim/share/vim/vim92/syntax/csc.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/csdl.vim b/software/vim/share/vim/vim92/syntax/csdl.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/csdl.vim rename to software/vim/share/vim/vim92/syntax/csdl.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/csh.vim b/software/vim/share/vim/vim92/syntax/csh.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/csh.vim rename to software/vim/share/vim/vim92/syntax/csh.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/csp.vim b/software/vim/share/vim/vim92/syntax/csp.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/csp.vim rename to software/vim/share/vim/vim92/syntax/csp.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/css.vim b/software/vim/share/vim/vim92/syntax/css.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/css.vim rename to software/vim/share/vim/vim92/syntax/css.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/cterm.vim b/software/vim/share/vim/vim92/syntax/cterm.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/cterm.vim rename to software/vim/share/vim/vim92/syntax/cterm.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/ctrlh.vim b/software/vim/share/vim/vim92/syntax/ctrlh.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/ctrlh.vim rename to software/vim/share/vim/vim92/syntax/ctrlh.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/cucumber.vim b/software/vim/share/vim/vim92/syntax/cucumber.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/cucumber.vim rename to software/vim/share/vim/vim92/syntax/cucumber.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/cuda.vim b/software/vim/share/vim/vim92/syntax/cuda.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/cuda.vim rename to software/vim/share/vim/vim92/syntax/cuda.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/cupl.vim b/software/vim/share/vim/vim92/syntax/cupl.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/cupl.vim rename to software/vim/share/vim/vim92/syntax/cupl.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/cuplsim.vim b/software/vim/share/vim/vim92/syntax/cuplsim.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/cuplsim.vim rename to software/vim/share/vim/vim92/syntax/cuplsim.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/cvs.vim b/software/vim/share/vim/vim92/syntax/cvs.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/cvs.vim rename to software/vim/share/vim/vim92/syntax/cvs.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/cvsrc.vim b/software/vim/share/vim/vim92/syntax/cvsrc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/cvsrc.vim rename to software/vim/share/vim/vim92/syntax/cvsrc.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/cweb.vim b/software/vim/share/vim/vim92/syntax/cweb.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/cweb.vim rename to software/vim/share/vim/vim92/syntax/cweb.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/cynlib.vim b/software/vim/share/vim/vim92/syntax/cynlib.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/cynlib.vim rename to software/vim/share/vim/vim92/syntax/cynlib.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/cynpp.vim b/software/vim/share/vim/vim92/syntax/cynpp.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/cynpp.vim rename to software/vim/share/vim/vim92/syntax/cynpp.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/d.vim b/software/vim/share/vim/vim92/syntax/d.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/d.vim rename to software/vim/share/vim/vim92/syntax/d.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/dart.vim b/software/vim/share/vim/vim92/syntax/dart.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/dart.vim rename to software/vim/share/vim/vim92/syntax/dart.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/datascript.vim b/software/vim/share/vim/vim92/syntax/datascript.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/datascript.vim rename to software/vim/share/vim/vim92/syntax/datascript.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/dcd.vim b/software/vim/share/vim/vim92/syntax/dcd.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/dcd.vim rename to software/vim/share/vim/vim92/syntax/dcd.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/dcl.vim b/software/vim/share/vim/vim92/syntax/dcl.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/dcl.vim rename to software/vim/share/vim/vim92/syntax/dcl.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/debchangelog.vim b/software/vim/share/vim/vim92/syntax/debchangelog.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/debchangelog.vim rename to software/vim/share/vim/vim92/syntax/debchangelog.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/debcontrol.vim b/software/vim/share/vim/vim92/syntax/debcontrol.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/debcontrol.vim rename to software/vim/share/vim/vim92/syntax/debcontrol.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/debcopyright.vim b/software/vim/share/vim/vim92/syntax/debcopyright.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/debcopyright.vim rename to software/vim/share/vim/vim92/syntax/debcopyright.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/debsources.vim b/software/vim/share/vim/vim92/syntax/debsources.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/debsources.vim rename to software/vim/share/vim/vim92/syntax/debsources.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/def.vim b/software/vim/share/vim/vim92/syntax/def.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/def.vim rename to software/vim/share/vim/vim92/syntax/def.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/denyhosts.vim b/software/vim/share/vim/vim92/syntax/denyhosts.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/denyhosts.vim rename to software/vim/share/vim/vim92/syntax/denyhosts.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/dep3patch.vim b/software/vim/share/vim/vim92/syntax/dep3patch.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/dep3patch.vim rename to software/vim/share/vim/vim92/syntax/dep3patch.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/desc.vim b/software/vim/share/vim/vim92/syntax/desc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/desc.vim rename to software/vim/share/vim/vim92/syntax/desc.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/desktop.vim b/software/vim/share/vim/vim92/syntax/desktop.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/desktop.vim rename to software/vim/share/vim/vim92/syntax/desktop.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/dictconf.vim b/software/vim/share/vim/vim92/syntax/dictconf.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/dictconf.vim rename to software/vim/share/vim/vim92/syntax/dictconf.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/dictdconf.vim b/software/vim/share/vim/vim92/syntax/dictdconf.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/dictdconf.vim rename to software/vim/share/vim/vim92/syntax/dictdconf.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/diff.vim b/software/vim/share/vim/vim92/syntax/diff.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/diff.vim rename to software/vim/share/vim/vim92/syntax/diff.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/dircolors.vim b/software/vim/share/vim/vim92/syntax/dircolors.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/dircolors.vim rename to software/vim/share/vim/vim92/syntax/dircolors.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/dirpager.vim b/software/vim/share/vim/vim92/syntax/dirpager.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/dirpager.vim rename to software/vim/share/vim/vim92/syntax/dirpager.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/diva.vim b/software/vim/share/vim/vim92/syntax/diva.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/diva.vim rename to software/vim/share/vim/vim92/syntax/diva.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/django.vim b/software/vim/share/vim/vim92/syntax/django.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/django.vim rename to software/vim/share/vim/vim92/syntax/django.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/dns.vim b/software/vim/share/vim/vim92/syntax/dns.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/dns.vim rename to software/vim/share/vim/vim92/syntax/dns.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/dnsmasq.vim b/software/vim/share/vim/vim92/syntax/dnsmasq.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/dnsmasq.vim rename to software/vim/share/vim/vim92/syntax/dnsmasq.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/docbk.vim b/software/vim/share/vim/vim92/syntax/docbk.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/docbk.vim rename to software/vim/share/vim/vim92/syntax/docbk.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/docbksgml.vim b/software/vim/share/vim/vim92/syntax/docbksgml.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/docbksgml.vim rename to software/vim/share/vim/vim92/syntax/docbksgml.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/docbkxml.vim b/software/vim/share/vim/vim92/syntax/docbkxml.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/docbkxml.vim rename to software/vim/share/vim/vim92/syntax/docbkxml.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/dockerfile.vim b/software/vim/share/vim/vim92/syntax/dockerfile.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/dockerfile.vim rename to software/vim/share/vim/vim92/syntax/dockerfile.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/dosbatch.vim b/software/vim/share/vim/vim92/syntax/dosbatch.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/dosbatch.vim rename to software/vim/share/vim/vim92/syntax/dosbatch.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/dosini.vim b/software/vim/share/vim/vim92/syntax/dosini.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/dosini.vim rename to software/vim/share/vim/vim92/syntax/dosini.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/dot.vim b/software/vim/share/vim/vim92/syntax/dot.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/dot.vim rename to software/vim/share/vim/vim92/syntax/dot.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/doxygen.vim b/software/vim/share/vim/vim92/syntax/doxygen.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/doxygen.vim rename to software/vim/share/vim/vim92/syntax/doxygen.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/dracula.vim b/software/vim/share/vim/vim92/syntax/dracula.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/dracula.vim rename to software/vim/share/vim/vim92/syntax/dracula.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/dsl.vim b/software/vim/share/vim/vim92/syntax/dsl.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/dsl.vim rename to software/vim/share/vim/vim92/syntax/dsl.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/dtd.vim b/software/vim/share/vim/vim92/syntax/dtd.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/dtd.vim rename to software/vim/share/vim/vim92/syntax/dtd.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/dtml.vim b/software/vim/share/vim/vim92/syntax/dtml.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/dtml.vim rename to software/vim/share/vim/vim92/syntax/dtml.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/dtrace.vim b/software/vim/share/vim/vim92/syntax/dtrace.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/dtrace.vim rename to software/vim/share/vim/vim92/syntax/dtrace.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/dts.vim b/software/vim/share/vim/vim92/syntax/dts.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/dts.vim rename to software/vim/share/vim/vim92/syntax/dts.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/dune.vim b/software/vim/share/vim/vim92/syntax/dune.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/dune.vim rename to software/vim/share/vim/vim92/syntax/dune.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/dylan.vim b/software/vim/share/vim/vim92/syntax/dylan.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/dylan.vim rename to software/vim/share/vim/vim92/syntax/dylan.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/dylanintr.vim b/software/vim/share/vim/vim92/syntax/dylanintr.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/dylanintr.vim rename to software/vim/share/vim/vim92/syntax/dylanintr.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/dylanlid.vim b/software/vim/share/vim/vim92/syntax/dylanlid.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/dylanlid.vim rename to software/vim/share/vim/vim92/syntax/dylanlid.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/ecd.vim b/software/vim/share/vim/vim92/syntax/ecd.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/ecd.vim rename to software/vim/share/vim/vim92/syntax/ecd.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/edif.vim b/software/vim/share/vim/vim92/syntax/edif.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/edif.vim rename to software/vim/share/vim/vim92/syntax/edif.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/editorconfig.vim b/software/vim/share/vim/vim92/syntax/editorconfig.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/editorconfig.vim rename to software/vim/share/vim/vim92/syntax/editorconfig.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/eiffel.vim b/software/vim/share/vim/vim92/syntax/eiffel.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/eiffel.vim rename to software/vim/share/vim/vim92/syntax/eiffel.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/elf.vim b/software/vim/share/vim/vim92/syntax/elf.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/elf.vim rename to software/vim/share/vim/vim92/syntax/elf.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/elinks.vim b/software/vim/share/vim/vim92/syntax/elinks.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/elinks.vim rename to software/vim/share/vim/vim92/syntax/elinks.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/elm.vim b/software/vim/share/vim/vim92/syntax/elm.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/elm.vim rename to software/vim/share/vim/vim92/syntax/elm.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/elmfilt.vim b/software/vim/share/vim/vim92/syntax/elmfilt.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/elmfilt.vim rename to software/vim/share/vim/vim92/syntax/elmfilt.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/erlang.vim b/software/vim/share/vim/vim92/syntax/erlang.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/erlang.vim rename to software/vim/share/vim/vim92/syntax/erlang.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/eruby.vim b/software/vim/share/vim/vim92/syntax/eruby.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/eruby.vim rename to software/vim/share/vim/vim92/syntax/eruby.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/esmtprc.vim b/software/vim/share/vim/vim92/syntax/esmtprc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/esmtprc.vim rename to software/vim/share/vim/vim92/syntax/esmtprc.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/esqlc.vim b/software/vim/share/vim/vim92/syntax/esqlc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/esqlc.vim rename to software/vim/share/vim/vim92/syntax/esqlc.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/esterel.vim b/software/vim/share/vim/vim92/syntax/esterel.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/esterel.vim rename to software/vim/share/vim/vim92/syntax/esterel.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/eterm.vim b/software/vim/share/vim/vim92/syntax/eterm.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/eterm.vim rename to software/vim/share/vim/vim92/syntax/eterm.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/euphoria3.vim b/software/vim/share/vim/vim92/syntax/euphoria3.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/euphoria3.vim rename to software/vim/share/vim/vim92/syntax/euphoria3.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/euphoria4.vim b/software/vim/share/vim/vim92/syntax/euphoria4.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/euphoria4.vim rename to software/vim/share/vim/vim92/syntax/euphoria4.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/eviews.vim b/software/vim/share/vim/vim92/syntax/eviews.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/eviews.vim rename to software/vim/share/vim/vim92/syntax/eviews.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/exim.vim b/software/vim/share/vim/vim92/syntax/exim.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/exim.vim rename to software/vim/share/vim/vim92/syntax/exim.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/expect.vim b/software/vim/share/vim/vim92/syntax/expect.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/expect.vim rename to software/vim/share/vim/vim92/syntax/expect.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/exports.vim b/software/vim/share/vim/vim92/syntax/exports.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/exports.vim rename to software/vim/share/vim/vim92/syntax/exports.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/falcon.vim b/software/vim/share/vim/vim92/syntax/falcon.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/falcon.vim rename to software/vim/share/vim/vim92/syntax/falcon.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/fan.vim b/software/vim/share/vim/vim92/syntax/fan.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/fan.vim rename to software/vim/share/vim/vim92/syntax/fan.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/fasm.vim b/software/vim/share/vim/vim92/syntax/fasm.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/fasm.vim rename to software/vim/share/vim/vim92/syntax/fasm.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/fdcc.vim b/software/vim/share/vim/vim92/syntax/fdcc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/fdcc.vim rename to software/vim/share/vim/vim92/syntax/fdcc.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/fetchmail.vim b/software/vim/share/vim/vim92/syntax/fetchmail.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/fetchmail.vim rename to software/vim/share/vim/vim92/syntax/fetchmail.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/fgl.vim b/software/vim/share/vim/vim92/syntax/fgl.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/fgl.vim rename to software/vim/share/vim/vim92/syntax/fgl.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/fish.vim b/software/vim/share/vim/vim92/syntax/fish.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/fish.vim rename to software/vim/share/vim/vim92/syntax/fish.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/flexwiki.vim b/software/vim/share/vim/vim92/syntax/flexwiki.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/flexwiki.vim rename to software/vim/share/vim/vim92/syntax/flexwiki.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/focexec.vim b/software/vim/share/vim/vim92/syntax/focexec.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/focexec.vim rename to software/vim/share/vim/vim92/syntax/focexec.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/form.vim b/software/vim/share/vim/vim92/syntax/form.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/form.vim rename to software/vim/share/vim/vim92/syntax/form.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/forth.vim b/software/vim/share/vim/vim92/syntax/forth.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/forth.vim rename to software/vim/share/vim/vim92/syntax/forth.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/fortran.vim b/software/vim/share/vim/vim92/syntax/fortran.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/fortran.vim rename to software/vim/share/vim/vim92/syntax/fortran.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/foxpro.vim b/software/vim/share/vim/vim92/syntax/foxpro.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/foxpro.vim rename to software/vim/share/vim/vim92/syntax/foxpro.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/fpcmake.vim b/software/vim/share/vim/vim92/syntax/fpcmake.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/fpcmake.vim rename to software/vim/share/vim/vim92/syntax/fpcmake.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/framescript.vim b/software/vim/share/vim/vim92/syntax/framescript.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/framescript.vim rename to software/vim/share/vim/vim92/syntax/framescript.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/freebasic.vim b/software/vim/share/vim/vim92/syntax/freebasic.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/freebasic.vim rename to software/vim/share/vim/vim92/syntax/freebasic.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/fstab.vim b/software/vim/share/vim/vim92/syntax/fstab.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/fstab.vim rename to software/vim/share/vim/vim92/syntax/fstab.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/fvwm.vim b/software/vim/share/vim/vim92/syntax/fvwm.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/fvwm.vim rename to software/vim/share/vim/vim92/syntax/fvwm.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/fvwm2m4.vim b/software/vim/share/vim/vim92/syntax/fvwm2m4.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/fvwm2m4.vim rename to software/vim/share/vim/vim92/syntax/fvwm2m4.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/gdb.vim b/software/vim/share/vim/vim92/syntax/gdb.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/gdb.vim rename to software/vim/share/vim/vim92/syntax/gdb.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/gdmo.vim b/software/vim/share/vim/vim92/syntax/gdmo.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/gdmo.vim rename to software/vim/share/vim/vim92/syntax/gdmo.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/gdresource.vim b/software/vim/share/vim/vim92/syntax/gdresource.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/gdresource.vim rename to software/vim/share/vim/vim92/syntax/gdresource.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/gdscript.vim b/software/vim/share/vim/vim92/syntax/gdscript.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/gdscript.vim rename to software/vim/share/vim/vim92/syntax/gdscript.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/gdshader.vim b/software/vim/share/vim/vim92/syntax/gdshader.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/gdshader.vim rename to software/vim/share/vim/vim92/syntax/gdshader.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/gedcom.vim b/software/vim/share/vim/vim92/syntax/gedcom.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/gedcom.vim rename to software/vim/share/vim/vim92/syntax/gedcom.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/gemtext.vim b/software/vim/share/vim/vim92/syntax/gemtext.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/gemtext.vim rename to software/vim/share/vim/vim92/syntax/gemtext.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/gift.vim b/software/vim/share/vim/vim92/syntax/gift.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/gift.vim rename to software/vim/share/vim/vim92/syntax/gift.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/git.vim b/software/vim/share/vim/vim92/syntax/git.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/git.vim rename to software/vim/share/vim/vim92/syntax/git.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/gitattributes.vim b/software/vim/share/vim/vim92/syntax/gitattributes.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/gitattributes.vim rename to software/vim/share/vim/vim92/syntax/gitattributes.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/gitcommit.vim b/software/vim/share/vim/vim92/syntax/gitcommit.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/gitcommit.vim rename to software/vim/share/vim/vim92/syntax/gitcommit.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/gitconfig.vim b/software/vim/share/vim/vim92/syntax/gitconfig.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/gitconfig.vim rename to software/vim/share/vim/vim92/syntax/gitconfig.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/gitignore.vim b/software/vim/share/vim/vim92/syntax/gitignore.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/gitignore.vim rename to software/vim/share/vim/vim92/syntax/gitignore.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/gitolite.vim b/software/vim/share/vim/vim92/syntax/gitolite.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/gitolite.vim rename to software/vim/share/vim/vim92/syntax/gitolite.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/gitrebase.vim b/software/vim/share/vim/vim92/syntax/gitrebase.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/gitrebase.vim rename to software/vim/share/vim/vim92/syntax/gitrebase.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/gitsendemail.vim b/software/vim/share/vim/vim92/syntax/gitsendemail.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/gitsendemail.vim rename to software/vim/share/vim/vim92/syntax/gitsendemail.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/gkrellmrc.vim b/software/vim/share/vim/vim92/syntax/gkrellmrc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/gkrellmrc.vim rename to software/vim/share/vim/vim92/syntax/gkrellmrc.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/gnash.vim b/software/vim/share/vim/vim92/syntax/gnash.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/gnash.vim rename to software/vim/share/vim/vim92/syntax/gnash.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/gnuplot.vim b/software/vim/share/vim/vim92/syntax/gnuplot.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/gnuplot.vim rename to software/vim/share/vim/vim92/syntax/gnuplot.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/go.vim b/software/vim/share/vim/vim92/syntax/go.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/go.vim rename to software/vim/share/vim/vim92/syntax/go.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/godoc.vim b/software/vim/share/vim/vim92/syntax/godoc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/godoc.vim rename to software/vim/share/vim/vim92/syntax/godoc.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/gp.vim b/software/vim/share/vim/vim92/syntax/gp.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/gp.vim rename to software/vim/share/vim/vim92/syntax/gp.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/gpg.vim b/software/vim/share/vim/vim92/syntax/gpg.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/gpg.vim rename to software/vim/share/vim/vim92/syntax/gpg.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/gprof.vim b/software/vim/share/vim/vim92/syntax/gprof.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/gprof.vim rename to software/vim/share/vim/vim92/syntax/gprof.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/grads.vim b/software/vim/share/vim/vim92/syntax/grads.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/grads.vim rename to software/vim/share/vim/vim92/syntax/grads.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/gretl.vim b/software/vim/share/vim/vim92/syntax/gretl.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/gretl.vim rename to software/vim/share/vim/vim92/syntax/gretl.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/groff.vim b/software/vim/share/vim/vim92/syntax/groff.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/groff.vim rename to software/vim/share/vim/vim92/syntax/groff.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/groovy.vim b/software/vim/share/vim/vim92/syntax/groovy.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/groovy.vim rename to software/vim/share/vim/vim92/syntax/groovy.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/group.vim b/software/vim/share/vim/vim92/syntax/group.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/group.vim rename to software/vim/share/vim/vim92/syntax/group.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/grub.vim b/software/vim/share/vim/vim92/syntax/grub.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/grub.vim rename to software/vim/share/vim/vim92/syntax/grub.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/gsp.vim b/software/vim/share/vim/vim92/syntax/gsp.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/gsp.vim rename to software/vim/share/vim/vim92/syntax/gsp.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/gtkrc.vim b/software/vim/share/vim/vim92/syntax/gtkrc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/gtkrc.vim rename to software/vim/share/vim/vim92/syntax/gtkrc.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/gvpr.vim b/software/vim/share/vim/vim92/syntax/gvpr.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/gvpr.vim rename to software/vim/share/vim/vim92/syntax/gvpr.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/gyp.vim b/software/vim/share/vim/vim92/syntax/gyp.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/gyp.vim rename to software/vim/share/vim/vim92/syntax/gyp.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/haml.vim b/software/vim/share/vim/vim92/syntax/haml.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/haml.vim rename to software/vim/share/vim/vim92/syntax/haml.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/hamster.vim b/software/vim/share/vim/vim92/syntax/hamster.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/hamster.vim rename to software/vim/share/vim/vim92/syntax/hamster.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/hare.vim b/software/vim/share/vim/vim92/syntax/hare.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/hare.vim rename to software/vim/share/vim/vim92/syntax/hare.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/haskell.vim b/software/vim/share/vim/vim92/syntax/haskell.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/haskell.vim rename to software/vim/share/vim/vim92/syntax/haskell.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/haste.vim b/software/vim/share/vim/vim92/syntax/haste.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/haste.vim rename to software/vim/share/vim/vim92/syntax/haste.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/hastepreproc.vim b/software/vim/share/vim/vim92/syntax/hastepreproc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/hastepreproc.vim rename to software/vim/share/vim/vim92/syntax/hastepreproc.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/hb.vim b/software/vim/share/vim/vim92/syntax/hb.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/hb.vim rename to software/vim/share/vim/vim92/syntax/hb.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/help.vim b/software/vim/share/vim/vim92/syntax/help.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/help.vim rename to software/vim/share/vim/vim92/syntax/help.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/help_ru.vim b/software/vim/share/vim/vim92/syntax/help_ru.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/help_ru.vim rename to software/vim/share/vim/vim92/syntax/help_ru.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/hercules.vim b/software/vim/share/vim/vim92/syntax/hercules.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/hercules.vim rename to software/vim/share/vim/vim92/syntax/hercules.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/hex.vim b/software/vim/share/vim/vim92/syntax/hex.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/hex.vim rename to software/vim/share/vim/vim92/syntax/hex.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/hgcommit.vim b/software/vim/share/vim/vim92/syntax/hgcommit.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/hgcommit.vim rename to software/vim/share/vim/vim92/syntax/hgcommit.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/hitest.vim b/software/vim/share/vim/vim92/syntax/hitest.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/hitest.vim rename to software/vim/share/vim/vim92/syntax/hitest.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/hlsplaylist.vim b/software/vim/share/vim/vim92/syntax/hlsplaylist.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/hlsplaylist.vim rename to software/vim/share/vim/vim92/syntax/hlsplaylist.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/hog.vim b/software/vim/share/vim/vim92/syntax/hog.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/hog.vim rename to software/vim/share/vim/vim92/syntax/hog.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/hollywood.vim b/software/vim/share/vim/vim92/syntax/hollywood.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/hollywood.vim rename to software/vim/share/vim/vim92/syntax/hollywood.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/hostconf.vim b/software/vim/share/vim/vim92/syntax/hostconf.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/hostconf.vim rename to software/vim/share/vim/vim92/syntax/hostconf.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/hostsaccess.vim b/software/vim/share/vim/vim92/syntax/hostsaccess.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/hostsaccess.vim rename to software/vim/share/vim/vim92/syntax/hostsaccess.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/html.vim b/software/vim/share/vim/vim92/syntax/html.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/html.vim rename to software/vim/share/vim/vim92/syntax/html.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/htmlcheetah.vim b/software/vim/share/vim/vim92/syntax/htmlcheetah.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/htmlcheetah.vim rename to software/vim/share/vim/vim92/syntax/htmlcheetah.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/htmldjango.vim b/software/vim/share/vim/vim92/syntax/htmldjango.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/htmldjango.vim rename to software/vim/share/vim/vim92/syntax/htmldjango.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/htmlm4.vim b/software/vim/share/vim/vim92/syntax/htmlm4.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/htmlm4.vim rename to software/vim/share/vim/vim92/syntax/htmlm4.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/htmlos.vim b/software/vim/share/vim/vim92/syntax/htmlos.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/htmlos.vim rename to software/vim/share/vim/vim92/syntax/htmlos.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/i3config.vim b/software/vim/share/vim/vim92/syntax/i3config.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/i3config.vim rename to software/vim/share/vim/vim92/syntax/i3config.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/ia64.vim b/software/vim/share/vim/vim92/syntax/ia64.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/ia64.vim rename to software/vim/share/vim/vim92/syntax/ia64.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/ibasic.vim b/software/vim/share/vim/vim92/syntax/ibasic.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/ibasic.vim rename to software/vim/share/vim/vim92/syntax/ibasic.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/icemenu.vim b/software/vim/share/vim/vim92/syntax/icemenu.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/icemenu.vim rename to software/vim/share/vim/vim92/syntax/icemenu.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/icon.vim b/software/vim/share/vim/vim92/syntax/icon.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/icon.vim rename to software/vim/share/vim/vim92/syntax/icon.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/idl.vim b/software/vim/share/vim/vim92/syntax/idl.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/idl.vim rename to software/vim/share/vim/vim92/syntax/idl.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/idlang.vim b/software/vim/share/vim/vim92/syntax/idlang.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/idlang.vim rename to software/vim/share/vim/vim92/syntax/idlang.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/indent.vim b/software/vim/share/vim/vim92/syntax/indent.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/indent.vim rename to software/vim/share/vim/vim92/syntax/indent.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/inform.vim b/software/vim/share/vim/vim92/syntax/inform.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/inform.vim rename to software/vim/share/vim/vim92/syntax/inform.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/initex.vim b/software/vim/share/vim/vim92/syntax/initex.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/initex.vim rename to software/vim/share/vim/vim92/syntax/initex.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/initng.vim b/software/vim/share/vim/vim92/syntax/initng.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/initng.vim rename to software/vim/share/vim/vim92/syntax/initng.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/inittab.vim b/software/vim/share/vim/vim92/syntax/inittab.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/inittab.vim rename to software/vim/share/vim/vim92/syntax/inittab.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/ipfilter.vim b/software/vim/share/vim/vim92/syntax/ipfilter.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/ipfilter.vim rename to software/vim/share/vim/vim92/syntax/ipfilter.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/ishd.vim b/software/vim/share/vim/vim92/syntax/ishd.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/ishd.vim rename to software/vim/share/vim/vim92/syntax/ishd.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/iss.vim b/software/vim/share/vim/vim92/syntax/iss.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/iss.vim rename to software/vim/share/vim/vim92/syntax/iss.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/ist.vim b/software/vim/share/vim/vim92/syntax/ist.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/ist.vim rename to software/vim/share/vim/vim92/syntax/ist.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/j.vim b/software/vim/share/vim/vim92/syntax/j.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/j.vim rename to software/vim/share/vim/vim92/syntax/j.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/jal.vim b/software/vim/share/vim/vim92/syntax/jal.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/jal.vim rename to software/vim/share/vim/vim92/syntax/jal.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/jam.vim b/software/vim/share/vim/vim92/syntax/jam.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/jam.vim rename to software/vim/share/vim/vim92/syntax/jam.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/jargon.vim b/software/vim/share/vim/vim92/syntax/jargon.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/jargon.vim rename to software/vim/share/vim/vim92/syntax/jargon.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/java.vim b/software/vim/share/vim/vim92/syntax/java.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/java.vim rename to software/vim/share/vim/vim92/syntax/java.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/javacc.vim b/software/vim/share/vim/vim92/syntax/javacc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/javacc.vim rename to software/vim/share/vim/vim92/syntax/javacc.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/javascript.vim b/software/vim/share/vim/vim92/syntax/javascript.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/javascript.vim rename to software/vim/share/vim/vim92/syntax/javascript.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/javascriptreact.vim b/software/vim/share/vim/vim92/syntax/javascriptreact.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/javascriptreact.vim rename to software/vim/share/vim/vim92/syntax/javascriptreact.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/jess.vim b/software/vim/share/vim/vim92/syntax/jess.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/jess.vim rename to software/vim/share/vim/vim92/syntax/jess.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/jgraph.vim b/software/vim/share/vim/vim92/syntax/jgraph.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/jgraph.vim rename to software/vim/share/vim/vim92/syntax/jgraph.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/jovial.vim b/software/vim/share/vim/vim92/syntax/jovial.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/jovial.vim rename to software/vim/share/vim/vim92/syntax/jovial.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/jproperties.vim b/software/vim/share/vim/vim92/syntax/jproperties.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/jproperties.vim rename to software/vim/share/vim/vim92/syntax/jproperties.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/json.vim b/software/vim/share/vim/vim92/syntax/json.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/json.vim rename to software/vim/share/vim/vim92/syntax/json.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/jsonc.vim b/software/vim/share/vim/vim92/syntax/jsonc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/jsonc.vim rename to software/vim/share/vim/vim92/syntax/jsonc.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/jsp.vim b/software/vim/share/vim/vim92/syntax/jsp.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/jsp.vim rename to software/vim/share/vim/vim92/syntax/jsp.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/julia.vim b/software/vim/share/vim/vim92/syntax/julia.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/julia.vim rename to software/vim/share/vim/vim92/syntax/julia.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/kconfig.vim b/software/vim/share/vim/vim92/syntax/kconfig.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/kconfig.vim rename to software/vim/share/vim/vim92/syntax/kconfig.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/kivy.vim b/software/vim/share/vim/vim92/syntax/kivy.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/kivy.vim rename to software/vim/share/vim/vim92/syntax/kivy.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/kix.vim b/software/vim/share/vim/vim92/syntax/kix.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/kix.vim rename to software/vim/share/vim/vim92/syntax/kix.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/krl.vim b/software/vim/share/vim/vim92/syntax/krl.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/krl.vim rename to software/vim/share/vim/vim92/syntax/krl.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/kscript.vim b/software/vim/share/vim/vim92/syntax/kscript.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/kscript.vim rename to software/vim/share/vim/vim92/syntax/kscript.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/kwt.vim b/software/vim/share/vim/vim92/syntax/kwt.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/kwt.vim rename to software/vim/share/vim/vim92/syntax/kwt.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/lace.vim b/software/vim/share/vim/vim92/syntax/lace.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/lace.vim rename to software/vim/share/vim/vim92/syntax/lace.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/latte.vim b/software/vim/share/vim/vim92/syntax/latte.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/latte.vim rename to software/vim/share/vim/vim92/syntax/latte.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/lc.vim b/software/vim/share/vim/vim92/syntax/lc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/lc.vim rename to software/vim/share/vim/vim92/syntax/lc.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/ld.vim b/software/vim/share/vim/vim92/syntax/ld.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/ld.vim rename to software/vim/share/vim/vim92/syntax/ld.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/ldapconf.vim b/software/vim/share/vim/vim92/syntax/ldapconf.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/ldapconf.vim rename to software/vim/share/vim/vim92/syntax/ldapconf.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/ldif.vim b/software/vim/share/vim/vim92/syntax/ldif.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/ldif.vim rename to software/vim/share/vim/vim92/syntax/ldif.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/less.vim b/software/vim/share/vim/vim92/syntax/less.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/less.vim rename to software/vim/share/vim/vim92/syntax/less.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/lex.vim b/software/vim/share/vim/vim92/syntax/lex.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/lex.vim rename to software/vim/share/vim/vim92/syntax/lex.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/lftp.vim b/software/vim/share/vim/vim92/syntax/lftp.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/lftp.vim rename to software/vim/share/vim/vim92/syntax/lftp.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/lhaskell.vim b/software/vim/share/vim/vim92/syntax/lhaskell.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/lhaskell.vim rename to software/vim/share/vim/vim92/syntax/lhaskell.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/libao.vim b/software/vim/share/vim/vim92/syntax/libao.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/libao.vim rename to software/vim/share/vim/vim92/syntax/libao.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/lifelines.vim b/software/vim/share/vim/vim92/syntax/lifelines.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/lifelines.vim rename to software/vim/share/vim/vim92/syntax/lifelines.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/lilo.vim b/software/vim/share/vim/vim92/syntax/lilo.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/lilo.vim rename to software/vim/share/vim/vim92/syntax/lilo.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/limits.vim b/software/vim/share/vim/vim92/syntax/limits.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/limits.vim rename to software/vim/share/vim/vim92/syntax/limits.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/liquid.vim b/software/vim/share/vim/vim92/syntax/liquid.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/liquid.vim rename to software/vim/share/vim/vim92/syntax/liquid.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/lisp.vim b/software/vim/share/vim/vim92/syntax/lisp.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/lisp.vim rename to software/vim/share/vim/vim92/syntax/lisp.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/lite.vim b/software/vim/share/vim/vim92/syntax/lite.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/lite.vim rename to software/vim/share/vim/vim92/syntax/lite.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/litestep.vim b/software/vim/share/vim/vim92/syntax/litestep.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/litestep.vim rename to software/vim/share/vim/vim92/syntax/litestep.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/loginaccess.vim b/software/vim/share/vim/vim92/syntax/loginaccess.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/loginaccess.vim rename to software/vim/share/vim/vim92/syntax/loginaccess.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/logindefs.vim b/software/vim/share/vim/vim92/syntax/logindefs.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/logindefs.vim rename to software/vim/share/vim/vim92/syntax/logindefs.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/logtalk.vim b/software/vim/share/vim/vim92/syntax/logtalk.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/logtalk.vim rename to software/vim/share/vim/vim92/syntax/logtalk.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/lotos.vim b/software/vim/share/vim/vim92/syntax/lotos.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/lotos.vim rename to software/vim/share/vim/vim92/syntax/lotos.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/lout.vim b/software/vim/share/vim/vim92/syntax/lout.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/lout.vim rename to software/vim/share/vim/vim92/syntax/lout.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/lpc.vim b/software/vim/share/vim/vim92/syntax/lpc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/lpc.vim rename to software/vim/share/vim/vim92/syntax/lpc.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/lprolog.vim b/software/vim/share/vim/vim92/syntax/lprolog.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/lprolog.vim rename to software/vim/share/vim/vim92/syntax/lprolog.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/lscript.vim b/software/vim/share/vim/vim92/syntax/lscript.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/lscript.vim rename to software/vim/share/vim/vim92/syntax/lscript.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/lsl.vim b/software/vim/share/vim/vim92/syntax/lsl.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/lsl.vim rename to software/vim/share/vim/vim92/syntax/lsl.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/lss.vim b/software/vim/share/vim/vim92/syntax/lss.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/lss.vim rename to software/vim/share/vim/vim92/syntax/lss.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/lua.vim b/software/vim/share/vim/vim92/syntax/lua.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/lua.vim rename to software/vim/share/vim/vim92/syntax/lua.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/lynx.vim b/software/vim/share/vim/vim92/syntax/lynx.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/lynx.vim rename to software/vim/share/vim/vim92/syntax/lynx.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/lyrics.vim b/software/vim/share/vim/vim92/syntax/lyrics.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/lyrics.vim rename to software/vim/share/vim/vim92/syntax/lyrics.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/m3build.vim b/software/vim/share/vim/vim92/syntax/m3build.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/m3build.vim rename to software/vim/share/vim/vim92/syntax/m3build.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/m3quake.vim b/software/vim/share/vim/vim92/syntax/m3quake.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/m3quake.vim rename to software/vim/share/vim/vim92/syntax/m3quake.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/m4.vim b/software/vim/share/vim/vim92/syntax/m4.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/m4.vim rename to software/vim/share/vim/vim92/syntax/m4.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/mail.vim b/software/vim/share/vim/vim92/syntax/mail.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/mail.vim rename to software/vim/share/vim/vim92/syntax/mail.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/mailaliases.vim b/software/vim/share/vim/vim92/syntax/mailaliases.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/mailaliases.vim rename to software/vim/share/vim/vim92/syntax/mailaliases.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/mailcap.vim b/software/vim/share/vim/vim92/syntax/mailcap.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/mailcap.vim rename to software/vim/share/vim/vim92/syntax/mailcap.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/make.vim b/software/vim/share/vim/vim92/syntax/make.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/make.vim rename to software/vim/share/vim/vim92/syntax/make.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/mallard.vim b/software/vim/share/vim/vim92/syntax/mallard.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/mallard.vim rename to software/vim/share/vim/vim92/syntax/mallard.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/man.vim b/software/vim/share/vim/vim92/syntax/man.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/man.vim rename to software/vim/share/vim/vim92/syntax/man.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/manconf.vim b/software/vim/share/vim/vim92/syntax/manconf.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/manconf.vim rename to software/vim/share/vim/vim92/syntax/manconf.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/manual.vim b/software/vim/share/vim/vim92/syntax/manual.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/manual.vim rename to software/vim/share/vim/vim92/syntax/manual.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/maple.vim b/software/vim/share/vim/vim92/syntax/maple.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/maple.vim rename to software/vim/share/vim/vim92/syntax/maple.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/markdown.vim b/software/vim/share/vim/vim92/syntax/markdown.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/markdown.vim rename to software/vim/share/vim/vim92/syntax/markdown.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/masm.vim b/software/vim/share/vim/vim92/syntax/masm.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/masm.vim rename to software/vim/share/vim/vim92/syntax/masm.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/mason.vim b/software/vim/share/vim/vim92/syntax/mason.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/mason.vim rename to software/vim/share/vim/vim92/syntax/mason.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/master.vim b/software/vim/share/vim/vim92/syntax/master.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/master.vim rename to software/vim/share/vim/vim92/syntax/master.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/matlab.vim b/software/vim/share/vim/vim92/syntax/matlab.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/matlab.vim rename to software/vim/share/vim/vim92/syntax/matlab.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/maxima.vim b/software/vim/share/vim/vim92/syntax/maxima.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/maxima.vim rename to software/vim/share/vim/vim92/syntax/maxima.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/mel.vim b/software/vim/share/vim/vim92/syntax/mel.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/mel.vim rename to software/vim/share/vim/vim92/syntax/mel.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/mermaid.vim b/software/vim/share/vim/vim92/syntax/mermaid.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/mermaid.vim rename to software/vim/share/vim/vim92/syntax/mermaid.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/meson.vim b/software/vim/share/vim/vim92/syntax/meson.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/meson.vim rename to software/vim/share/vim/vim92/syntax/meson.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/messages.vim b/software/vim/share/vim/vim92/syntax/messages.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/messages.vim rename to software/vim/share/vim/vim92/syntax/messages.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/mf.vim b/software/vim/share/vim/vim92/syntax/mf.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/mf.vim rename to software/vim/share/vim/vim92/syntax/mf.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/mgl.vim b/software/vim/share/vim/vim92/syntax/mgl.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/mgl.vim rename to software/vim/share/vim/vim92/syntax/mgl.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/mgp.vim b/software/vim/share/vim/vim92/syntax/mgp.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/mgp.vim rename to software/vim/share/vim/vim92/syntax/mgp.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/mib.vim b/software/vim/share/vim/vim92/syntax/mib.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/mib.vim rename to software/vim/share/vim/vim92/syntax/mib.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/mix.vim b/software/vim/share/vim/vim92/syntax/mix.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/mix.vim rename to software/vim/share/vim/vim92/syntax/mix.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/mma.vim b/software/vim/share/vim/vim92/syntax/mma.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/mma.vim rename to software/vim/share/vim/vim92/syntax/mma.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/mmix.vim b/software/vim/share/vim/vim92/syntax/mmix.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/mmix.vim rename to software/vim/share/vim/vim92/syntax/mmix.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/mmp.vim b/software/vim/share/vim/vim92/syntax/mmp.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/mmp.vim rename to software/vim/share/vim/vim92/syntax/mmp.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/modconf.vim b/software/vim/share/vim/vim92/syntax/modconf.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/modconf.vim rename to software/vim/share/vim/vim92/syntax/modconf.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/model.vim b/software/vim/share/vim/vim92/syntax/model.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/model.vim rename to software/vim/share/vim/vim92/syntax/model.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/modsim3.vim b/software/vim/share/vim/vim92/syntax/modsim3.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/modsim3.vim rename to software/vim/share/vim/vim92/syntax/modsim3.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/modula2.vim b/software/vim/share/vim/vim92/syntax/modula2.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/modula2.vim rename to software/vim/share/vim/vim92/syntax/modula2.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/modula3.vim b/software/vim/share/vim/vim92/syntax/modula3.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/modula3.vim rename to software/vim/share/vim/vim92/syntax/modula3.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/monk.vim b/software/vim/share/vim/vim92/syntax/monk.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/monk.vim rename to software/vim/share/vim/vim92/syntax/monk.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/moo.vim b/software/vim/share/vim/vim92/syntax/moo.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/moo.vim rename to software/vim/share/vim/vim92/syntax/moo.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/mp.vim b/software/vim/share/vim/vim92/syntax/mp.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/mp.vim rename to software/vim/share/vim/vim92/syntax/mp.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/mplayerconf.vim b/software/vim/share/vim/vim92/syntax/mplayerconf.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/mplayerconf.vim rename to software/vim/share/vim/vim92/syntax/mplayerconf.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/mrxvtrc.vim b/software/vim/share/vim/vim92/syntax/mrxvtrc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/mrxvtrc.vim rename to software/vim/share/vim/vim92/syntax/mrxvtrc.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/msidl.vim b/software/vim/share/vim/vim92/syntax/msidl.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/msidl.vim rename to software/vim/share/vim/vim92/syntax/msidl.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/msmessages.vim b/software/vim/share/vim/vim92/syntax/msmessages.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/msmessages.vim rename to software/vim/share/vim/vim92/syntax/msmessages.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/msql.vim b/software/vim/share/vim/vim92/syntax/msql.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/msql.vim rename to software/vim/share/vim/vim92/syntax/msql.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/mupad.vim b/software/vim/share/vim/vim92/syntax/mupad.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/mupad.vim rename to software/vim/share/vim/vim92/syntax/mupad.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/murphi.vim b/software/vim/share/vim/vim92/syntax/murphi.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/murphi.vim rename to software/vim/share/vim/vim92/syntax/murphi.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/mush.vim b/software/vim/share/vim/vim92/syntax/mush.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/mush.vim rename to software/vim/share/vim/vim92/syntax/mush.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/muttrc.vim b/software/vim/share/vim/vim92/syntax/muttrc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/muttrc.vim rename to software/vim/share/vim/vim92/syntax/muttrc.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/mysql.vim b/software/vim/share/vim/vim92/syntax/mysql.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/mysql.vim rename to software/vim/share/vim/vim92/syntax/mysql.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/n1ql.vim b/software/vim/share/vim/vim92/syntax/n1ql.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/n1ql.vim rename to software/vim/share/vim/vim92/syntax/n1ql.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/named.vim b/software/vim/share/vim/vim92/syntax/named.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/named.vim rename to software/vim/share/vim/vim92/syntax/named.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/nanorc.vim b/software/vim/share/vim/vim92/syntax/nanorc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/nanorc.vim rename to software/vim/share/vim/vim92/syntax/nanorc.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/nasm.vim b/software/vim/share/vim/vim92/syntax/nasm.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/nasm.vim rename to software/vim/share/vim/vim92/syntax/nasm.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/nastran.vim b/software/vim/share/vim/vim92/syntax/nastran.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/nastran.vim rename to software/vim/share/vim/vim92/syntax/nastran.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/natural.vim b/software/vim/share/vim/vim92/syntax/natural.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/natural.vim rename to software/vim/share/vim/vim92/syntax/natural.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/ncf.vim b/software/vim/share/vim/vim92/syntax/ncf.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/ncf.vim rename to software/vim/share/vim/vim92/syntax/ncf.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/neomuttrc.vim b/software/vim/share/vim/vim92/syntax/neomuttrc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/neomuttrc.vim rename to software/vim/share/vim/vim92/syntax/neomuttrc.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/netrc.vim b/software/vim/share/vim/vim92/syntax/netrc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/netrc.vim rename to software/vim/share/vim/vim92/syntax/netrc.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/netrw.vim b/software/vim/share/vim/vim92/syntax/netrw.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/netrw.vim rename to software/vim/share/vim/vim92/syntax/netrw.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/nginx.vim b/software/vim/share/vim/vim92/syntax/nginx.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/nginx.vim rename to software/vim/share/vim/vim92/syntax/nginx.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/ninja.vim b/software/vim/share/vim/vim92/syntax/ninja.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/ninja.vim rename to software/vim/share/vim/vim92/syntax/ninja.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/nix.vim b/software/vim/share/vim/vim92/syntax/nix.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/nix.vim rename to software/vim/share/vim/vim92/syntax/nix.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/nosyntax.vim b/software/vim/share/vim/vim92/syntax/nosyntax.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/nosyntax.vim rename to software/vim/share/vim/vim92/syntax/nosyntax.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/nqc.vim b/software/vim/share/vim/vim92/syntax/nqc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/nqc.vim rename to software/vim/share/vim/vim92/syntax/nqc.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/nroff.vim b/software/vim/share/vim/vim92/syntax/nroff.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/nroff.vim rename to software/vim/share/vim/vim92/syntax/nroff.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/nsis.vim b/software/vim/share/vim/vim92/syntax/nsis.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/nsis.vim rename to software/vim/share/vim/vim92/syntax/nsis.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/obj.vim b/software/vim/share/vim/vim92/syntax/obj.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/obj.vim rename to software/vim/share/vim/vim92/syntax/obj.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/objc.vim b/software/vim/share/vim/vim92/syntax/objc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/objc.vim rename to software/vim/share/vim/vim92/syntax/objc.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/objcpp.vim b/software/vim/share/vim/vim92/syntax/objcpp.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/objcpp.vim rename to software/vim/share/vim/vim92/syntax/objcpp.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/obse.vim b/software/vim/share/vim/vim92/syntax/obse.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/obse.vim rename to software/vim/share/vim/vim92/syntax/obse.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/ocaml.vim b/software/vim/share/vim/vim92/syntax/ocaml.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/ocaml.vim rename to software/vim/share/vim/vim92/syntax/ocaml.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/occam.vim b/software/vim/share/vim/vim92/syntax/occam.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/occam.vim rename to software/vim/share/vim/vim92/syntax/occam.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/omnimark.vim b/software/vim/share/vim/vim92/syntax/omnimark.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/omnimark.vim rename to software/vim/share/vim/vim92/syntax/omnimark.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/opam.vim b/software/vim/share/vim/vim92/syntax/opam.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/opam.vim rename to software/vim/share/vim/vim92/syntax/opam.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/openroad.vim b/software/vim/share/vim/vim92/syntax/openroad.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/openroad.vim rename to software/vim/share/vim/vim92/syntax/openroad.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/openscad.vim b/software/vim/share/vim/vim92/syntax/openscad.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/openscad.vim rename to software/vim/share/vim/vim92/syntax/openscad.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/openvpn.vim b/software/vim/share/vim/vim92/syntax/openvpn.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/openvpn.vim rename to software/vim/share/vim/vim92/syntax/openvpn.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/opl.vim b/software/vim/share/vim/vim92/syntax/opl.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/opl.vim rename to software/vim/share/vim/vim92/syntax/opl.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/ora.vim b/software/vim/share/vim/vim92/syntax/ora.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/ora.vim rename to software/vim/share/vim/vim92/syntax/ora.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/pamconf.vim b/software/vim/share/vim/vim92/syntax/pamconf.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/pamconf.vim rename to software/vim/share/vim/vim92/syntax/pamconf.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/pamenv.vim b/software/vim/share/vim/vim92/syntax/pamenv.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/pamenv.vim rename to software/vim/share/vim/vim92/syntax/pamenv.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/papp.vim b/software/vim/share/vim/vim92/syntax/papp.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/papp.vim rename to software/vim/share/vim/vim92/syntax/papp.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/pascal.vim b/software/vim/share/vim/vim92/syntax/pascal.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/pascal.vim rename to software/vim/share/vim/vim92/syntax/pascal.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/passwd.vim b/software/vim/share/vim/vim92/syntax/passwd.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/passwd.vim rename to software/vim/share/vim/vim92/syntax/passwd.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/pbtxt.vim b/software/vim/share/vim/vim92/syntax/pbtxt.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/pbtxt.vim rename to software/vim/share/vim/vim92/syntax/pbtxt.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/pcap.vim b/software/vim/share/vim/vim92/syntax/pcap.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/pcap.vim rename to software/vim/share/vim/vim92/syntax/pcap.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/pccts.vim b/software/vim/share/vim/vim92/syntax/pccts.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/pccts.vim rename to software/vim/share/vim/vim92/syntax/pccts.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/pdf.vim b/software/vim/share/vim/vim92/syntax/pdf.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/pdf.vim rename to software/vim/share/vim/vim92/syntax/pdf.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/perl.vim b/software/vim/share/vim/vim92/syntax/perl.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/perl.vim rename to software/vim/share/vim/vim92/syntax/perl.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/pf.vim b/software/vim/share/vim/vim92/syntax/pf.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/pf.vim rename to software/vim/share/vim/vim92/syntax/pf.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/pfmain.vim b/software/vim/share/vim/vim92/syntax/pfmain.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/pfmain.vim rename to software/vim/share/vim/vim92/syntax/pfmain.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/php.vim b/software/vim/share/vim/vim92/syntax/php.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/php.vim rename to software/vim/share/vim/vim92/syntax/php.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/phtml.vim b/software/vim/share/vim/vim92/syntax/phtml.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/phtml.vim rename to software/vim/share/vim/vim92/syntax/phtml.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/pic.vim b/software/vim/share/vim/vim92/syntax/pic.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/pic.vim rename to software/vim/share/vim/vim92/syntax/pic.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/pike.vim b/software/vim/share/vim/vim92/syntax/pike.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/pike.vim rename to software/vim/share/vim/vim92/syntax/pike.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/pilrc.vim b/software/vim/share/vim/vim92/syntax/pilrc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/pilrc.vim rename to software/vim/share/vim/vim92/syntax/pilrc.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/pine.vim b/software/vim/share/vim/vim92/syntax/pine.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/pine.vim rename to software/vim/share/vim/vim92/syntax/pine.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/pinfo.vim b/software/vim/share/vim/vim92/syntax/pinfo.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/pinfo.vim rename to software/vim/share/vim/vim92/syntax/pinfo.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/plaintex.vim b/software/vim/share/vim/vim92/syntax/plaintex.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/plaintex.vim rename to software/vim/share/vim/vim92/syntax/plaintex.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/pli.vim b/software/vim/share/vim/vim92/syntax/pli.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/pli.vim rename to software/vim/share/vim/vim92/syntax/pli.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/plm.vim b/software/vim/share/vim/vim92/syntax/plm.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/plm.vim rename to software/vim/share/vim/vim92/syntax/plm.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/plp.vim b/software/vim/share/vim/vim92/syntax/plp.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/plp.vim rename to software/vim/share/vim/vim92/syntax/plp.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/plsql.vim b/software/vim/share/vim/vim92/syntax/plsql.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/plsql.vim rename to software/vim/share/vim/vim92/syntax/plsql.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/po.vim b/software/vim/share/vim/vim92/syntax/po.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/po.vim rename to software/vim/share/vim/vim92/syntax/po.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/pod.vim b/software/vim/share/vim/vim92/syntax/pod.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/pod.vim rename to software/vim/share/vim/vim92/syntax/pod.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/poefilter.vim b/software/vim/share/vim/vim92/syntax/poefilter.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/poefilter.vim rename to software/vim/share/vim/vim92/syntax/poefilter.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/poke.vim b/software/vim/share/vim/vim92/syntax/poke.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/poke.vim rename to software/vim/share/vim/vim92/syntax/poke.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/postscr.vim b/software/vim/share/vim/vim92/syntax/postscr.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/postscr.vim rename to software/vim/share/vim/vim92/syntax/postscr.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/pov.vim b/software/vim/share/vim/vim92/syntax/pov.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/pov.vim rename to software/vim/share/vim/vim92/syntax/pov.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/povini.vim b/software/vim/share/vim/vim92/syntax/povini.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/povini.vim rename to software/vim/share/vim/vim92/syntax/povini.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/ppd.vim b/software/vim/share/vim/vim92/syntax/ppd.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/ppd.vim rename to software/vim/share/vim/vim92/syntax/ppd.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/ppwiz.vim b/software/vim/share/vim/vim92/syntax/ppwiz.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/ppwiz.vim rename to software/vim/share/vim/vim92/syntax/ppwiz.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/prescribe.vim b/software/vim/share/vim/vim92/syntax/prescribe.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/prescribe.vim rename to software/vim/share/vim/vim92/syntax/prescribe.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/privoxy.vim b/software/vim/share/vim/vim92/syntax/privoxy.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/privoxy.vim rename to software/vim/share/vim/vim92/syntax/privoxy.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/procmail.vim b/software/vim/share/vim/vim92/syntax/procmail.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/procmail.vim rename to software/vim/share/vim/vim92/syntax/procmail.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/progress.vim b/software/vim/share/vim/vim92/syntax/progress.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/progress.vim rename to software/vim/share/vim/vim92/syntax/progress.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/prolog.vim b/software/vim/share/vim/vim92/syntax/prolog.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/prolog.vim rename to software/vim/share/vim/vim92/syntax/prolog.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/promela.vim b/software/vim/share/vim/vim92/syntax/promela.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/promela.vim rename to software/vim/share/vim/vim92/syntax/promela.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/proto.vim b/software/vim/share/vim/vim92/syntax/proto.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/proto.vim rename to software/vim/share/vim/vim92/syntax/proto.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/protocols.vim b/software/vim/share/vim/vim92/syntax/protocols.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/protocols.vim rename to software/vim/share/vim/vim92/syntax/protocols.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/ps1.vim b/software/vim/share/vim/vim92/syntax/ps1.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/ps1.vim rename to software/vim/share/vim/vim92/syntax/ps1.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/ps1xml.vim b/software/vim/share/vim/vim92/syntax/ps1xml.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/ps1xml.vim rename to software/vim/share/vim/vim92/syntax/ps1xml.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/psf.vim b/software/vim/share/vim/vim92/syntax/psf.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/psf.vim rename to software/vim/share/vim/vim92/syntax/psf.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/psl.vim b/software/vim/share/vim/vim92/syntax/psl.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/psl.vim rename to software/vim/share/vim/vim92/syntax/psl.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/ptcap.vim b/software/vim/share/vim/vim92/syntax/ptcap.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/ptcap.vim rename to software/vim/share/vim/vim92/syntax/ptcap.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/purifylog.vim b/software/vim/share/vim/vim92/syntax/purifylog.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/purifylog.vim rename to software/vim/share/vim/vim92/syntax/purifylog.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/pyrex.vim b/software/vim/share/vim/vim92/syntax/pyrex.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/pyrex.vim rename to software/vim/share/vim/vim92/syntax/pyrex.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/python.vim b/software/vim/share/vim/vim92/syntax/python.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/python.vim rename to software/vim/share/vim/vim92/syntax/python.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/python2.vim b/software/vim/share/vim/vim92/syntax/python2.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/python2.vim rename to software/vim/share/vim/vim92/syntax/python2.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/qb64.vim b/software/vim/share/vim/vim92/syntax/qb64.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/qb64.vim rename to software/vim/share/vim/vim92/syntax/qb64.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/qf.vim b/software/vim/share/vim/vim92/syntax/qf.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/qf.vim rename to software/vim/share/vim/vim92/syntax/qf.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/quake.vim b/software/vim/share/vim/vim92/syntax/quake.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/quake.vim rename to software/vim/share/vim/vim92/syntax/quake.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/quarto.vim b/software/vim/share/vim/vim92/syntax/quarto.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/quarto.vim rename to software/vim/share/vim/vim92/syntax/quarto.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/r.vim b/software/vim/share/vim/vim92/syntax/r.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/r.vim rename to software/vim/share/vim/vim92/syntax/r.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/racc.vim b/software/vim/share/vim/vim92/syntax/racc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/racc.vim rename to software/vim/share/vim/vim92/syntax/racc.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/racket.vim b/software/vim/share/vim/vim92/syntax/racket.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/racket.vim rename to software/vim/share/vim/vim92/syntax/racket.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/radiance.vim b/software/vim/share/vim/vim92/syntax/radiance.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/radiance.vim rename to software/vim/share/vim/vim92/syntax/radiance.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/raku.vim b/software/vim/share/vim/vim92/syntax/raku.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/raku.vim rename to software/vim/share/vim/vim92/syntax/raku.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/raml.vim b/software/vim/share/vim/vim92/syntax/raml.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/raml.vim rename to software/vim/share/vim/vim92/syntax/raml.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/ratpoison.vim b/software/vim/share/vim/vim92/syntax/ratpoison.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/ratpoison.vim rename to software/vim/share/vim/vim92/syntax/ratpoison.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/rc.vim b/software/vim/share/vim/vim92/syntax/rc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/rc.vim rename to software/vim/share/vim/vim92/syntax/rc.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/rcs.vim b/software/vim/share/vim/vim92/syntax/rcs.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/rcs.vim rename to software/vim/share/vim/vim92/syntax/rcs.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/rcslog.vim b/software/vim/share/vim/vim92/syntax/rcslog.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/rcslog.vim rename to software/vim/share/vim/vim92/syntax/rcslog.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/readline.vim b/software/vim/share/vim/vim92/syntax/readline.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/readline.vim rename to software/vim/share/vim/vim92/syntax/readline.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/rebol.vim b/software/vim/share/vim/vim92/syntax/rebol.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/rebol.vim rename to software/vim/share/vim/vim92/syntax/rebol.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/redif.vim b/software/vim/share/vim/vim92/syntax/redif.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/redif.vim rename to software/vim/share/vim/vim92/syntax/redif.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/registry.vim b/software/vim/share/vim/vim92/syntax/registry.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/registry.vim rename to software/vim/share/vim/vim92/syntax/registry.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/rego.vim b/software/vim/share/vim/vim92/syntax/rego.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/rego.vim rename to software/vim/share/vim/vim92/syntax/rego.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/remind.vim b/software/vim/share/vim/vim92/syntax/remind.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/remind.vim rename to software/vim/share/vim/vim92/syntax/remind.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/resolv.vim b/software/vim/share/vim/vim92/syntax/resolv.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/resolv.vim rename to software/vim/share/vim/vim92/syntax/resolv.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/reva.vim b/software/vim/share/vim/vim92/syntax/reva.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/reva.vim rename to software/vim/share/vim/vim92/syntax/reva.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/rexx.vim b/software/vim/share/vim/vim92/syntax/rexx.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/rexx.vim rename to software/vim/share/vim/vim92/syntax/rexx.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/rhelp.vim b/software/vim/share/vim/vim92/syntax/rhelp.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/rhelp.vim rename to software/vim/share/vim/vim92/syntax/rhelp.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/rib.vim b/software/vim/share/vim/vim92/syntax/rib.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/rib.vim rename to software/vim/share/vim/vim92/syntax/rib.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/rmd.vim b/software/vim/share/vim/vim92/syntax/rmd.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/rmd.vim rename to software/vim/share/vim/vim92/syntax/rmd.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/rnc.vim b/software/vim/share/vim/vim92/syntax/rnc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/rnc.vim rename to software/vim/share/vim/vim92/syntax/rnc.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/rng.vim b/software/vim/share/vim/vim92/syntax/rng.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/rng.vim rename to software/vim/share/vim/vim92/syntax/rng.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/rnoweb.vim b/software/vim/share/vim/vim92/syntax/rnoweb.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/rnoweb.vim rename to software/vim/share/vim/vim92/syntax/rnoweb.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/robots.vim b/software/vim/share/vim/vim92/syntax/robots.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/robots.vim rename to software/vim/share/vim/vim92/syntax/robots.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/routeros.vim b/software/vim/share/vim/vim92/syntax/routeros.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/routeros.vim rename to software/vim/share/vim/vim92/syntax/routeros.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/rpcgen.vim b/software/vim/share/vim/vim92/syntax/rpcgen.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/rpcgen.vim rename to software/vim/share/vim/vim92/syntax/rpcgen.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/rpl.vim b/software/vim/share/vim/vim92/syntax/rpl.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/rpl.vim rename to software/vim/share/vim/vim92/syntax/rpl.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/rrst.vim b/software/vim/share/vim/vim92/syntax/rrst.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/rrst.vim rename to software/vim/share/vim/vim92/syntax/rrst.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/rst.vim b/software/vim/share/vim/vim92/syntax/rst.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/rst.vim rename to software/vim/share/vim/vim92/syntax/rst.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/rtf.vim b/software/vim/share/vim/vim92/syntax/rtf.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/rtf.vim rename to software/vim/share/vim/vim92/syntax/rtf.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/ruby.vim b/software/vim/share/vim/vim92/syntax/ruby.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/ruby.vim rename to software/vim/share/vim/vim92/syntax/ruby.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/rust.vim b/software/vim/share/vim/vim92/syntax/rust.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/rust.vim rename to software/vim/share/vim/vim92/syntax/rust.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/samba.vim b/software/vim/share/vim/vim92/syntax/samba.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/samba.vim rename to software/vim/share/vim/vim92/syntax/samba.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/sas.vim b/software/vim/share/vim/vim92/syntax/sas.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/sas.vim rename to software/vim/share/vim/vim92/syntax/sas.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/sass.vim b/software/vim/share/vim/vim92/syntax/sass.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/sass.vim rename to software/vim/share/vim/vim92/syntax/sass.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/sather.vim b/software/vim/share/vim/vim92/syntax/sather.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/sather.vim rename to software/vim/share/vim/vim92/syntax/sather.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/sbt.vim b/software/vim/share/vim/vim92/syntax/sbt.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/sbt.vim rename to software/vim/share/vim/vim92/syntax/sbt.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/scala.vim b/software/vim/share/vim/vim92/syntax/scala.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/scala.vim rename to software/vim/share/vim/vim92/syntax/scala.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/scdoc.vim b/software/vim/share/vim/vim92/syntax/scdoc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/scdoc.vim rename to software/vim/share/vim/vim92/syntax/scdoc.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/scheme.vim b/software/vim/share/vim/vim92/syntax/scheme.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/scheme.vim rename to software/vim/share/vim/vim92/syntax/scheme.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/scilab.vim b/software/vim/share/vim/vim92/syntax/scilab.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/scilab.vim rename to software/vim/share/vim/vim92/syntax/scilab.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/screen.vim b/software/vim/share/vim/vim92/syntax/screen.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/screen.vim rename to software/vim/share/vim/vim92/syntax/screen.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/scss.vim b/software/vim/share/vim/vim92/syntax/scss.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/scss.vim rename to software/vim/share/vim/vim92/syntax/scss.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/sd.vim b/software/vim/share/vim/vim92/syntax/sd.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/sd.vim rename to software/vim/share/vim/vim92/syntax/sd.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/sdc.vim b/software/vim/share/vim/vim92/syntax/sdc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/sdc.vim rename to software/vim/share/vim/vim92/syntax/sdc.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/sdl.vim b/software/vim/share/vim/vim92/syntax/sdl.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/sdl.vim rename to software/vim/share/vim/vim92/syntax/sdl.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/sed.vim b/software/vim/share/vim/vim92/syntax/sed.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/sed.vim rename to software/vim/share/vim/vim92/syntax/sed.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/sendpr.vim b/software/vim/share/vim/vim92/syntax/sendpr.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/sendpr.vim rename to software/vim/share/vim/vim92/syntax/sendpr.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/sensors.vim b/software/vim/share/vim/vim92/syntax/sensors.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/sensors.vim rename to software/vim/share/vim/vim92/syntax/sensors.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/services.vim b/software/vim/share/vim/vim92/syntax/services.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/services.vim rename to software/vim/share/vim/vim92/syntax/services.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/setserial.vim b/software/vim/share/vim/vim92/syntax/setserial.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/setserial.vim rename to software/vim/share/vim/vim92/syntax/setserial.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/sexplib.vim b/software/vim/share/vim/vim92/syntax/sexplib.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/sexplib.vim rename to software/vim/share/vim/vim92/syntax/sexplib.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/sgml.vim b/software/vim/share/vim/vim92/syntax/sgml.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/sgml.vim rename to software/vim/share/vim/vim92/syntax/sgml.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/sgmldecl.vim b/software/vim/share/vim/vim92/syntax/sgmldecl.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/sgmldecl.vim rename to software/vim/share/vim/vim92/syntax/sgmldecl.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/sgmllnx.vim b/software/vim/share/vim/vim92/syntax/sgmllnx.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/sgmllnx.vim rename to software/vim/share/vim/vim92/syntax/sgmllnx.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/sh.vim b/software/vim/share/vim/vim92/syntax/sh.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/sh.vim rename to software/vim/share/vim/vim92/syntax/sh.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/shared/context-data-context.vim b/software/vim/share/vim/vim92/syntax/shared/context-data-context.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/shared/context-data-context.vim rename to software/vim/share/vim/vim92/syntax/shared/context-data-context.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/shared/context-data-interfaces.vim b/software/vim/share/vim/vim92/syntax/shared/context-data-interfaces.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/shared/context-data-interfaces.vim rename to software/vim/share/vim/vim92/syntax/shared/context-data-interfaces.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/shared/context-data-metafun.vim b/software/vim/share/vim/vim92/syntax/shared/context-data-metafun.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/shared/context-data-metafun.vim rename to software/vim/share/vim/vim92/syntax/shared/context-data-metafun.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/shared/context-data-tex.vim b/software/vim/share/vim/vim92/syntax/shared/context-data-tex.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/shared/context-data-tex.vim rename to software/vim/share/vim/vim92/syntax/shared/context-data-tex.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/shared/hgcommitDiff.vim b/software/vim/share/vim/vim92/syntax/shared/hgcommitDiff.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/shared/hgcommitDiff.vim rename to software/vim/share/vim/vim92/syntax/shared/hgcommitDiff.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/shared/typescriptcommon.vim b/software/vim/share/vim/vim92/syntax/shared/typescriptcommon.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/shared/typescriptcommon.vim rename to software/vim/share/vim/vim92/syntax/shared/typescriptcommon.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/sicad.vim b/software/vim/share/vim/vim92/syntax/sicad.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/sicad.vim rename to software/vim/share/vim/vim92/syntax/sicad.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/sieve.vim b/software/vim/share/vim/vim92/syntax/sieve.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/sieve.vim rename to software/vim/share/vim/vim92/syntax/sieve.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/sil.vim b/software/vim/share/vim/vim92/syntax/sil.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/sil.vim rename to software/vim/share/vim/vim92/syntax/sil.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/simula.vim b/software/vim/share/vim/vim92/syntax/simula.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/simula.vim rename to software/vim/share/vim/vim92/syntax/simula.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/sinda.vim b/software/vim/share/vim/vim92/syntax/sinda.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/sinda.vim rename to software/vim/share/vim/vim92/syntax/sinda.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/sindacmp.vim b/software/vim/share/vim/vim92/syntax/sindacmp.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/sindacmp.vim rename to software/vim/share/vim/vim92/syntax/sindacmp.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/sindaout.vim b/software/vim/share/vim/vim92/syntax/sindaout.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/sindaout.vim rename to software/vim/share/vim/vim92/syntax/sindaout.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/sisu.vim b/software/vim/share/vim/vim92/syntax/sisu.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/sisu.vim rename to software/vim/share/vim/vim92/syntax/sisu.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/skill.vim b/software/vim/share/vim/vim92/syntax/skill.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/skill.vim rename to software/vim/share/vim/vim92/syntax/skill.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/sl.vim b/software/vim/share/vim/vim92/syntax/sl.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/sl.vim rename to software/vim/share/vim/vim92/syntax/sl.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/slang.vim b/software/vim/share/vim/vim92/syntax/slang.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/slang.vim rename to software/vim/share/vim/vim92/syntax/slang.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/slice.vim b/software/vim/share/vim/vim92/syntax/slice.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/slice.vim rename to software/vim/share/vim/vim92/syntax/slice.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/slpconf.vim b/software/vim/share/vim/vim92/syntax/slpconf.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/slpconf.vim rename to software/vim/share/vim/vim92/syntax/slpconf.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/slpreg.vim b/software/vim/share/vim/vim92/syntax/slpreg.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/slpreg.vim rename to software/vim/share/vim/vim92/syntax/slpreg.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/slpspi.vim b/software/vim/share/vim/vim92/syntax/slpspi.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/slpspi.vim rename to software/vim/share/vim/vim92/syntax/slpspi.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/slrnrc.vim b/software/vim/share/vim/vim92/syntax/slrnrc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/slrnrc.vim rename to software/vim/share/vim/vim92/syntax/slrnrc.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/slrnsc.vim b/software/vim/share/vim/vim92/syntax/slrnsc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/slrnsc.vim rename to software/vim/share/vim/vim92/syntax/slrnsc.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/sm.vim b/software/vim/share/vim/vim92/syntax/sm.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/sm.vim rename to software/vim/share/vim/vim92/syntax/sm.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/smarty.vim b/software/vim/share/vim/vim92/syntax/smarty.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/smarty.vim rename to software/vim/share/vim/vim92/syntax/smarty.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/smcl.vim b/software/vim/share/vim/vim92/syntax/smcl.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/smcl.vim rename to software/vim/share/vim/vim92/syntax/smcl.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/smil.vim b/software/vim/share/vim/vim92/syntax/smil.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/smil.vim rename to software/vim/share/vim/vim92/syntax/smil.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/smith.vim b/software/vim/share/vim/vim92/syntax/smith.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/smith.vim rename to software/vim/share/vim/vim92/syntax/smith.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/sml.vim b/software/vim/share/vim/vim92/syntax/sml.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/sml.vim rename to software/vim/share/vim/vim92/syntax/sml.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/snnsnet.vim b/software/vim/share/vim/vim92/syntax/snnsnet.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/snnsnet.vim rename to software/vim/share/vim/vim92/syntax/snnsnet.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/snnspat.vim b/software/vim/share/vim/vim92/syntax/snnspat.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/snnspat.vim rename to software/vim/share/vim/vim92/syntax/snnspat.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/snnsres.vim b/software/vim/share/vim/vim92/syntax/snnsres.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/snnsres.vim rename to software/vim/share/vim/vim92/syntax/snnsres.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/snobol4.vim b/software/vim/share/vim/vim92/syntax/snobol4.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/snobol4.vim rename to software/vim/share/vim/vim92/syntax/snobol4.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/solidity.vim b/software/vim/share/vim/vim92/syntax/solidity.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/solidity.vim rename to software/vim/share/vim/vim92/syntax/solidity.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/spec.vim b/software/vim/share/vim/vim92/syntax/spec.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/spec.vim rename to software/vim/share/vim/vim92/syntax/spec.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/specman.vim b/software/vim/share/vim/vim92/syntax/specman.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/specman.vim rename to software/vim/share/vim/vim92/syntax/specman.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/spice.vim b/software/vim/share/vim/vim92/syntax/spice.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/spice.vim rename to software/vim/share/vim/vim92/syntax/spice.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/splint.vim b/software/vim/share/vim/vim92/syntax/splint.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/splint.vim rename to software/vim/share/vim/vim92/syntax/splint.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/spup.vim b/software/vim/share/vim/vim92/syntax/spup.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/spup.vim rename to software/vim/share/vim/vim92/syntax/spup.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/spyce.vim b/software/vim/share/vim/vim92/syntax/spyce.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/spyce.vim rename to software/vim/share/vim/vim92/syntax/spyce.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/sql.vim b/software/vim/share/vim/vim92/syntax/sql.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/sql.vim rename to software/vim/share/vim/vim92/syntax/sql.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/sqlanywhere.vim b/software/vim/share/vim/vim92/syntax/sqlanywhere.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/sqlanywhere.vim rename to software/vim/share/vim/vim92/syntax/sqlanywhere.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/sqlforms.vim b/software/vim/share/vim/vim92/syntax/sqlforms.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/sqlforms.vim rename to software/vim/share/vim/vim92/syntax/sqlforms.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/sqlhana.vim b/software/vim/share/vim/vim92/syntax/sqlhana.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/sqlhana.vim rename to software/vim/share/vim/vim92/syntax/sqlhana.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/sqlinformix.vim b/software/vim/share/vim/vim92/syntax/sqlinformix.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/sqlinformix.vim rename to software/vim/share/vim/vim92/syntax/sqlinformix.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/sqlj.vim b/software/vim/share/vim/vim92/syntax/sqlj.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/sqlj.vim rename to software/vim/share/vim/vim92/syntax/sqlj.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/sqloracle.vim b/software/vim/share/vim/vim92/syntax/sqloracle.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/sqloracle.vim rename to software/vim/share/vim/vim92/syntax/sqloracle.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/sqr.vim b/software/vim/share/vim/vim92/syntax/sqr.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/sqr.vim rename to software/vim/share/vim/vim92/syntax/sqr.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/squid.vim b/software/vim/share/vim/vim92/syntax/squid.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/squid.vim rename to software/vim/share/vim/vim92/syntax/squid.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/squirrel.vim b/software/vim/share/vim/vim92/syntax/squirrel.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/squirrel.vim rename to software/vim/share/vim/vim92/syntax/squirrel.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/srec.vim b/software/vim/share/vim/vim92/syntax/srec.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/srec.vim rename to software/vim/share/vim/vim92/syntax/srec.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/srt.vim b/software/vim/share/vim/vim92/syntax/srt.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/srt.vim rename to software/vim/share/vim/vim92/syntax/srt.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/ssa.vim b/software/vim/share/vim/vim92/syntax/ssa.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/ssa.vim rename to software/vim/share/vim/vim92/syntax/ssa.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/sshconfig.vim b/software/vim/share/vim/vim92/syntax/sshconfig.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/sshconfig.vim rename to software/vim/share/vim/vim92/syntax/sshconfig.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/sshdconfig.vim b/software/vim/share/vim/vim92/syntax/sshdconfig.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/sshdconfig.vim rename to software/vim/share/vim/vim92/syntax/sshdconfig.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/st.vim b/software/vim/share/vim/vim92/syntax/st.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/st.vim rename to software/vim/share/vim/vim92/syntax/st.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/stata.vim b/software/vim/share/vim/vim92/syntax/stata.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/stata.vim rename to software/vim/share/vim/vim92/syntax/stata.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/stp.vim b/software/vim/share/vim/vim92/syntax/stp.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/stp.vim rename to software/vim/share/vim/vim92/syntax/stp.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/strace.vim b/software/vim/share/vim/vim92/syntax/strace.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/strace.vim rename to software/vim/share/vim/vim92/syntax/strace.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/structurizr.vim b/software/vim/share/vim/vim92/syntax/structurizr.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/structurizr.vim rename to software/vim/share/vim/vim92/syntax/structurizr.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/sudoers.vim b/software/vim/share/vim/vim92/syntax/sudoers.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/sudoers.vim rename to software/vim/share/vim/vim92/syntax/sudoers.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/svg.vim b/software/vim/share/vim/vim92/syntax/svg.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/svg.vim rename to software/vim/share/vim/vim92/syntax/svg.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/svn.vim b/software/vim/share/vim/vim92/syntax/svn.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/svn.vim rename to software/vim/share/vim/vim92/syntax/svn.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/swayconfig.vim b/software/vim/share/vim/vim92/syntax/swayconfig.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/swayconfig.vim rename to software/vim/share/vim/vim92/syntax/swayconfig.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/swift.vim b/software/vim/share/vim/vim92/syntax/swift.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/swift.vim rename to software/vim/share/vim/vim92/syntax/swift.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/swiftgyb.vim b/software/vim/share/vim/vim92/syntax/swiftgyb.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/swiftgyb.vim rename to software/vim/share/vim/vim92/syntax/swiftgyb.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/syncolor.vim b/software/vim/share/vim/vim92/syntax/syncolor.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/syncolor.vim rename to software/vim/share/vim/vim92/syntax/syncolor.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/synload.vim b/software/vim/share/vim/vim92/syntax/synload.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/synload.vim rename to software/vim/share/vim/vim92/syntax/synload.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/syntax.vim b/software/vim/share/vim/vim92/syntax/syntax.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/syntax.vim rename to software/vim/share/vim/vim92/syntax/syntax.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/sysctl.vim b/software/vim/share/vim/vim92/syntax/sysctl.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/sysctl.vim rename to software/vim/share/vim/vim92/syntax/sysctl.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/systemd.vim b/software/vim/share/vim/vim92/syntax/systemd.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/systemd.vim rename to software/vim/share/vim/vim92/syntax/systemd.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/systemverilog.vim b/software/vim/share/vim/vim92/syntax/systemverilog.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/systemverilog.vim rename to software/vim/share/vim/vim92/syntax/systemverilog.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/tads.vim b/software/vim/share/vim/vim92/syntax/tads.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/tads.vim rename to software/vim/share/vim/vim92/syntax/tads.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/tags.vim b/software/vim/share/vim/vim92/syntax/tags.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/tags.vim rename to software/vim/share/vim/vim92/syntax/tags.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/tak.vim b/software/vim/share/vim/vim92/syntax/tak.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/tak.vim rename to software/vim/share/vim/vim92/syntax/tak.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/takcmp.vim b/software/vim/share/vim/vim92/syntax/takcmp.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/takcmp.vim rename to software/vim/share/vim/vim92/syntax/takcmp.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/takout.vim b/software/vim/share/vim/vim92/syntax/takout.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/takout.vim rename to software/vim/share/vim/vim92/syntax/takout.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/tap.vim b/software/vim/share/vim/vim92/syntax/tap.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/tap.vim rename to software/vim/share/vim/vim92/syntax/tap.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/tar.vim b/software/vim/share/vim/vim92/syntax/tar.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/tar.vim rename to software/vim/share/vim/vim92/syntax/tar.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/taskdata.vim b/software/vim/share/vim/vim92/syntax/taskdata.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/taskdata.vim rename to software/vim/share/vim/vim92/syntax/taskdata.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/taskedit.vim b/software/vim/share/vim/vim92/syntax/taskedit.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/taskedit.vim rename to software/vim/share/vim/vim92/syntax/taskedit.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/tasm.vim b/software/vim/share/vim/vim92/syntax/tasm.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/tasm.vim rename to software/vim/share/vim/vim92/syntax/tasm.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/tcl.vim b/software/vim/share/vim/vim92/syntax/tcl.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/tcl.vim rename to software/vim/share/vim/vim92/syntax/tcl.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/tcsh.vim b/software/vim/share/vim/vim92/syntax/tcsh.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/tcsh.vim rename to software/vim/share/vim/vim92/syntax/tcsh.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/template.vim b/software/vim/share/vim/vim92/syntax/template.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/template.vim rename to software/vim/share/vim/vim92/syntax/template.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/teraterm.vim b/software/vim/share/vim/vim92/syntax/teraterm.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/teraterm.vim rename to software/vim/share/vim/vim92/syntax/teraterm.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/terminfo.vim b/software/vim/share/vim/vim92/syntax/terminfo.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/terminfo.vim rename to software/vim/share/vim/vim92/syntax/terminfo.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/tex.vim b/software/vim/share/vim/vim92/syntax/tex.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/tex.vim rename to software/vim/share/vim/vim92/syntax/tex.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/texinfo.vim b/software/vim/share/vim/vim92/syntax/texinfo.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/texinfo.vim rename to software/vim/share/vim/vim92/syntax/texinfo.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/texmf.vim b/software/vim/share/vim/vim92/syntax/texmf.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/texmf.vim rename to software/vim/share/vim/vim92/syntax/texmf.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/tf.vim b/software/vim/share/vim/vim92/syntax/tf.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/tf.vim rename to software/vim/share/vim/vim92/syntax/tf.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/tidy.vim b/software/vim/share/vim/vim92/syntax/tidy.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/tidy.vim rename to software/vim/share/vim/vim92/syntax/tidy.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/tilde.vim b/software/vim/share/vim/vim92/syntax/tilde.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/tilde.vim rename to software/vim/share/vim/vim92/syntax/tilde.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/tli.vim b/software/vim/share/vim/vim92/syntax/tli.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/tli.vim rename to software/vim/share/vim/vim92/syntax/tli.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/tmux.vim b/software/vim/share/vim/vim92/syntax/tmux.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/tmux.vim rename to software/vim/share/vim/vim92/syntax/tmux.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/toml.vim b/software/vim/share/vim/vim92/syntax/toml.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/toml.vim rename to software/vim/share/vim/vim92/syntax/toml.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/tpp.vim b/software/vim/share/vim/vim92/syntax/tpp.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/tpp.vim rename to software/vim/share/vim/vim92/syntax/tpp.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/trasys.vim b/software/vim/share/vim/vim92/syntax/trasys.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/trasys.vim rename to software/vim/share/vim/vim92/syntax/trasys.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/treetop.vim b/software/vim/share/vim/vim92/syntax/treetop.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/treetop.vim rename to software/vim/share/vim/vim92/syntax/treetop.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/trustees.vim b/software/vim/share/vim/vim92/syntax/trustees.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/trustees.vim rename to software/vim/share/vim/vim92/syntax/trustees.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/tsalt.vim b/software/vim/share/vim/vim92/syntax/tsalt.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/tsalt.vim rename to software/vim/share/vim/vim92/syntax/tsalt.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/tsscl.vim b/software/vim/share/vim/vim92/syntax/tsscl.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/tsscl.vim rename to software/vim/share/vim/vim92/syntax/tsscl.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/tssgm.vim b/software/vim/share/vim/vim92/syntax/tssgm.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/tssgm.vim rename to software/vim/share/vim/vim92/syntax/tssgm.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/tssop.vim b/software/vim/share/vim/vim92/syntax/tssop.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/tssop.vim rename to software/vim/share/vim/vim92/syntax/tssop.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/tt2.vim b/software/vim/share/vim/vim92/syntax/tt2.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/tt2.vim rename to software/vim/share/vim/vim92/syntax/tt2.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/tt2html.vim b/software/vim/share/vim/vim92/syntax/tt2html.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/tt2html.vim rename to software/vim/share/vim/vim92/syntax/tt2html.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/tt2js.vim b/software/vim/share/vim/vim92/syntax/tt2js.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/tt2js.vim rename to software/vim/share/vim/vim92/syntax/tt2js.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/typescript.vim b/software/vim/share/vim/vim92/syntax/typescript.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/typescript.vim rename to software/vim/share/vim/vim92/syntax/typescript.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/typescriptreact.vim b/software/vim/share/vim/vim92/syntax/typescriptreact.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/typescriptreact.vim rename to software/vim/share/vim/vim92/syntax/typescriptreact.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/uc.vim b/software/vim/share/vim/vim92/syntax/uc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/uc.vim rename to software/vim/share/vim/vim92/syntax/uc.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/udevconf.vim b/software/vim/share/vim/vim92/syntax/udevconf.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/udevconf.vim rename to software/vim/share/vim/vim92/syntax/udevconf.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/udevperm.vim b/software/vim/share/vim/vim92/syntax/udevperm.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/udevperm.vim rename to software/vim/share/vim/vim92/syntax/udevperm.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/udevrules.vim b/software/vim/share/vim/vim92/syntax/udevrules.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/udevrules.vim rename to software/vim/share/vim/vim92/syntax/udevrules.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/uil.vim b/software/vim/share/vim/vim92/syntax/uil.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/uil.vim rename to software/vim/share/vim/vim92/syntax/uil.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/updatedb.vim b/software/vim/share/vim/vim92/syntax/updatedb.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/updatedb.vim rename to software/vim/share/vim/vim92/syntax/updatedb.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/upstart.vim b/software/vim/share/vim/vim92/syntax/upstart.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/upstart.vim rename to software/vim/share/vim/vim92/syntax/upstart.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/upstreamdat.vim b/software/vim/share/vim/vim92/syntax/upstreamdat.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/upstreamdat.vim rename to software/vim/share/vim/vim92/syntax/upstreamdat.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/upstreaminstalllog.vim b/software/vim/share/vim/vim92/syntax/upstreaminstalllog.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/upstreaminstalllog.vim rename to software/vim/share/vim/vim92/syntax/upstreaminstalllog.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/upstreamlog.vim b/software/vim/share/vim/vim92/syntax/upstreamlog.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/upstreamlog.vim rename to software/vim/share/vim/vim92/syntax/upstreamlog.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/upstreamrpt.vim b/software/vim/share/vim/vim92/syntax/upstreamrpt.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/upstreamrpt.vim rename to software/vim/share/vim/vim92/syntax/upstreamrpt.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/usserverlog.vim b/software/vim/share/vim/vim92/syntax/usserverlog.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/usserverlog.vim rename to software/vim/share/vim/vim92/syntax/usserverlog.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/usw2kagtlog.vim b/software/vim/share/vim/vim92/syntax/usw2kagtlog.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/usw2kagtlog.vim rename to software/vim/share/vim/vim92/syntax/usw2kagtlog.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/valgrind.vim b/software/vim/share/vim/vim92/syntax/valgrind.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/valgrind.vim rename to software/vim/share/vim/vim92/syntax/valgrind.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/vb.vim b/software/vim/share/vim/vim92/syntax/vb.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/vb.vim rename to software/vim/share/vim/vim92/syntax/vb.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/vdf.vim b/software/vim/share/vim/vim92/syntax/vdf.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/vdf.vim rename to software/vim/share/vim/vim92/syntax/vdf.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/vera.vim b/software/vim/share/vim/vim92/syntax/vera.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/vera.vim rename to software/vim/share/vim/vim92/syntax/vera.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/verilog.vim b/software/vim/share/vim/vim92/syntax/verilog.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/verilog.vim rename to software/vim/share/vim/vim92/syntax/verilog.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/verilogams.vim b/software/vim/share/vim/vim92/syntax/verilogams.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/verilogams.vim rename to software/vim/share/vim/vim92/syntax/verilogams.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/vgrindefs.vim b/software/vim/share/vim/vim92/syntax/vgrindefs.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/vgrindefs.vim rename to software/vim/share/vim/vim92/syntax/vgrindefs.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/vhdl.vim b/software/vim/share/vim/vim92/syntax/vhdl.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/vhdl.vim rename to software/vim/share/vim/vim92/syntax/vhdl.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/vim.vim b/software/vim/share/vim/vim92/syntax/vim.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/vim.vim rename to software/vim/share/vim/vim92/syntax/vim.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/viminfo.vim b/software/vim/share/vim/vim92/syntax/viminfo.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/viminfo.vim rename to software/vim/share/vim/vim92/syntax/viminfo.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/virata.vim b/software/vim/share/vim/vim92/syntax/virata.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/virata.vim rename to software/vim/share/vim/vim92/syntax/virata.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/vmasm.vim b/software/vim/share/vim/vim92/syntax/vmasm.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/vmasm.vim rename to software/vim/share/vim/vim92/syntax/vmasm.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/voscm.vim b/software/vim/share/vim/vim92/syntax/voscm.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/voscm.vim rename to software/vim/share/vim/vim92/syntax/voscm.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/vrml.vim b/software/vim/share/vim/vim92/syntax/vrml.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/vrml.vim rename to software/vim/share/vim/vim92/syntax/vrml.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/vroom.vim b/software/vim/share/vim/vim92/syntax/vroom.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/vroom.vim rename to software/vim/share/vim/vim92/syntax/vroom.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/vsejcl.vim b/software/vim/share/vim/vim92/syntax/vsejcl.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/vsejcl.vim rename to software/vim/share/vim/vim92/syntax/vsejcl.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/vue.vim b/software/vim/share/vim/vim92/syntax/vue.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/vue.vim rename to software/vim/share/vim/vim92/syntax/vue.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/wast.vim b/software/vim/share/vim/vim92/syntax/wast.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/wast.vim rename to software/vim/share/vim/vim92/syntax/wast.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/wdiff.vim b/software/vim/share/vim/vim92/syntax/wdiff.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/wdiff.vim rename to software/vim/share/vim/vim92/syntax/wdiff.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/wdl.vim b/software/vim/share/vim/vim92/syntax/wdl.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/wdl.vim rename to software/vim/share/vim/vim92/syntax/wdl.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/web.vim b/software/vim/share/vim/vim92/syntax/web.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/web.vim rename to software/vim/share/vim/vim92/syntax/web.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/webmacro.vim b/software/vim/share/vim/vim92/syntax/webmacro.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/webmacro.vim rename to software/vim/share/vim/vim92/syntax/webmacro.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/wget.vim b/software/vim/share/vim/vim92/syntax/wget.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/wget.vim rename to software/vim/share/vim/vim92/syntax/wget.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/wget2.vim b/software/vim/share/vim/vim92/syntax/wget2.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/wget2.vim rename to software/vim/share/vim/vim92/syntax/wget2.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/whitespace.vim b/software/vim/share/vim/vim92/syntax/whitespace.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/whitespace.vim rename to software/vim/share/vim/vim92/syntax/whitespace.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/winbatch.vim b/software/vim/share/vim/vim92/syntax/winbatch.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/winbatch.vim rename to software/vim/share/vim/vim92/syntax/winbatch.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/wml.vim b/software/vim/share/vim/vim92/syntax/wml.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/wml.vim rename to software/vim/share/vim/vim92/syntax/wml.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/wsh.vim b/software/vim/share/vim/vim92/syntax/wsh.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/wsh.vim rename to software/vim/share/vim/vim92/syntax/wsh.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/wsml.vim b/software/vim/share/vim/vim92/syntax/wsml.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/wsml.vim rename to software/vim/share/vim/vim92/syntax/wsml.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/wvdial.vim b/software/vim/share/vim/vim92/syntax/wvdial.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/wvdial.vim rename to software/vim/share/vim/vim92/syntax/wvdial.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/xbl.vim b/software/vim/share/vim/vim92/syntax/xbl.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/xbl.vim rename to software/vim/share/vim/vim92/syntax/xbl.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/xdefaults.vim b/software/vim/share/vim/vim92/syntax/xdefaults.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/xdefaults.vim rename to software/vim/share/vim/vim92/syntax/xdefaults.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/xf86conf.vim b/software/vim/share/vim/vim92/syntax/xf86conf.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/xf86conf.vim rename to software/vim/share/vim/vim92/syntax/xf86conf.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/xhtml.vim b/software/vim/share/vim/vim92/syntax/xhtml.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/xhtml.vim rename to software/vim/share/vim/vim92/syntax/xhtml.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/xinetd.vim b/software/vim/share/vim/vim92/syntax/xinetd.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/xinetd.vim rename to software/vim/share/vim/vim92/syntax/xinetd.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/xkb.vim b/software/vim/share/vim/vim92/syntax/xkb.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/xkb.vim rename to software/vim/share/vim/vim92/syntax/xkb.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/xmath.vim b/software/vim/share/vim/vim92/syntax/xmath.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/xmath.vim rename to software/vim/share/vim/vim92/syntax/xmath.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/xml.vim b/software/vim/share/vim/vim92/syntax/xml.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/xml.vim rename to software/vim/share/vim/vim92/syntax/xml.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/xmodmap.vim b/software/vim/share/vim/vim92/syntax/xmodmap.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/xmodmap.vim rename to software/vim/share/vim/vim92/syntax/xmodmap.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/xpm.vim b/software/vim/share/vim/vim92/syntax/xpm.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/xpm.vim rename to software/vim/share/vim/vim92/syntax/xpm.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/xpm2.vim b/software/vim/share/vim/vim92/syntax/xpm2.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/xpm2.vim rename to software/vim/share/vim/vim92/syntax/xpm2.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/xquery.vim b/software/vim/share/vim/vim92/syntax/xquery.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/xquery.vim rename to software/vim/share/vim/vim92/syntax/xquery.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/xs.vim b/software/vim/share/vim/vim92/syntax/xs.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/xs.vim rename to software/vim/share/vim/vim92/syntax/xs.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/xsd.vim b/software/vim/share/vim/vim92/syntax/xsd.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/xsd.vim rename to software/vim/share/vim/vim92/syntax/xsd.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/xslt.vim b/software/vim/share/vim/vim92/syntax/xslt.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/xslt.vim rename to software/vim/share/vim/vim92/syntax/xslt.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/xxd.vim b/software/vim/share/vim/vim92/syntax/xxd.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/xxd.vim rename to software/vim/share/vim/vim92/syntax/xxd.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/yacc.vim b/software/vim/share/vim/vim92/syntax/yacc.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/yacc.vim rename to software/vim/share/vim/vim92/syntax/yacc.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/yaml.vim b/software/vim/share/vim/vim92/syntax/yaml.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/yaml.vim rename to software/vim/share/vim/vim92/syntax/yaml.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/z8a.vim b/software/vim/share/vim/vim92/syntax/z8a.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/z8a.vim rename to software/vim/share/vim/vim92/syntax/z8a.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/zig.vim b/software/vim/share/vim/vim92/syntax/zig.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/zig.vim rename to software/vim/share/vim/vim92/syntax/zig.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/zimbu.vim b/software/vim/share/vim/vim92/syntax/zimbu.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/zimbu.vim rename to software/vim/share/vim/vim92/syntax/zimbu.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/zir.vim b/software/vim/share/vim/vim92/syntax/zir.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/zir.vim rename to software/vim/share/vim/vim92/syntax/zir.vim diff --git a/registry/software/vim/share/vim/vim92/syntax/zsh.vim b/software/vim/share/vim/vim92/syntax/zsh.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/syntax/zsh.vim rename to software/vim/share/vim/vim92/syntax/zsh.vim diff --git a/registry/software/vim/share/vim/vim92/vimrc_example.vim b/software/vim/share/vim/vim92/vimrc_example.vim similarity index 100% rename from registry/software/vim/share/vim/vim92/vimrc_example.vim rename to software/vim/share/vim/vim92/vimrc_example.vim diff --git a/registry/software/vim/src/index.ts b/software/vim/src/index.ts similarity index 100% rename from registry/software/vim/src/index.ts rename to software/vim/src/index.ts diff --git a/software/vim/test/manifest.test.ts b/software/vim/test/manifest.test.ts new file mode 100644 index 0000000000..89139d33b8 --- /dev/null +++ b/software/vim/test/manifest.test.ts @@ -0,0 +1,19 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +const packageDir = new URL("..", import.meta.url).pathname; + +describe("package manifest", () => { + it("declares command binaries", () => { + const manifest = JSON.parse( + readFileSync(join(packageDir, "agentos-package.json"), "utf8"), + ); + + expect(manifest.commands?.length ?? 0).toBeGreaterThan(0); + for (const command of manifest.commands) { + expect(typeof command).toBe("string"); + expect(command.length).toBeGreaterThan(0); + } + }); +}); diff --git a/software/vim/test/vim.test.ts b/software/vim/test/vim.test.ts new file mode 100644 index 0000000000..0fb5ab7724 --- /dev/null +++ b/software/vim/test/vim.test.ts @@ -0,0 +1,115 @@ +import { existsSync } from "node:fs"; +import { cp, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { + NodeFileSystem, + createKernel, + createWasmVmRuntime, + describeIf, +} from "@agentos/test-harness"; +import type { Kernel } from "@agentos/test-harness"; +import { afterEach, expect, it } from "vitest"; + +const VIM_COMMAND_DIR = fileURLToPath(new URL("../bin", import.meta.url)); +const VIM_RUNTIME_DIR = fileURLToPath( + new URL("../dist/package/share/vim/vim92", import.meta.url), +); +const hasVimPackage = existsSync(join(VIM_COMMAND_DIR, "vim")) && + existsSync(join(VIM_RUNTIME_DIR, "defaults.vim")); + +let tempRoot: string | undefined; + +async function writeFixture(path: string, contents: string): Promise { + if (!tempRoot) throw new Error("fixture root not initialized"); + const hostPath = join(tempRoot, path.replace(/^\/+/, "")); + await mkdir(dirname(hostPath), { recursive: true }); + await writeFile(hostPath, contents); +} + +async function createTestVFS(): Promise { + tempRoot = await mkdtemp(join(tmpdir(), "agentos-vim-")); + await writeFixture("/project/input.txt", "alpha\nbeta\ngamma\n"); + await writeFixture( + "/project/edit.vim", + "set nomore\nedit /project/input.txt\n%s/beta/delta/\nwrite\nquitall!\n", + ); + await cp(VIM_RUNTIME_DIR, join(tempRoot, "usr/local/share/vim/vim92"), { + recursive: true, + }); + return new NodeFileSystem({ root: tempRoot }); +} + +describeIf(hasVimPackage, "vim command", { timeout: 60_000 }, () => { + let kernel: Kernel | undefined; + + afterEach(async () => { + await kernel?.dispose(); + kernel = undefined; + if (tempRoot) { + await rm(tempRoot, { recursive: true, force: true }); + tempRoot = undefined; + } + }); + + async function mountFixture(): Promise { + const vfs = await createTestVFS(); + kernel = createKernel({ filesystem: vfs }); + await kernel.mount(createWasmVmRuntime({ commandDirs: [VIM_COMMAND_DIR] })); + } + + async function runVim(args: string[]) { + if (!kernel) throw new Error("kernel not mounted"); + let stdout = ""; + let stderr = ""; + const proc = kernel.spawn("vim", args, { + streamStdin: true, + env: { + TERM: "xterm", + VIM: "/usr/local/share/vim", + VIMRUNTIME: "/usr/local/share/vim/vim92", + }, + onStdout: (chunk) => { + stdout += Buffer.from(chunk).toString("utf8"); + }, + onStderr: (chunk) => { + stderr += Buffer.from(chunk).toString("utf8"); + }, + }); + proc.closeStdin(); + const exitCode = await proc.wait(); + await new Promise((resolve) => setTimeout(resolve, 0)); + return { stdout, stderr, exitCode }; + } + + it("starts the packaged binary and reports Vim features", async () => { + await mountFixture(); + + const result = await runVim(["--version"]); + + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(result.stdout).toContain("VIM - Vi IMproved"); + expect(result.stdout).toContain("-libcall"); + }); + + it("edits and writes a file in Ex mode", async () => { + await mountFixture(); + + const result = await runVim([ + "-Nu", + "NONE", + "-n", + "-es", + "-S", + "/project/edit.vim", + ]); + + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + if (!kernel) throw new Error("kernel not mounted"); + const edited = Buffer.from(await kernel.readFile("/project/input.txt")).toString( + "utf8", + ); + expect(edited).toBe("alpha\ndelta\ngamma\n"); + }); +}); diff --git a/software/vim/tsconfig.json b/software/vim/tsconfig.json new file mode 100644 index 0000000000..03ce790ab7 --- /dev/null +++ b/software/vim/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*"] +} diff --git a/software/wget/agentos-package.json b/software/wget/agentos-package.json new file mode 100644 index 0000000000..e8b8178957 --- /dev/null +++ b/software/wget/agentos-package.json @@ -0,0 +1,12 @@ +{ + "commands": [ + "wget" + ], + "registry": { + "title": "wget", + "description": "GNU Wget file downloader.", + "priority": 55, + "image": "/images/registry/wget.svg", + "category": "networking" + } +} diff --git a/software/wget/native/c/overlay/src/wasi_ssl.c b/software/wget/native/c/overlay/src/wasi_ssl.c new file mode 100644 index 0000000000..fe74f12ab2 --- /dev/null +++ b/software/wget/native/c/overlay/src/wasi_ssl.c @@ -0,0 +1,1063 @@ +/* In-guest TLS backend for GNU Wget on wasm32-wasip1, built on mbedTLS. + + GNU Wget has no upstream mbedTLS backend; its TLS abstraction is the four + functions declared in src/ssl.h (ssl_init, ssl_cleanup, ssl_connect_wget, + ssl_check_certificate). This file implements those four against mbedTLS, + performing a real TLS handshake and X.509 chain + hostname verification + entirely inside the guest -- the sidecar is a dumb ciphertext pipe. + + The already-connected TCP file descriptor handed to ssl_connect_wget is a + real socket carried by the patched wasi-libc sysroot (host_net imports), so + the mbedTLS BIO callbacks simply read()/write() that fd. On success the fd is + registered with Wget's transport layer (fd_register_transport) so that all + subsequent fd_read/fd_write/fd_peek calls flow through the TLS session. + + Trust configuration mirrors Linux Wget: certificates are verified against + /etc/ssl/certs/ca-certificates.crt by default, --ca-certificate + (opt.ca_cert) and --ca-directory (opt.ca_directory) override the trust + anchors, --crl-file (opt.crl_file) adds a revocation list, and + --no-check-certificate (opt.check_cert != CHECK_CERT_ON) downgrades a + verification failure to a warning. A failed handshake or a rejected + certificate makes the relevant function return false, so http.c reports + CONSSLERR / VERIFCERTERR exactly as with the GnuTLS/OpenSSL backends. */ + +#include "wget.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "connect.h" +#include "log.h" +#include "ssl.h" +#include "url.h" +#include "utils.h" + +/* Debian-shaped default trust store, seeded into the VM by the native + bootstrap. Matches curl's compile-time CA bundle default and OpenSSL's + OPENSSLDIR resolution on Debian. */ +#ifndef WASI_TLS_DEFAULT_CA_BUNDLE +# define WASI_TLS_DEFAULT_CA_BUNDLE "/etc/ssl/certs/ca-certificates.crt" +#endif + +/* The TLS transport singleton, defined after the I/O callbacks below. */ +static struct transport_implementation wasi_tls_transport; + +struct wasi_ssl_context +{ + mbedtls_ssl_context ssl; + mbedtls_ssl_config conf; + mbedtls_ssl_session session; + bool have_session; + mbedtls_x509_crt cacert; + mbedtls_x509_crl crl; + bool have_crl; + mbedtls_x509_crt client_cert; + mbedtls_pk_context client_key; + int *ciphersuites; + mbedtls_ctr_drbg_context ctr_drbg; + mbedtls_entropy_context entropy; + int fd; + int last_err; /* last mbedTLS error, for errstr */ + unsigned char *peekbuf; /* buffered-but-unconsumed plaintext */ + int peeklen; + int peekcap; +}; + +/* mbedTLS BIO send/recv over the raw, already-connected socket fd. TLS sockets + are made nonblocking before the handshake so every WANT_READ/WANT_WRITE can + be driven by readiness with Wget's Linux timeout semantics. */ + +static int +wasi_bio_send (void *arg, const unsigned char *buf, size_t len) +{ + struct wasi_ssl_context *ctx = arg; + ssize_t n; + + do + n = write (ctx->fd, buf, len); + while (n < 0 && errno == EINTR); + + if (n >= 0) + return (int) n; + if (errno == EAGAIN || errno == EWOULDBLOCK) + return MBEDTLS_ERR_SSL_WANT_WRITE; + return MBEDTLS_ERR_NET_SEND_FAILED; +} + +static int +wasi_bio_recv (void *arg, unsigned char *buf, size_t len) +{ + struct wasi_ssl_context *ctx = arg; + ssize_t n; + + do + n = read (ctx->fd, buf, len); + while (n < 0 && errno == EINTR); + + if (n >= 0) + return (int) n; + if (errno == EAGAIN || errno == EWOULDBLOCK) + return MBEDTLS_ERR_SSL_WANT_READ; + return MBEDTLS_ERR_NET_RECV_FAILED; +} + +static double +wasi_monotonic_seconds (void) +{ + struct timespec now; + + if (clock_gettime (CLOCK_MONOTONIC, &now) < 0) + return -1; + return (double) now.tv_sec + (double) now.tv_nsec / 1000000000.0; +} + +/* Wait for the readiness mbedTLS requested. TIMEOUT == 0 means no timeout, + matching Wget's command-line timeout semantics (select_fd itself uses zero + as an immediate poll). START makes repeated WANT_* retries share one budget. */ +static int +wasi_wait_fd (int fd, double timeout, double start, int wait_for) +{ + if (timeout == 0) + { + struct pollfd pfd; + int ret; + + pfd.fd = fd; + pfd.events = 0; + pfd.revents = 0; + if (wait_for & WAIT_FOR_READ) + pfd.events |= POLLIN; + if (wait_for & WAIT_FOR_WRITE) + pfd.events |= POLLOUT; + + do + ret = poll (&pfd, 1, -1); + while (ret < 0 && errno == EINTR); + return ret; + } + else + { + double now = wasi_monotonic_seconds (); + double remaining; + int ret; + + if (now < 0) + return -1; + remaining = timeout - (now - start); + if (remaining <= 0) + { + errno = ETIMEDOUT; + return -1; + } + + ret = select_fd (fd, remaining, wait_for); + if (ret == 0) + { + errno = ETIMEDOUT; + return -1; + } + return ret; + } +} + +/* Global init. mbedTLS keeps no process-wide state we must set up, so this is + a no-op that simply reports readiness, like the GnuTLS backend. */ +bool +ssl_init (void) +{ + return true; +} + +void +ssl_cleanup (void) +{ +} + +/* Map --secure-protocol onto the mbedTLS min/max TLS version knobs. mbedTLS + 3.6 only speaks TLS 1.2 and 1.3, so the legacy SSLv3/TLS1.0/1.1 selectors + pin the floor at TLS 1.2 (the lowest still supported), matching how a modern + Wget build behaves. */ + +static void +wasi_cipher_canonical_name (const char *input, char *output, size_t output_size) +{ + const char *p = input; + size_t written = 0; + + if (0 == strncasecmp (p, "TLS", 3)) + { + p += 3; + /* mbedTLS spells TLS 1.3 suites TLS1-3-..., while OpenSSL uses the + standard TLS_AES_... form. Ignore both prefixes for comparison. */ + if (p[0] == '1' && (p[1] == '-' || p[1] == '_') && p[2] == '3') + p += 3; + } + + while (*p && written + 1 < output_size) + { + if (0 == strncasecmp (p, "WITH", 4)) + { + p += 4; + continue; + } + if (isalnum ((unsigned char) *p)) + output[written++] = toupper ((unsigned char) *p); + ++p; + } + output[written] = '\0'; +} + +static bool +wasi_cipher_name_matches (const char *name, const char *selector) +{ + char name_key[160]; + char selector_key[160]; + char rsa_selector_key[164]; + + wasi_cipher_canonical_name (name, name_key, sizeof name_key); + wasi_cipher_canonical_name (selector, selector_key, sizeof selector_key); + if (0 == strcmp (name_key, selector_key)) + return true; + + /* OpenSSL omits the RSA key-exchange prefix in names such as + AES128-GCM-SHA256; mbedTLS uses TLS-RSA-WITH-AES-128-GCM-SHA256. */ + if (strlen (selector_key) + 4 < sizeof rsa_selector_key) + { + strcpy (rsa_selector_key, "RSA"); + strcat (rsa_selector_key, selector_key); + if (0 == strcmp (name_key, rsa_selector_key)) + return true; + } + return false; +} + +static bool +wasi_cipher_is_tls13 (int id) +{ + const char *name = mbedtls_ssl_get_ciphersuite_name (id); + return name && 0 == strncmp (name, "TLS1-3-", 7); +} + +static bool +wasi_cipher_matches_class (int id, const char *selector) +{ + const char *name = mbedtls_ssl_get_ciphersuite_name (id); + const mbedtls_ssl_ciphersuite_t *info = + mbedtls_ssl_ciphersuite_from_id (id); + + if (!name || !info) + return false; + if (0 == strcasecmp (selector, "ALL") + || 0 == strcasecmp (selector, "DEFAULT")) + return true; + if (0 == strcasecmp (selector, "HIGH")) + return mbedtls_ssl_ciphersuite_get_cipher_key_bitlen (info) >= 128 + && !strstr (name, "-NULL-"); + if (0 == strcasecmp (selector, "aNULL")) + return strstr (name, "-ANON-") != NULL; + if (0 == strcasecmp (selector, "eNULL") + || 0 == strcasecmp (selector, "NULL")) + return strstr (name, "-NULL-") != NULL; + if (0 == strcasecmp (selector, "kRSA")) + return 0 == strncmp (name, "TLS-RSA-WITH-", 13); + if (0 == strcasecmp (selector, "aRSA") + || 0 == strcasecmp (selector, "RSA")) + return strstr (name, "-RSA-") != NULL; + if (0 == strcasecmp (selector, "PSK")) + return strstr (name, "-PSK-") != NULL; + if (0 == strcasecmp (selector, "SRP")) + return strstr (name, "-SRP-") != NULL; + if (0 == strcasecmp (selector, "RC4")) + return strstr (name, "-RC4-") != NULL; + if (0 == strcasecmp (selector, "MD5")) + return strstr (name, "-MD5") != NULL; + if (0 == strcasecmp (selector, "3DES")) + return strstr (name, "-3DES-") != NULL; + if (0 == strcasecmp (selector, "DES")) + return strstr (name, "-DES-") != NULL + || strstr (name, "-3DES-") != NULL; + if (0 == strcasecmp (selector, "AES")) + return strstr (name, "-AES-") != NULL; + if (0 == strcasecmp (selector, "AESGCM")) + return strstr (name, "-AES-") != NULL + && strstr (name, "-GCM-") != NULL; + if (0 == strcasecmp (selector, "CHACHA20")) + return strstr (name, "-CHACHA20-") != NULL; + return wasi_cipher_name_matches (name, selector); +} + +static bool +wasi_configure_cipher_policy (struct wasi_ssl_context *ctx, + const char *policy) +{ + const int *available = mbedtls_ssl_list_ciphersuites (); + size_t available_count = 0; + size_t selected_count = 0; + bool *disabled; + char *copy; + char *saveptr = NULL; + char *token; + + while (available[available_count] != 0) + ++available_count; + ctx->ciphersuites = xnmalloc (available_count + 1, + sizeof *ctx->ciphersuites); + disabled = xnmalloc (available_count, sizeof *disabled); + memset (disabled, 0, available_count * sizeof *disabled); + copy = xstrdup (policy); + + for (token = strtok_r (copy, ":, ", &saveptr); + token; + token = strtok_r (NULL, ":, ", &saveptr)) + { + char operation = 0; + bool matched = false; + + if (*token == '!' || *token == '-' || *token == '+') + operation = *token++; + if (*token == '\0' || *token == '@') + goto unsupported; + + if (operation == '+') + { + /* OpenSSL's +selector moves already-enabled suites to the end. */ + size_t keep = 0; + size_t moved = 0; + int *move = xnmalloc (selected_count, sizeof *move); + for (size_t i = 0; i < selected_count; ++i) + if (wasi_cipher_matches_class (ctx->ciphersuites[i], token)) + { + move[moved++] = ctx->ciphersuites[i]; + matched = true; + } + else + ctx->ciphersuites[keep++] = ctx->ciphersuites[i]; + memcpy (ctx->ciphersuites + keep, move, moved * sizeof *move); + selected_count = keep + moved; + xfree (move); + continue; + } + + for (size_t i = 0; i < available_count; ++i) + { + bool already_selected = false; + /* Native OpenSSL Wget applies --ciphers with + SSL_CTX_set_cipher_list(), which intentionally controls only + TLS 1.2-and-older suites. TLS 1.3 suites use a separate OpenSSL + API and remain at their defaults. Preserve that split instead of + allowing a TLS 1.2 policy to disable TLS 1.3 accidentally. */ + if (wasi_cipher_is_tls13 (available[i])) + continue; + if (!wasi_cipher_matches_class (available[i], token)) + continue; + matched = true; + + if (operation == '!' || operation == '-') + { + size_t out = 0; + if (operation == '!') + disabled[i] = true; + for (size_t j = 0; j < selected_count; ++j) + if (ctx->ciphersuites[j] != available[i]) + ctx->ciphersuites[out++] = ctx->ciphersuites[j]; + selected_count = out; + continue; + } + + if (disabled[i]) + continue; + for (size_t j = 0; j < selected_count; ++j) + if (ctx->ciphersuites[j] == available[i]) + already_selected = true; + if (!already_selected) + ctx->ciphersuites[selected_count++] = available[i]; + } + + /* OpenSSL accepts exclusions for algorithms absent from the compiled + backend (for example !RC4 on a modern build). Unknown positive + selectors are errors because otherwise the policy would broaden. */ + if (!matched && operation != '!' && operation != '-') + goto unsupported; + } + + xfree (copy); + xfree (disabled); + if (selected_count == 0) + { + logprintf (LOG_NOTQUIET, + _("Cipher policy %s selects no mbedTLS ciphersuites.\n"), + quote (policy)); + return false; + } + + /* mbedTLS has one ciphersuite configuration API for every TLS version. + Append its default TLS 1.3 suites after validating the OpenSSL-style + pre-TLS-1.3 policy above, matching SSL_CTX_set_cipher_list semantics. */ + for (size_t i = 0; i < available_count; ++i) + if (wasi_cipher_is_tls13 (available[i])) + ctx->ciphersuites[selected_count++] = available[i]; + ctx->ciphersuites[selected_count] = 0; + mbedtls_ssl_conf_ciphersuites (&ctx->conf, ctx->ciphersuites); + return true; + + unsupported: + logprintf (LOG_NOTQUIET, + _("Unsupported mbedTLS cipher policy token '%s' in %s.\n"), + token, quote (policy)); + xfree (copy); + xfree (disabled); + return false; +} + +static bool +wasi_apply_secure_protocol (struct wasi_ssl_context *ctx) +{ + mbedtls_ssl_config *conf = &ctx->conf; + bool cipher_policy_configured = opt.tls_ciphers_string != NULL; + + if (cipher_policy_configured + && !wasi_configure_cipher_policy (ctx, opt.tls_ciphers_string)) + return false; + + switch (opt.secure_protocol) + { + case secure_protocol_tlsv1_3: + mbedtls_ssl_conf_min_tls_version (conf, MBEDTLS_SSL_VERSION_TLS1_3); + mbedtls_ssl_conf_max_tls_version (conf, MBEDTLS_SSL_VERSION_TLS1_3); + return true; + case secure_protocol_tlsv1_2: + /* GNU Wget treats TLSv1_2 as a minimum, not an exact version: a + TLS-1.3-only endpoint must remain reachable. */ + mbedtls_ssl_conf_min_tls_version (conf, MBEDTLS_SSL_VERSION_TLS1_2); + return true; + case secure_protocol_tlsv1: + case secure_protocol_tlsv1_1: + /* Not supported by mbedTLS 3.6; fall back to the lowest available. */ + mbedtls_ssl_conf_min_tls_version (conf, MBEDTLS_SSL_VERSION_TLS1_2); + return true; + case secure_protocol_sslv2: + case secure_protocol_sslv3: + /* A native modern Wget build rejects these unavailable protocols. Do + not silently upgrade an explicitly requested SSLv2/SSLv3 connection + to TLS, which could make a command succeed with different policy. */ + logprintf (LOG_NOTQUIET, + _("mbedTLS does not support requested protocol %s.\n"), + opt.secure_protocol_name); + return false; + case secure_protocol_auto: + default: + /* Library defaults: TLS 1.2 .. 1.3. */ + return true; + case secure_protocol_pfs: + { + const int *available = mbedtls_ssl_list_ciphersuites (); + size_t count = 0; + size_t selected = 0; + + /* Like native OpenSSL Wget, an explicit --ciphers policy overrides + the PFS preset rather than being intersected with it. */ + if (cipher_policy_configured) + return true; + + while (available[count] != 0) + ++count; + ctx->ciphersuites = xnmalloc (count + 1, sizeof *ctx->ciphersuites); + + for (size_t i = 0; i < count; ++i) + { + const char *name = + mbedtls_ssl_get_ciphersuite_name (available[i]); + /* TLS 1.3 always uses (EC)DHE for certificate-authenticated + handshakes. For TLS 1.2, retain only explicitly ephemeral + DHE/ECDHE suites, excluding static RSA/ECDH and plain PSK. */ + if (name + && (0 == strncmp (name, "TLS1-3-", 7) + || strstr (name, "-ECDHE-") + || strstr (name, "-DHE-"))) + ctx->ciphersuites[selected++] = available[i]; + } + ctx->ciphersuites[selected] = 0; + if (selected == 0) + { + logprintf (LOG_NOTQUIET, + _("mbedTLS has no forward-secret ciphersuites enabled.\n")); + return false; + } + mbedtls_ssl_conf_ciphersuites (conf, ctx->ciphersuites); + return true; + } + } +} + +/* Configure the client certificate/key pair accepted by native Wget's + --certificate and --private-key options. If only one path is provided, + native OpenSSL Wget treats it as a combined PEM containing both objects. */ +static int +wasi_load_client_credentials (struct wasi_ssl_context *ctx) +{ + const char *cert_path; + const char *key_path; + int ret; + + if (!opt.cert_file && !opt.private_key) + return 0; + + cert_path = opt.cert_file ? opt.cert_file : opt.private_key; + key_path = opt.private_key ? opt.private_key : opt.cert_file; + + ret = mbedtls_x509_crt_parse_file (&ctx->client_cert, cert_path); + if (ret < 0) + return ret; + + ret = mbedtls_pk_parse_keyfile (&ctx->client_key, key_path, NULL, + mbedtls_ctr_drbg_random, &ctx->ctr_drbg); + if (ret == MBEDTLS_ERR_PK_PASSWORD_REQUIRED) + { + char *password = getpass (_("Enter PEM pass phrase: ")); + if (!password) + return ret; + mbedtls_pk_free (&ctx->client_key); + mbedtls_pk_init (&ctx->client_key); + ret = mbedtls_pk_parse_keyfile (&ctx->client_key, key_path, password, + mbedtls_ctr_drbg_random, &ctx->ctr_drbg); + } + if (ret < 0) + return ret; + + ret = mbedtls_ssl_conf_own_cert (&ctx->conf, &ctx->client_cert, + &ctx->client_key); + return ret; +} + +/* Load trust anchors in the same order as Linux Wget's GnuTLS backend: use the + system trust store unless --ca-directory replaces it, then add the bundle + supplied through --ca-certificate. Returns the number of sources parsed + (>= 0) or a negative mbedTLS error. */ +static int +wasi_load_trust (struct wasi_ssl_context *ctx) +{ + int loaded = 0; + int ret; + + if (opt.ca_directory && opt.ca_directory[0] != '\0') + { + ret = mbedtls_x509_crt_parse_path (&ctx->cacert, opt.ca_directory); + /* parse_path returns the number of files that failed to parse as a + positive value; only a negative value is a hard error. */ + if (ret < 0) + return ret; + loaded += 1; + } + else + { + ret = mbedtls_x509_crt_parse_file (&ctx->cacert, + WASI_TLS_DEFAULT_CA_BUNDLE); + if (ret < 0) + return ret; + loaded += 1; + } + + if (opt.ca_cert) + { + ret = mbedtls_x509_crt_parse_file (&ctx->cacert, opt.ca_cert); + if (ret < 0) + return ret; + loaded += 1; + } + + return loaded; +} + +/* Perform the TLS handshake on FD and, on success, register the TLS transport + so Wget's fd_* helpers use it. CONTINUE_SESSION requests the native Wget + FTPS behavior: resume the control-channel session on a protected data + connection. Returns true on success. */ +bool +ssl_connect_wget (int fd, const char *hostname, int *continue_session) +{ + struct wasi_ssl_context *ctx; + int original_flags = -1; + bool nonblocking = false; + double handshake_start; + int ret; + + DEBUGP (("Initiating SSL handshake (mbedTLS).\n")); + + if (!hostname || hostname[0] == '\0') + { + errno = EINVAL; + logprintf (LOG_NOTQUIET, + _("SSL handshake requires a non-empty server hostname.\n")); + return false; + } + + ctx = xnew0 (struct wasi_ssl_context); + ctx->fd = fd; + + mbedtls_ssl_init (&ctx->ssl); + mbedtls_ssl_config_init (&ctx->conf); + mbedtls_ssl_session_init (&ctx->session); + mbedtls_x509_crt_init (&ctx->cacert); + mbedtls_x509_crl_init (&ctx->crl); + mbedtls_x509_crt_init (&ctx->client_cert); + mbedtls_pk_init (&ctx->client_key); + mbedtls_ctr_drbg_init (&ctx->ctr_drbg); + mbedtls_entropy_init (&ctx->entropy); + + ret = mbedtls_ctr_drbg_seed (&ctx->ctr_drbg, mbedtls_entropy_func, + &ctx->entropy, + (const unsigned char *) "agentos-wget-tls", 16); + if (ret != 0) + goto error; + + /* Load trust anchors. A failure to read the bundle is only fatal when the + user asked us to verify; with --no-check-certificate we proceed with an + empty trust store (verification is skipped in ssl_check_certificate). */ + ret = wasi_load_trust (ctx); + if (ret < 0 && opt.check_cert == CHECK_CERT_ON) + { + char errbuf[128]; + mbedtls_strerror (ret, errbuf, sizeof errbuf); + logprintf (LOG_NOTQUIET, + _("Could not load CA certificates: %s\n"), errbuf); + goto error; + } + + if (opt.crl_file) + { + ret = mbedtls_x509_crl_parse_file (&ctx->crl, opt.crl_file); + if (ret != 0 && opt.check_cert == CHECK_CERT_ON) + { + char errbuf[128]; + mbedtls_strerror (ret, errbuf, sizeof errbuf); + logprintf (LOG_NOTQUIET, + _("Could not load CRL from %s: %s\n"), + opt.crl_file, errbuf); + goto error; + } + if (ret == 0) + ctx->have_crl = true; + } + + ret = mbedtls_ssl_config_defaults (&ctx->conf, MBEDTLS_SSL_IS_CLIENT, + MBEDTLS_SSL_TRANSPORT_STREAM, + MBEDTLS_SSL_PRESET_DEFAULT); + if (ret != 0) + goto error; + + if (!wasi_apply_secure_protocol (ctx)) + goto error; + + ret = wasi_load_client_credentials (ctx); + if (ret != 0) + { + char errbuf[128]; + mbedtls_strerror (ret, errbuf, sizeof errbuf); + logprintf (LOG_NOTQUIET, + _("Could not load TLS client certificate/key: %s\n"), + errbuf); + goto error; + } + + /* Verify optionally: the handshake always completes and records the chain + + hostname result, which ssl_check_certificate reads and turns into a + pass/fail decision honoring opt.check_cert -- the same split the OpenSSL + backend uses (SSL_VERIFY_NONE at handshake, manual check afterwards). */ + mbedtls_ssl_conf_authmode (&ctx->conf, MBEDTLS_SSL_VERIFY_OPTIONAL); + mbedtls_ssl_conf_ca_chain (&ctx->conf, &ctx->cacert, + ctx->have_crl ? &ctx->crl : NULL); + mbedtls_ssl_conf_rng (&ctx->conf, mbedtls_ctr_drbg_random, &ctx->ctr_drbg); + + ret = mbedtls_ssl_setup (&ctx->ssl, &ctx->conf); + if (ret != 0) + goto error; + + if (continue_session) + { + struct wasi_ssl_context *previous = + fd_transport_context (*continue_session); + if (previous && !previous->have_session) + { + /* Export lazily after Wget has finished using the control channel + for commands. Exporting a TLS 1.3 session immediately after the + handshake can precede the post-handshake NewSessionTicket and + disturb application-record processing in mbedTLS 3.6. */ + mbedtls_ssl_session_free (&previous->session); + mbedtls_ssl_session_init (&previous->session); + if (mbedtls_ssl_get_session (&previous->ssl, + &previous->session) == 0) + previous->have_session = true; + } + if (!previous || !previous->have_session + || mbedtls_ssl_set_session (&ctx->ssl, &previous->session) != 0) + { + logprintf (LOG_NOTQUIET, + _("Could not resume TLS session for socket %d.\n"), fd); + goto error; + } + } + + /* SNI + the name checked during verification. Covers DNS and IP-address + SANs (mbedTLS matches an IP literal against iPAddress SAN entries). */ + ret = mbedtls_ssl_set_hostname (&ctx->ssl, hostname); + if (ret != 0) + goto error; + + mbedtls_ssl_set_bio (&ctx->ssl, ctx, wasi_bio_send, wasi_bio_recv, NULL); + + original_flags = fcntl (fd, F_GETFL); + if (original_flags < 0 || fcntl (fd, F_SETFL, original_flags | O_NONBLOCK) < 0) + goto error; + nonblocking = true; + + handshake_start = wasi_monotonic_seconds (); + if (handshake_start < 0) + goto error; + + do + { + ret = mbedtls_ssl_handshake (&ctx->ssl); + if (ret == MBEDTLS_ERR_SSL_WANT_READ) + { + if (wasi_wait_fd (fd, opt.read_timeout, handshake_start, + WAIT_FOR_READ) <= 0) + { + ret = MBEDTLS_ERR_SSL_TIMEOUT; + break; + } + } + else if (ret == MBEDTLS_ERR_SSL_WANT_WRITE) + { + if (wasi_wait_fd (fd, opt.read_timeout, handshake_start, + WAIT_FOR_WRITE) <= 0) + { + ret = MBEDTLS_ERR_SSL_TIMEOUT; + break; + } + } + } + while (ret == MBEDTLS_ERR_SSL_WANT_READ || ret == MBEDTLS_ERR_SSL_WANT_WRITE); + + if (ret != 0) + { + char errbuf[128]; + mbedtls_strerror (ret, errbuf, sizeof errbuf); + DEBUGP (("SSL handshake failed: %s\n", errbuf)); + logprintf (LOG_NOTQUIET, _("SSL handshake failed: %s\n"), errbuf); + goto error; + } + + /* Register FD with Wget's transport layer so fd_read/fd_write/fd_peek use + our TLS callbacks from here on. */ + fd_register_transport (fd, &wasi_tls_transport, ctx); + DEBUGP (("Handshake successful; TLS registered on socket %d\n", fd)); + + return true; + + error: + if (nonblocking) + { + int saved_errno = errno; + if (fcntl (fd, F_SETFL, original_flags) < 0) + logprintf (LOG_NOTQUIET, + _("Failed to restore socket flags after TLS handshake failure: %s\n"), + strerror (errno)); + errno = saved_errno; + } + mbedtls_ssl_free (&ctx->ssl); + mbedtls_ssl_config_free (&ctx->conf); + mbedtls_ssl_session_free (&ctx->session); + mbedtls_x509_crt_free (&ctx->cacert); + mbedtls_x509_crl_free (&ctx->crl); + mbedtls_x509_crt_free (&ctx->client_cert); + mbedtls_pk_free (&ctx->client_key); + mbedtls_ctr_drbg_free (&ctx->ctr_drbg); + mbedtls_entropy_free (&ctx->entropy); + xfree (ctx->ciphersuites); + xfree (ctx); + return false; +} + +/* --- Wget transport implementation over the TLS session --- */ + +static int +wasi_tls_read (int fd, char *buf, int bufsize, void *arg, double timeout) +{ + struct wasi_ssl_context *ctx = arg; + double start; + int ret; + + /* Serve any peeked-but-unconsumed plaintext first. */ + if (ctx->peeklen > 0) + { + int n = ctx->peeklen < bufsize ? ctx->peeklen : bufsize; + memcpy (buf, ctx->peekbuf, n); + if (n < ctx->peeklen) + memmove (ctx->peekbuf, ctx->peekbuf + n, ctx->peeklen - n); + ctx->peeklen -= n; + return n; + } + + if (timeout == -1) + timeout = opt.read_timeout; + start = wasi_monotonic_seconds (); + if (start < 0) + return -1; + + do + { + ret = mbedtls_ssl_read (&ctx->ssl, (unsigned char *) buf, bufsize); + if (ret == MBEDTLS_ERR_SSL_WANT_READ + && wasi_wait_fd (fd, timeout, start, WAIT_FOR_READ) <= 0) + return -1; + if (ret == MBEDTLS_ERR_SSL_WANT_WRITE + && wasi_wait_fd (fd, timeout, start, WAIT_FOR_WRITE) <= 0) + return -1; + } + while (ret == MBEDTLS_ERR_SSL_WANT_READ || ret == MBEDTLS_ERR_SSL_WANT_WRITE); + + if (ret == MBEDTLS_ERR_SSL_PEER_CLOSE_NOTIFY) + return 0; + if (ret < 0) + { + ctx->last_err = ret; + return -1; + } + return ret; +} + +static int +wasi_tls_write (int fd, char *buf, int bufsize, void *arg) +{ + struct wasi_ssl_context *ctx = arg; + int written = 0; + + while (written < bufsize) + { + int ret = mbedtls_ssl_write (&ctx->ssl, + (const unsigned char *) buf + written, + bufsize - written); + if (ret == MBEDTLS_ERR_SSL_WANT_READ) + { + /* Linux Wget's OpenSSL/GnuTLS write callbacks block without applying + the read timeout. Keep the socket nonblocking internally, but wait + indefinitely for the readiness mbedTLS requested. */ + if (wasi_wait_fd (fd, 0, 0, WAIT_FOR_READ) <= 0) + { + ctx->last_err = MBEDTLS_ERR_SSL_TIMEOUT; + return -1; + } + continue; + } + if (ret == MBEDTLS_ERR_SSL_WANT_WRITE) + { + if (wasi_wait_fd (fd, 0, 0, WAIT_FOR_WRITE) <= 0) + { + ctx->last_err = MBEDTLS_ERR_SSL_TIMEOUT; + return -1; + } + continue; + } + if (ret < 0) + { + ctx->last_err = ret; + return -1; + } + if (ret == 0) + { + errno = EIO; + return -1; + } + written += ret; + } + return written; +} + +static int +wasi_tls_poll (int fd, double timeout, int wait_for, void *arg) +{ + struct wasi_ssl_context *ctx = arg; + + if ((wait_for & WAIT_FOR_READ) + && (ctx->peeklen > 0 || mbedtls_ssl_get_bytes_avail (&ctx->ssl) > 0)) + return 1; + if (timeout == -1) + timeout = opt.read_timeout; + return select_fd (fd, timeout, wait_for); +} + +static int +wasi_tls_peek (int fd, char *buf, int bufsize, void *arg, double timeout) +{ + struct wasi_ssl_context *ctx = arg; + double start; + int n; + + /* Mirror recv(MSG_PEEK)/SSL_peek semantics: preview data without consuming + it, returning as soon as *some* data is available -- one TLS record's + worth. Never try to fill BUFSIZE (that would block on a keep-alive + connection once the whole response fits in one record). Wget's + fd_read_hunk consumes each preview and re-peeks for the next record, so + returning one chunk at a time is sufficient and cannot deadlock. + + Data that mbedtls_ssl_read pulls off the wire here is retained in peekbuf + so wasi_tls_read drains it first -- i.e. the "peek" does not lose it. */ + if (ctx->peeklen == 0) + { + int ret; + + if (ctx->peekcap < bufsize) + { + ctx->peekbuf = xrealloc (ctx->peekbuf, bufsize); + ctx->peekcap = bufsize; + } + + if (timeout == -1) + timeout = opt.read_timeout; + start = wasi_monotonic_seconds (); + if (start < 0) + return -1; + + do + { + ret = mbedtls_ssl_read (&ctx->ssl, ctx->peekbuf, bufsize); + if (ret == MBEDTLS_ERR_SSL_WANT_READ + && wasi_wait_fd (fd, timeout, start, WAIT_FOR_READ) <= 0) + return -1; + if (ret == MBEDTLS_ERR_SSL_WANT_WRITE + && wasi_wait_fd (fd, timeout, start, WAIT_FOR_WRITE) <= 0) + return -1; + } + while (ret == MBEDTLS_ERR_SSL_WANT_READ + || ret == MBEDTLS_ERR_SSL_WANT_WRITE); + + if (ret == MBEDTLS_ERR_SSL_PEER_CLOSE_NOTIFY) + return 0; + if (ret < 0) + { + ctx->last_err = ret; + return -1; + } + ctx->peeklen = ret; + } + + n = ctx->peeklen < bufsize ? ctx->peeklen : bufsize; + if (n > 0) + memcpy (buf, ctx->peekbuf, n); + return n; +} + +static const char * +wasi_tls_errstr (int fd _GL_UNUSED, void *arg) +{ + struct wasi_ssl_context *ctx = arg; + static char errbuf[160]; + + if (ctx->last_err == 0) + return NULL; + mbedtls_strerror (ctx->last_err, errbuf, sizeof errbuf); + return errbuf; +} + +static void +wasi_tls_close (int fd, void *arg) +{ + struct wasi_ssl_context *ctx = arg; + + mbedtls_ssl_close_notify (&ctx->ssl); + mbedtls_ssl_free (&ctx->ssl); + mbedtls_ssl_config_free (&ctx->conf); + mbedtls_ssl_session_free (&ctx->session); + mbedtls_x509_crt_free (&ctx->cacert); + mbedtls_x509_crl_free (&ctx->crl); + mbedtls_x509_crt_free (&ctx->client_cert); + mbedtls_pk_free (&ctx->client_key); + mbedtls_ctr_drbg_free (&ctx->ctr_drbg); + mbedtls_entropy_free (&ctx->entropy); + xfree (ctx->ciphersuites); + xfree (ctx->peekbuf); + xfree (ctx); + + close (fd); + DEBUGP (("Closed %d/SSL (mbedTLS)\n", fd)); +} + +static struct transport_implementation wasi_tls_transport = { + wasi_tls_read, wasi_tls_write, wasi_tls_poll, + wasi_tls_peek, wasi_tls_errstr, wasi_tls_close +}; + +/* Verify the peer certificate against the configured trust anchors and check + that it matches HOST. Reads the verify result recorded during the handshake. + Returns false only when verification failed AND the user requested checking + (opt.check_cert == CHECK_CERT_ON); otherwise it warns and returns true, + matching Wget's --no-check-certificate semantics. */ +bool +ssl_check_certificate (int fd, const char *host) +{ + struct wasi_ssl_context *ctx = fd_transport_context (fd); + uint32_t flags; + bool success; + const char *severity = opt.check_cert ? _("ERROR") : _("WARNING"); + + if (!host || host[0] == '\0') + { + logprintf (LOG_NOTQUIET, + _("ERROR: cannot verify a certificate without a server hostname.\n")); + return false; + } + + if (!ctx) + return opt.check_cert != CHECK_CERT_ON; + + flags = mbedtls_ssl_get_verify_result (&ctx->ssl); + success = (flags == 0); + + if (!success) + { + char vbuf[512]; + int n = mbedtls_x509_crt_verify_info (vbuf, sizeof vbuf, " ", flags); + if (n < 0) + { + vbuf[0] = '\0'; + n = 0; + } + + logprintf (LOG_NOTQUIET, + _("%s: cannot verify %s's certificate:\n"), + severity, quote (host)); + if (n > 0) + logprintf (LOG_NOTQUIET, "%s", vbuf); + + if (opt.check_cert == CHECK_CERT_ON) + logprintf (LOG_NOTQUIET, + _("To connect to %s insecurely, use `--no-check-certificate'.\n"), + quote (host)); + } + else + DEBUGP (("X509 certificate successfully verified and matches host %s\n", + quote (host))); + + return opt.check_cert == CHECK_CERT_ON ? success : true; +} + +/* + * vim: tabstop=2 shiftwidth=2 softtabstop=2 + */ diff --git a/software/wget/package.json b/software/wget/package.json new file mode 100644 index 0000000000..8147e9e893 --- /dev/null +++ b/software/wget/package.json @@ -0,0 +1,33 @@ +{ + "name": "@agentos-software/wget", + "version": "0.3.3", + "type": "module", + "license": "Apache-2.0", + "description": "GNU Wget HTTP client for AgentOS VMs", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "files": [ + "dist", + "!dist/package", + "!dist/package.tar" + ], + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "scripts": { + "build": "agentos-toolchain stage --commands-dir ../../toolchain/target/wasm32-wasip1/release/commands --if-missing skip && tsc && agentos-toolchain build", + "check-types": "tsc --noEmit", + "test": "vitest run test/ --passWithNoTests" + }, + "devDependencies": { + "@agentos-software/manifest": "workspace:*", + "@agentos/test-harness": "workspace:*", + "@rivet-dev/agentos-toolchain": "workspace:*", + "@types/node": "^22.10.2", + "typescript": "^5.9.2", + "vitest": "^2.1.9" + } +} diff --git a/registry/software/vix/src/index.ts b/software/wget/src/index.ts similarity index 100% rename from registry/software/vix/src/index.ts rename to software/wget/src/index.ts diff --git a/software/wget/test/wget.test.ts b/software/wget/test/wget.test.ts new file mode 100644 index 0000000000..1d3acbaf64 --- /dev/null +++ b/software/wget/test/wget.test.ts @@ -0,0 +1,813 @@ +import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; +import { execSync } from "node:child_process"; +import { randomBytes } from "node:crypto"; +import { + existsSync, + mkdtempSync, + readFileSync, + rmSync, + unlinkSync, + writeFileSync, +} from "node:fs"; +import { + createServer, + type IncomingMessage, + type Server, + type ServerResponse, +} from "node:http"; +import { + createServer as createHttpsServer, + type Server as HttpsServer, +} from "node:https"; +import { + createServer as createTlsServer, + type Server as TlsServer, +} from "node:tls"; +import { + createServer as createNetServer, + type Server as NetServer, +} from "node:net"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { gzipSync } from "node:zlib"; +import { createWasmVmRuntime } from "@agentos/test-harness"; +import { + allowAll, + C_BUILD_DIR, + COMMANDS_DIR, + createInMemoryFileSystem, + createKernel, + describeIf, + itIf, +} from "@agentos/test-harness"; +import type { Kernel } from "@agentos/test-harness"; + +const WGET_COMMAND_DIRS = [C_BUILD_DIR, COMMANDS_DIR].filter((dir) => + existsSync(dir), +); +const hasWgetBinary = WGET_COMMAND_DIRS.some((dir) => + existsSync(resolve(dir, "wget")), +); +const WGET_EXEC_TIMEOUT_MS = 10_000; + +let hasOpenssl = false; +try { + execSync("openssl version", { stdio: "pipe" }); + hasOpenssl = true; +} catch { + hasOpenssl = false; +} + +// A long, highly compressible payload so the gzipped body is clearly distinct +// from the plaintext (proving wget actually inflated it via zlib). +const COMPRESSION_PAYLOAD = + "agentos-wget-compression " + + "the quick brown fox jumps over the lazy dog. ".repeat(64); + +// Build a real CA and a leaf cert signed by it, with a SAN covering the +// 127.0.0.1 loopback endpoint the tests connect to. This lets wget's mbedTLS +// backend perform genuine chain + hostname verification, exactly like Linux +// wget against a private CA. +function makeCaSignedCert(caCommonName: string): { + caPem: string; + serverKey: string; + serverCert: string; +} { + const dir = mkdtempSync(join(tmpdir(), "wget-ca-")); + try { + execSync( + `openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out "${dir}/ca.key" 2>/dev/null`, + ); + execSync( + `openssl req -x509 -new -key "${dir}/ca.key" -days 3650 -subj "/CN=${caCommonName}" -out "${dir}/ca.crt" 2>/dev/null`, + ); + execSync( + `openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out "${dir}/srv.key" 2>/dev/null`, + ); + execSync( + `openssl req -new -key "${dir}/srv.key" -subj "/CN=localhost" -out "${dir}/srv.csr" 2>/dev/null`, + ); + writeFileSync(`${dir}/ext.cnf`, "subjectAltName=DNS:localhost,IP:127.0.0.1\n"); + execSync( + `openssl x509 -req -in "${dir}/srv.csr" -CA "${dir}/ca.crt" -CAkey "${dir}/ca.key" ` + + `-CAcreateserial -days 3650 -extfile "${dir}/ext.cnf" -out "${dir}/srv.crt" 2>/dev/null`, + ); + return { + caPem: readFileSync(`${dir}/ca.crt`, "utf8"), + serverKey: readFileSync(`${dir}/srv.key`, "utf8"), + serverCert: readFileSync(`${dir}/srv.crt`, "utf8"), + }; + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +function makeMutualTlsCerts(caCommonName: string): { + caPem: string; + serverKey: string; + serverCert: string; + clientKey: string; + clientCert: string; +} { + const dir = mkdtempSync(join(tmpdir(), "wget-mtls-")); + try { + execSync( + `openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out "${dir}/ca.key" 2>/dev/null`, + ); + execSync( + `openssl req -x509 -new -key "${dir}/ca.key" -days 3650 -subj "/CN=${caCommonName}" -out "${dir}/ca.crt" 2>/dev/null`, + ); + for (const [name, commonName, extensions] of [ + ["server", "localhost", "subjectAltName=DNS:localhost,IP:127.0.0.1\nextendedKeyUsage=serverAuth\n"], + ["client", "wget-client", "extendedKeyUsage=clientAuth\n"], + ] as const) { + execSync( + `openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out "${dir}/${name}.key" 2>/dev/null`, + ); + execSync( + `openssl req -new -key "${dir}/${name}.key" -subj "/CN=${commonName}" -out "${dir}/${name}.csr" 2>/dev/null`, + ); + writeFileSync(`${dir}/${name}.cnf`, extensions); + execSync( + `openssl x509 -req -in "${dir}/${name}.csr" -CA "${dir}/ca.crt" -CAkey "${dir}/ca.key" ` + + `-CAcreateserial -days 3650 -extfile "${dir}/${name}.cnf" -out "${dir}/${name}.crt" 2>/dev/null`, + ); + } + return { + caPem: readFileSync(`${dir}/ca.crt`, "utf8"), + serverKey: readFileSync(`${dir}/server.key`, "utf8"), + serverCert: readFileSync(`${dir}/server.crt`, "utf8"), + clientKey: readFileSync(`${dir}/client.key`, "utf8"), + clientCert: readFileSync(`${dir}/client.crt`, "utf8"), + }; + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +function generateSelfSignedCert(): { key: string; cert: string } { + const keyPath = join(tmpdir(), `wget-test-key-${process.pid}-${Date.now()}.pem`); + try { + const key = execSync( + "openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 2>/dev/null", + { encoding: "utf8" }, + ); + writeFileSync(keyPath, key); + const cert = execSync( + `openssl req -new -x509 -key "${keyPath}" -days 1 -subj "/CN=localhost" -addext "subjectAltName=DNS:localhost,IP:127.0.0.1" 2>/dev/null`, + { encoding: "utf8" }, + ); + return { key, cert }; + } finally { + try { + unlinkSync(keyPath); + } catch { + // Best effort cleanup for test temp files. + } + } +} + +describeIf(hasWgetBinary, "wget command", () => { + let kernel: Kernel; + let server: Server; + let selfSignedServer: HttpsServer; + let validHttpsServer: HttpsServer; + let caHttpsServer: HttpsServer; + let mutualTlsServer: HttpsServer; + let handshakeStallServer: NetServer; + let readStallServer: TlsServer; + let ftpsControlServer: TlsServer; + let ftpsDataServer: TlsServer; + let port: number; + let selfSignedPort: number; + let validHttpsPort: number; + let caHttpsPort: number; + let mutualTlsPort: number; + let handshakeStallPort: number; + let readStallPort: number; + let ftpsControlPort: number; + let ftpsDataPort: number; + // CA (PEM) trusted by the seeded /etc/ssl/certs/ca-certificates.crt bundle; + // it signs validHttpsServer's leaf. caOnlyPem signs caHttpsServer's leaf and + // is deliberately NOT in the bundle, so it verifies only via + // --ca-certificate. + let seededCaPem = ""; + let caOnlyPem = ""; + let clientKeyPem = ""; + let clientCertPem = ""; + let mutualCaPem = ""; + let ftpsDataSessionReused = false; + + beforeAll(async () => { + server = createServer((req: IncomingMessage, res: ServerResponse) => { + const url = req.url ?? "/"; + + if (url === "/file.txt") { + res.writeHead(200, { "Content-Type": "text/plain" }); + res.end("downloaded content"); + return; + } + + if (url === "/data.json") { + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ status: "ok" })); + return; + } + + if (url === "/gzip") { + const body = gzipSync(Buffer.from(COMPRESSION_PAYLOAD)); + res.writeHead(200, { + "Content-Type": "text/plain", + "Content-Encoding": "gzip", + "Content-Length": String(body.length), + }); + res.end(body); + return; + } + + if (url === "/redirect") { + res.writeHead(302, { + Location: `http://127.0.0.1:${port}/redirected`, + }); + res.end(); + return; + } + + if (url === "/redirected") { + res.writeHead(200, { "Content-Type": "text/plain" }); + res.end("arrived after redirect"); + return; + } + + res.writeHead(404, { "Content-Type": "text/plain" }); + res.end("not found"); + }); + + await new Promise((resolveListen) => + server.listen(0, "127.0.0.1", resolveListen), + ); + port = (server.address() as import("node:net").AddressInfo).port; + + // Accept TCP but never send a TLS ServerHello. Native Wget applies its + // read timeout while handshaking; the mbedTLS backend must do the same. + handshakeStallServer = createNetServer((socket) => { + setTimeout(() => socket.destroy(), 3_000); + }); + await new Promise((resolveListen) => + handshakeStallServer.listen(0, "127.0.0.1", resolveListen), + ); + handshakeStallPort = ( + handshakeStallServer.address() as import("node:net").AddressInfo + ).port; + + if (hasOpenssl) { + // Self-signed leaf: no chain to any trusted CA -> must fail verify. + const selfSigned = generateSelfSignedCert(); + selfSignedServer = createHttpsServer( + { key: selfSigned.key, cert: selfSigned.cert }, + (req, res) => { + res.writeHead(200, { "Content-Type": "text/plain" }); + res.end("self-signed secure content"); + }, + ); + await new Promise((resolveListen) => + selfSignedServer.listen(0, "127.0.0.1", resolveListen), + ); + selfSignedPort = ( + selfSignedServer.address() as import("node:net").AddressInfo + ).port; + + // Leaf chaining to a CA seeded into the guest's bundle -> verifies + // with no --no-check-certificate / --ca-certificate. + const trusted = makeCaSignedCert("AgentOS Wget Test Root CA"); + seededCaPem = trusted.caPem; + validHttpsServer = createHttpsServer( + { + key: trusted.serverKey, + cert: trusted.serverCert, + minVersion: "TLSv1.3", + }, + (req, res) => { + res.writeHead(200, { "Content-Type": "text/plain" }); + res.end("verified https content"); + }, + ); + await new Promise((resolveListen) => + validHttpsServer.listen(0, "127.0.0.1", resolveListen), + ); + validHttpsPort = ( + validHttpsServer.address() as import("node:net").AddressInfo + ).port; + + // Complete TLS and send a deliberately incomplete fixed-length body. + // A read timeout is an error, not a clean EOF/truncated success. + readStallServer = createTlsServer( + { key: trusted.serverKey, cert: trusted.serverCert }, + (socket) => { + socket.write( + "HTTP/1.1 200 OK\r\nContent-Length: 64\r\nConnection: close\r\n\r\npartial", + ); + setTimeout(() => socket.destroy(), 3_000); + }, + ); + await new Promise((resolveListen) => + readStallServer.listen(0, "127.0.0.1", resolveListen), + ); + readStallPort = ( + readStallServer.address() as import("node:net").AddressInfo + ).port; + + // Leaf whose CA is provided ONLY via --ca-certificate (not in bundle). + const caOnly = makeCaSignedCert("AgentOS Wget Cacert-Only CA"); + caOnlyPem = caOnly.caPem; + caHttpsServer = createHttpsServer( + { + key: caOnly.serverKey, + cert: caOnly.serverCert, + maxVersion: "TLSv1.2", + }, + (req, res) => { + res.writeHead(200, { "Content-Type": "text/plain" }); + res.end("cacert https content"); + }, + ); + await new Promise((resolveListen) => + caHttpsServer.listen(0, "127.0.0.1", resolveListen), + ); + caHttpsPort = ( + caHttpsServer.address() as import("node:net").AddressInfo + ).port; + + const mutual = makeMutualTlsCerts("AgentOS Wget Mutual TLS CA"); + mutualCaPem = mutual.caPem; + clientKeyPem = mutual.clientKey; + clientCertPem = mutual.clientCert; + mutualTlsServer = createHttpsServer( + { + key: mutual.serverKey, + cert: mutual.serverCert, + ca: mutual.caPem, + requestCert: true, + rejectUnauthorized: true, + }, + (_req, res) => { + res.writeHead(200, { "Content-Type": "text/plain" }); + res.end("mutual tls content"); + }, + ); + await new Promise((resolveListen) => + mutualTlsServer.listen(0, "127.0.0.1", resolveListen), + ); + mutualTlsPort = ( + mutualTlsServer.address() as import("node:net").AddressInfo + ).port; + + // Minimal implicit-FTPS endpoint. Control and data listeners share TLS + // ticket keys; the data listener records whether Wget resumed the + // control session, as native Wget does for protected data channels. + const ticketKeys = randomBytes(48); + let releaseData: (() => void) | undefined; + const dataMayFlow = new Promise((resolveData) => { + releaseData = resolveData; + }); + const ftpsPayload = "resumed ftps content\n"; + const ftpsTlsOptions = { + key: trusted.serverKey, + cert: trusted.serverCert, + minVersion: "TLSv1.2" as const, + maxVersion: "TLSv1.2" as const, + ticketKeys, + }; + ftpsDataServer = createTlsServer(ftpsTlsOptions, async (socket) => { + ftpsDataSessionReused = socket.isSessionReused(); + await dataMayFlow; + socket.end(ftpsPayload); + }); + await new Promise((resolveListen) => + ftpsDataServer.listen(0, "127.0.0.1", resolveListen), + ); + ftpsDataPort = ( + ftpsDataServer.address() as import("node:net").AddressInfo + ).port; + + ftpsControlServer = createTlsServer(ftpsTlsOptions, (socket) => { + let buffered = ""; + socket.write("220 AgentOS FTPS ready\r\n"); + socket.on("data", (chunk) => { + buffered += chunk.toString("utf8"); + for (;;) { + const newline = buffered.indexOf("\n"); + if (newline < 0) break; + const line = buffered.slice(0, newline).trim(); + buffered = buffered.slice(newline + 1); + const [command = "", ...args] = line.split(/\s+/); + const argument = args.join(" "); + switch (command.toUpperCase()) { + case "USER": socket.write("331 Password required\r\n"); break; + case "PASS": socket.write("230 Logged in\r\n"); break; + case "SYST": socket.write("215 UNIX Type: L8\r\n"); break; + case "PWD": socket.write('257 "/" is current directory\r\n'); break; + case "TYPE": + case "PBSZ": + case "PROT": + case "OPTS": socket.write("200 OK\r\n"); break; + case "SIZE": socket.write(`213 ${Buffer.byteLength(ftpsPayload)}\r\n`); break; + case "MDTM": socket.write("213 20260714000000\r\n"); break; + case "EPSV": socket.write(`229 Entering Extended Passive Mode (|||${ftpsDataPort}|)\r\n`); break; + case "PASV": socket.write(`227 Entering Passive Mode (127,0,0,1,${Math.floor(ftpsDataPort / 256)},${ftpsDataPort % 256})\r\n`); break; + case "RETR": + socket.write(`150 Opening data connection for ${argument}\r\n`); + releaseData?.(); + setTimeout(() => socket.write("226 Transfer complete\r\n"), 25); + break; + case "QUIT": socket.end("221 Goodbye\r\n"); break; + default: socket.write("200 OK\r\n"); break; + } + } + }); + }); + await new Promise((resolveListen) => + ftpsControlServer.listen(0, "127.0.0.1", resolveListen), + ); + ftpsControlPort = ( + ftpsControlServer.address() as import("node:net").AddressInfo + ).port; + } + }); + + afterAll(async () => { + for (const s of [ + server, + selfSignedServer, + validHttpsServer, + caHttpsServer, + mutualTlsServer, + handshakeStallServer, + readStallServer, + ftpsControlServer, + ftpsDataServer, + ]) { + if (s) { + await new Promise((resolveClose) => s.close(() => resolveClose())); + } + } + }); + + afterEach(async () => { + await kernel?.dispose(); + }); + + async function mountKernel() { + const filesystem = createInMemoryFileSystem(); + kernel = createKernel({ + filesystem, + permissions: allowAll, + loopbackExemptPorts: [ + port, + selfSignedPort, + validHttpsPort, + caHttpsPort, + mutualTlsPort, + handshakeStallPort, + readStallPort, + ftpsControlPort, + ftpsDataPort, + ].filter((p) => typeof p === "number"), + }); + await kernel.mount(createWasmVmRuntime({ commandDirs: WGET_COMMAND_DIRS })); + + // Seed the Debian-shaped trust store the way the native VM bootstrap + // does, so wget's default CA bundle resolves in-guest. Only the + // "trusted" CA is placed here; the cacert-only CA is intentionally + // absent. + if (seededCaPem) { + await filesystem.mkdir("/etc/ssl/certs", { recursive: true }); + await kernel.writeFile("/etc/ssl/certs/ca-certificates.crt", seededCaPem); + } + return filesystem; + } + + it("downloads a file using the URL basename", async () => { + const filesystem = await mountKernel(); + + const result = await kernel.exec(`wget http://127.0.0.1:${port}/file.txt`, { + timeout: WGET_EXEC_TIMEOUT_MS, + }); + + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(await filesystem.readTextFile("/workspace/file.txt")).toBe( + "downloaded content", + ); + }, 15_000); + + it("-O saves to the requested output path", async () => { + const filesystem = await mountKernel(); + + const result = await kernel.exec( + `wget -O /output.txt http://127.0.0.1:${port}/data.json`, + { timeout: WGET_EXEC_TIMEOUT_MS }, + ); + + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(await filesystem.readTextFile("/output.txt")).toContain( + '"status":"ok"', + ); + }, 15_000); + + it("-q suppresses progress output", async () => { + const filesystem = await mountKernel(); + + const result = await kernel.exec( + `wget -q -O /quiet.txt http://127.0.0.1:${port}/file.txt`, + { timeout: WGET_EXEC_TIMEOUT_MS }, + ); + + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(result.stderr).toBe(""); + expect(await filesystem.readTextFile("/quiet.txt")).toBe( + "downloaded content", + ); + }, 15_000); + + it("reports failure for a 404 URL", async () => { + await mountKernel(); + + const result = await kernel.exec( + `wget http://127.0.0.1:${port}/missing.txt`, + { timeout: WGET_EXEC_TIMEOUT_MS }, + ); + + expect(result.exitCode).not.toBe(0); + expect(result.stderr).toMatch(/404|not found|error/i); + }, 15_000); + + it("follows redirects by default", async () => { + const filesystem = await mountKernel(); + + const result = await kernel.exec( + `wget -O /redirected.txt http://127.0.0.1:${port}/redirect`, + { timeout: WGET_EXEC_TIMEOUT_MS }, + ); + + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(await filesystem.readTextFile("/redirected.txt")).toBe( + "arrived after redirect", + ); + }, 15_000); + + it("--version reports the mbedTLS HTTPS backend", async () => { + await mountKernel(); + + const result = await kernel.exec("wget --version", { + timeout: WGET_EXEC_TIMEOUT_MS, + }); + + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(result.stdout).toContain("GNU Wget 1.24.5"); + // Real in-guest TLS: HTTPS is compiled in and the backend is mbedTLS. + expect(result.stdout).toMatch(/\+https/); + expect(result.stdout).toMatch(/ssl\/mbedtls/i); + }, 15_000); + + it("--compression=auto inflates a gzip response body", async () => { + const filesystem = await mountKernel(); + + const result = await kernel.exec( + `wget --compression=auto -O /gz.txt http://127.0.0.1:${port}/gzip`, + { timeout: WGET_EXEC_TIMEOUT_MS }, + ); + + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(await filesystem.readTextFile("/gz.txt")).toBe(COMPRESSION_PAYLOAD); + }, 15_000); + + it("times out a stalled TLS handshake instead of hanging", async () => { + await mountKernel(); + + const result = await kernel.exec( + `wget --tries=1 --connect-timeout=1 --read-timeout=1 --no-check-certificate -O /handshake-timeout.txt https://127.0.0.1:${handshakeStallPort}/`, + { timeout: WGET_EXEC_TIMEOUT_MS }, + ); + + expect(result.exitCode).not.toBe(0); + expect(result.stderr).toMatch(/timed out|timeout/i); + }, 15_000); + + itIf( + hasOpenssl, + "treats a stalled HTTPS response body as a timeout, not clean EOF", + async () => { + await mountKernel(); + + const result = await kernel.exec( + `wget --tries=1 --read-timeout=1 -O /truncated.txt https://127.0.0.1:${readStallPort}/`, + { timeout: WGET_EXEC_TIMEOUT_MS }, + ); + + expect(result.exitCode).not.toBe(0); + expect(result.stderr).toMatch(/timed out|timeout/i); + }, + 15_000, + ); + + itIf(hasOpenssl, "downloads over HTTPS verifying against the seeded CA bundle", async () => { + const filesystem = await mountKernel(); + + // No --no-check-certificate, no --ca-certificate: trust comes solely + // from the seeded /etc/ssl/certs/ca-certificates.crt, like Debian wget. + const result = await kernel.exec( + `wget -O /secure.txt https://127.0.0.1:${validHttpsPort}/file`, + { timeout: WGET_EXEC_TIMEOUT_MS }, + ); + + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(await filesystem.readTextFile("/secure.txt")).toBe( + "verified https content", + ); + }, 15_000); + + itIf(hasOpenssl, "fails with a real cert error on an untrusted (self-signed) server", async () => { + await mountKernel(); + + const result = await kernel.exec( + `wget -O /nope.txt https://127.0.0.1:${selfSignedPort}/file`, + { timeout: WGET_EXEC_TIMEOUT_MS }, + ); + + // VERIFCERTERR -> WGET_EXIT_SSL_AUTH_FAIL == 5, the native taxonomy. + expect(result.exitCode).toBe(5); + expect(result.stderr).toMatch(/cannot verify|certificate|not trusted/i); + }, 15_000); + + itIf(hasOpenssl, "--no-check-certificate accepts a self-signed server", async () => { + const filesystem = await mountKernel(); + + const result = await kernel.exec( + `wget --no-check-certificate -O /insecure.txt https://127.0.0.1:${selfSignedPort}/file`, + { timeout: WGET_EXEC_TIMEOUT_MS }, + ); + + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(await filesystem.readTextFile("/insecure.txt")).toBe( + "self-signed secure content", + ); + }, 15_000); + + itIf(hasOpenssl, "--ca-certificate trusts a server signed by that CA", async () => { + const filesystem = await mountKernel(); + + // caHttpsServer's CA is NOT in the seeded bundle, so this only passes if + // --ca-certificate is honored (real file read + chain build in-guest). + await kernel.writeFile("/tmp/cacert-only.pem", caOnlyPem); + const result = await kernel.exec( + `wget --ca-certificate=/tmp/cacert-only.pem -O /cacert.txt https://127.0.0.1:${caHttpsPort}/file`, + { timeout: WGET_EXEC_TIMEOUT_MS }, + ); + + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(await filesystem.readTextFile("/cacert.txt")).toBe( + "cacert https content", + ); + }, 15_000); + + itIf(hasOpenssl, "--ca-certificate augments rather than replaces system trust", async () => { + const filesystem = await mountKernel(); + + // Linux Wget loads system trust first, then adds --ca-certificate. An + // unrelated supplemental CA must not make an otherwise trusted server + // fail verification. + await kernel.writeFile("/tmp/additional-ca.pem", caOnlyPem); + const result = await kernel.exec( + `wget --ca-certificate=/tmp/additional-ca.pem -O /system-trust.txt https://127.0.0.1:${validHttpsPort}/file`, + { timeout: WGET_EXEC_TIMEOUT_MS }, + ); + + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(await filesystem.readTextFile("/system-trust.txt")).toBe( + "verified https content", + ); + }, 15_000); + + itIf(hasOpenssl, "--ca-certificate with the wrong CA still fails verification", async () => { + await mountKernel(); + + // Point --ca-certificate at the seeded CA, which did NOT sign + // caHttpsServer's leaf. + await kernel.writeFile("/tmp/wrong-ca.pem", seededCaPem); + const result = await kernel.exec( + `wget --ca-certificate=/tmp/wrong-ca.pem -O /wrong.txt https://127.0.0.1:${caHttpsPort}/file`, + { timeout: WGET_EXEC_TIMEOUT_MS }, + ); + + expect(result.exitCode).toBe(5); + expect(result.stderr).toMatch(/cannot verify|certificate|not trusted/i); + }, 15_000); + + itIf(hasOpenssl, "--secure-protocol=TLSv1_2 remains a minimum and permits TLS 1.3", async () => { + const filesystem = await mountKernel(); + const result = await kernel.exec( + `wget --secure-protocol=TLSv1_2 -O /tls13.txt https://127.0.0.1:${validHttpsPort}/file`, + { timeout: WGET_EXEC_TIMEOUT_MS }, + ); + + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(await filesystem.readTextFile("/tls13.txt")).toBe( + "verified https content", + ); + }, 15_000); + + itIf(hasOpenssl, "rejects unavailable SSLv3 instead of silently upgrading to TLS", async () => { + await mountKernel(); + const result = await kernel.exec( + `wget --secure-protocol=SSLv3 -O /old.txt https://127.0.0.1:${validHttpsPort}/file`, + { timeout: WGET_EXEC_TIMEOUT_MS }, + ); + + expect(result.exitCode).not.toBe(0); + expect(result.stderr).toMatch(/does not support requested protocol|SSLv3/i); + }, 15_000); + + itIf(hasOpenssl, "honors common OpenSSL HIGH/exclusion cipher policy syntax", async () => { + const filesystem = await mountKernel(); + const result = await kernel.exec( + `wget --ciphers='HIGH:!aNULL:!RC4:!MD5:!SRP:!PSK' -O /cipher.txt https://127.0.0.1:${validHttpsPort}/file`, + { timeout: WGET_EXEC_TIMEOUT_MS }, + ); + + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(await filesystem.readTextFile("/cipher.txt")).toBe( + "verified https content", + ); + }, 15_000); + + itIf(hasOpenssl, "translates an explicit OpenSSL TLS 1.2 cipher name", async () => { + const filesystem = await mountKernel(); + await kernel.writeFile("/tmp/cipher-ca.pem", caOnlyPem); + const result = await kernel.exec( + `wget --ciphers=ECDHE-RSA-AES128-GCM-SHA256 --ca-certificate=/tmp/cipher-ca.pem ` + + `-O /explicit-cipher.txt https://127.0.0.1:${caHttpsPort}/file`, + { timeout: WGET_EXEC_TIMEOUT_MS }, + ); + + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(await filesystem.readTextFile("/explicit-cipher.txt")).toBe( + "cacert https content", + ); + }, 15_000); + + itIf(hasOpenssl, "leaves TLS 1.3 enabled when --ciphers names a TLS 1.2 suite", async () => { + const filesystem = await mountKernel(); + const result = await kernel.exec( + `wget --ciphers=ECDHE-RSA-AES128-GCM-SHA256 -O /tls13-cipher.txt ` + + `https://127.0.0.1:${validHttpsPort}/file`, + { timeout: WGET_EXEC_TIMEOUT_MS }, + ); + + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(await filesystem.readTextFile("/tls13-cipher.txt")).toBe( + "verified https content", + ); + }, 15_000); + + itIf(hasOpenssl, "fails explicitly for an unsupported cipher policy token", async () => { + await mountKernel(); + const result = await kernel.exec( + `wget --ciphers=NOT-A-CIPHER -O /bad-cipher.txt https://127.0.0.1:${validHttpsPort}/file`, + { timeout: WGET_EXEC_TIMEOUT_MS }, + ); + + expect(result.exitCode).not.toBe(0); + expect(result.stderr).toMatch(/unsupported.*cipher policy token|NOT-A-CIPHER/i); + }, 15_000); + + itIf(hasOpenssl, "presents --certificate and --private-key to a mutual-TLS server", async () => { + const filesystem = await mountKernel(); + await kernel.writeFile("/tmp/mutual-ca.pem", mutualCaPem); + await kernel.writeFile("/tmp/client.crt", clientCertPem); + await kernel.writeFile("/tmp/client.key", clientKeyPem); + const result = await kernel.exec( + `wget --ca-certificate=/tmp/mutual-ca.pem --certificate=/tmp/client.crt ` + + `--private-key=/tmp/client.key -O /mutual.txt https://127.0.0.1:${mutualTlsPort}/file`, + { timeout: WGET_EXEC_TIMEOUT_MS }, + ); + + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(await filesystem.readTextFile("/mutual.txt")).toBe( + "mutual tls content", + ); + }, 15_000); + + itIf(hasOpenssl, "resumes the FTPS control session on the protected data channel", async () => { + const filesystem = await mountKernel(); + const result = await kernel.exec( + `wget --ftps-implicit -O /ftps.txt ftps://127.0.0.1:${ftpsControlPort}/file.txt`, + { timeout: WGET_EXEC_TIMEOUT_MS }, + ); + + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(await filesystem.readTextFile("/ftps.txt")).toBe( + "resumed ftps content\n", + ); + expect(ftpsDataSessionReused).toBe(true); + }, 20_000); +}); diff --git a/software/wget/tsconfig.json b/software/wget/tsconfig.json new file mode 100644 index 0000000000..db523bef56 --- /dev/null +++ b/software/wget/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*"] +} diff --git a/software/yq/agentos-package.json b/software/yq/agentos-package.json new file mode 100644 index 0000000000..d6de19566d --- /dev/null +++ b/software/yq/agentos-package.json @@ -0,0 +1,11 @@ +{ + "commands": [ + "yq" + ], + "registry": { + "title": "yq", + "description": "YAML/JSON processor.", + "priority": 4, + "category": "data" + } +} diff --git a/software/yq/native/crates/cmd-yq/Cargo.toml b/software/yq/native/crates/cmd-yq/Cargo.toml new file mode 100644 index 0000000000..dfa19879f9 --- /dev/null +++ b/software/yq/native/crates/cmd-yq/Cargo.toml @@ -0,0 +1,14 @@ +[package] +workspace = "../../../../../toolchain" +name = "cmd-yq" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "yq standalone binary for secure-exec VM" + +[[bin]] +name = "yq" +path = "src/main.rs" + +[dependencies] +secureexec-yq = { path = "../yq" } diff --git a/registry/native/crates/commands/yq/src/main.rs b/software/yq/native/crates/cmd-yq/src/main.rs similarity index 100% rename from registry/native/crates/commands/yq/src/main.rs rename to software/yq/native/crates/cmd-yq/src/main.rs diff --git a/software/yq/native/crates/yq/Cargo.toml b/software/yq/native/crates/yq/Cargo.toml new file mode 100644 index 0000000000..5712402a21 --- /dev/null +++ b/software/yq/native/crates/yq/Cargo.toml @@ -0,0 +1,16 @@ +[package] +workspace = "../../../../../toolchain" +name = "secureexec-yq" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "yq (YAML/XML/TOML/JSON processor) implementation for secure-exec standalone binaries" + +[dependencies] +jaq-core = "2.2" +jaq-std = "2.1" +jaq-json = { version = "1.1", features = ["serde_json"] } +serde_json = "1" +serde_yaml = "0.9" +toml = "0.8" +quick-xml = "0.37" diff --git a/software/yq/native/crates/yq/src/lib.rs b/software/yq/native/crates/yq/src/lib.rs new file mode 100644 index 0000000000..e5332476bd --- /dev/null +++ b/software/yq/native/crates/yq/src/lib.rs @@ -0,0 +1,1049 @@ +//! yq — YAML/XML/TOML/JSON processor using jaq filter engine. +//! +//! Converts input to JSON, runs jaq filter, converts output to requested format. +//! Reuses jaq-core/jaq-std/jaq-json (same engine as jq command). + +use std::ffi::OsString; +use std::fmt; +use std::fs::File as FsFile; +use std::io::{self, Read, Write}; + +use jaq_core::load::{Arena, File, Loader}; +use jaq_core::{Compiler, Ctx, RcIter}; +use jaq_json::Val; + +const MAX_INPUT_BYTES: usize = 16 * 1024 * 1024; +const MAX_FORMATTED_OUTPUT_BYTES: usize = 16 * 1024 * 1024; +const MAX_OUTPUT_VALUES: usize = 100_000; +const MAX_XML_DEPTH: usize = 256; +const MAX_XML_NODES: usize = 100_000; +const MAX_XML_ATTRIBUTES_PER_ELEMENT: usize = 4096; +const MAX_XML_TEXT_BYTES: usize = 16 * 1024 * 1024; + +#[derive(Clone, Copy, PartialEq)] +enum Format { + Yaml, + Json, + Toml, + Xml, +} + +struct YqOptions { + filter: String, + input_format: Option, + output_format: Option, + raw_output: bool, + compact: bool, + null_input: bool, + slurp: bool, + input_paths: Vec, +} + +/// Entry point for yq command. +pub fn main(args: Vec) -> i32 { + let str_args: Vec = args + .iter() + .skip(1) + .map(|a| a.to_string_lossy().to_string()) + .collect(); + + match run_yq(&str_args) { + Ok(code) => code, + Err(msg) => { + eprintln!("yq: {}", msg); + 2 + } + } +} + +fn parse_format(s: &str) -> Result { + match s { + "yaml" | "y" => Ok(Format::Yaml), + "json" | "j" => Ok(Format::Json), + "toml" | "t" => Ok(Format::Toml), + "xml" | "x" => Ok(Format::Xml), + _ => Err(format!( + "unknown format: {} (expected yaml, json, toml, xml)", + s + )), + } +} + +fn parse_args(args: &[String]) -> Result { + let mut opts = YqOptions { + filter: String::new(), + input_format: None, + output_format: None, + raw_output: false, + compact: false, + null_input: false, + slurp: false, + input_paths: Vec::new(), + }; + + let mut filter_set = false; + let mut i = 0; + + while i < args.len() { + let arg = &args[i]; + + if arg == "--" { + let remaining = &args[i + 1..]; + if !filter_set { + if let Some(filter) = remaining.first() { + opts.filter = filter.clone(); + filter_set = true; + opts.input_paths.extend(remaining.iter().skip(1).cloned()); + } + } else { + opts.input_paths.extend(remaining.iter().cloned()); + } + break; + } + + if arg == "-p" || arg == "--input-format" { + i += 1; + if i >= args.len() { + return Err("-p requires a format argument".to_string()); + } + opts.input_format = Some(parse_format(&args[i])?); + } else if arg == "-o" || arg == "--output-format" { + i += 1; + if i >= args.len() { + return Err("-o requires a format argument".to_string()); + } + opts.output_format = Some(parse_format(&args[i])?); + } else if arg == "-r" || arg == "--raw-output" { + opts.raw_output = true; + } else if arg == "-c" || arg == "--compact-output" { + opts.compact = true; + } else if arg == "-n" || arg == "--null-input" { + opts.null_input = true; + } else if arg == "-s" || arg == "--slurp" { + opts.slurp = true; + } else if arg.starts_with('-') && arg.len() > 1 && !arg.starts_with("--") { + // Combined short flags like -rc + let flags = &arg[1..]; + let mut chars = flags.chars().peekable(); + while let Some(c) = chars.next() { + match c { + 'r' => opts.raw_output = true, + 'c' => opts.compact = true, + 'n' => opts.null_input = true, + 's' => opts.slurp = true, + 'p' => { + let rest: String = chars.collect(); + if !rest.is_empty() { + opts.input_format = Some(parse_format(&rest)?); + } else { + i += 1; + if i >= args.len() { + return Err("-p requires a format argument".to_string()); + } + opts.input_format = Some(parse_format(&args[i])?); + } + break; + } + 'o' => { + let rest: String = chars.collect(); + if !rest.is_empty() { + opts.output_format = Some(parse_format(&rest)?); + } else { + i += 1; + if i >= args.len() { + return Err("-o requires a format argument".to_string()); + } + opts.output_format = Some(parse_format(&args[i])?); + } + break; + } + _ => return Err(format!("unknown option: -{}", c)), + } + } + } else if !filter_set { + opts.filter = arg.clone(); + filter_set = true; + } else { + opts.input_paths.push(arg.clone()); + } + + i += 1; + } + + if !filter_set { + opts.filter = ".".to_string(); + } + + Ok(opts) +} + +fn detect_format(input: &str) -> Format { + let trimmed = input.trim_start(); + + // JSON: starts with { or [ + if trimmed.starts_with('{') || trimmed.starts_with('[') { + if serde_json::from_str::(trimmed).is_ok() { + return Format::Json; + } + } + + // XML: starts with < (including (trimmed).is_ok() + { + return Format::Toml; + } + } + + // Default: YAML + Format::Yaml +} + +fn parse_input(input: &str, format: Format) -> Result { + match format { + Format::Json => serde_json::from_str(input).map_err(|e| format!("invalid JSON: {}", e)), + Format::Yaml => serde_yaml::from_str(input).map_err(|e| format!("invalid YAML: {}", e)), + Format::Toml => { + let toml_val: toml::Value = + toml::from_str(input).map_err(|e| format!("invalid TOML: {}", e))?; + toml_to_json(toml_val) + } + Format::Xml => xml_to_json(input), + } +} + +fn toml_to_json(val: toml::Value) -> Result { + Ok(match val { + toml::Value::String(s) => serde_json::Value::String(s), + toml::Value::Integer(i) => serde_json::Value::Number(serde_json::Number::from(i)), + toml::Value::Float(f) => serde_json::Number::from_f64(f) + .map(serde_json::Value::Number) + .unwrap_or(serde_json::Value::Null), + toml::Value::Boolean(b) => serde_json::Value::Bool(b), + toml::Value::Datetime(dt) => serde_json::Value::String(dt.to_string()), + toml::Value::Array(arr) => { + let items: Result, _> = arr.into_iter().map(toml_to_json).collect(); + serde_json::Value::Array(items?) + } + toml::Value::Table(table) => { + let mut map = serde_json::Map::new(); + for (k, v) in table { + map.insert(k, toml_to_json(v)?); + } + serde_json::Value::Object(map) + } + }) +} + +fn json_to_toml(val: &serde_json::Value) -> Result { + Ok(match val { + serde_json::Value::Null => return Err("TOML does not support null values".to_string()), + serde_json::Value::Bool(b) => toml::Value::Boolean(*b), + serde_json::Value::Number(n) => { + if let Some(i) = n.as_i64() { + toml::Value::Integer(i) + } else if let Some(f) = n.as_f64() { + toml::Value::Float(f) + } else { + return Err("unsupported number for TOML".to_string()); + } + } + serde_json::Value::String(s) => toml::Value::String(s.clone()), + serde_json::Value::Array(arr) => { + let items: Result, _> = arr.iter().map(json_to_toml).collect(); + toml::Value::Array(items?) + } + serde_json::Value::Object(map) => { + let mut table = toml::map::Map::new(); + for (k, v) in map { + table.insert(k.clone(), json_to_toml(v)?); + } + toml::Value::Table(table) + } + }) +} + +// --- XML parsing --- + +fn xml_to_json(input: &str) -> Result { + use quick_xml::events::Event; + use quick_xml::Reader; + + struct StackEntry { + name: String, + children: serde_json::Map, + text: String, + } + + let mut reader = Reader::from_str(input); + let mut stack: Vec = Vec::new(); + let mut root = serde_json::Map::new(); + let mut nodes = 0usize; + + loop { + match reader.read_event() { + Ok(Event::Start(ref e)) => { + count_xml_node(&mut nodes)?; + if stack.len() >= MAX_XML_DEPTH { + return Err("XML exceeds maximum nesting depth".to_string()); + } + let name = String::from_utf8_lossy(e.name().as_ref()).to_string(); + let mut children = serde_json::Map::new(); + let mut attr_count = 0usize; + for attr in e.attributes() { + let attr = attr.map_err(|e| format!("invalid XML attribute: {}", e))?; + attr_count += 1; + if attr_count > MAX_XML_ATTRIBUTES_PER_ELEMENT { + return Err("XML element has too many attributes".to_string()); + } + let key = format!("@{}", String::from_utf8_lossy(attr.key.as_ref())); + let val = String::from_utf8_lossy(&attr.value).to_string(); + children.insert(key, serde_json::Value::String(val)); + } + stack.push(StackEntry { + name, + children, + text: String::new(), + }); + } + Ok(Event::End(_)) => { + let entry = stack.pop().ok_or("unexpected closing tag")?; + let text = entry.text.trim().to_string(); + + let value = if entry.children.is_empty() && text.is_empty() { + serde_json::Value::Null + } else if entry.children.is_empty() { + serde_json::Value::String(text) + } else { + let mut obj = entry.children; + if !text.is_empty() { + obj.insert("#text".to_string(), serde_json::Value::String(text)); + } + serde_json::Value::Object(obj) + }; + + let target = if let Some(parent) = stack.last_mut() { + &mut parent.children + } else { + &mut root + }; + + insert_or_array(target, entry.name, value); + } + Ok(Event::Empty(ref e)) => { + count_xml_node(&mut nodes)?; + if stack.len() >= MAX_XML_DEPTH { + return Err("XML exceeds maximum nesting depth".to_string()); + } + let name = String::from_utf8_lossy(e.name().as_ref()).to_string(); + let mut attrs = serde_json::Map::new(); + let mut attr_count = 0usize; + for attr in e.attributes() { + let attr = attr.map_err(|e| format!("invalid XML attribute: {}", e))?; + attr_count += 1; + if attr_count > MAX_XML_ATTRIBUTES_PER_ELEMENT { + return Err("XML element has too many attributes".to_string()); + } + let key = format!("@{}", String::from_utf8_lossy(attr.key.as_ref())); + let val = String::from_utf8_lossy(&attr.value).to_string(); + attrs.insert(key, serde_json::Value::String(val)); + } + + let value = if attrs.is_empty() { + serde_json::Value::Null + } else { + serde_json::Value::Object(attrs) + }; + + let target = if let Some(parent) = stack.last_mut() { + &mut parent.children + } else { + &mut root + }; + + insert_or_array(target, name, value); + } + Ok(Event::Text(ref e)) => { + if let Some(entry) = stack.last_mut() { + let text = e + .unescape() + .map_err(|e| format!("invalid XML text: {}", e))?; + let next_len = entry + .text + .len() + .checked_add(text.len()) + .ok_or("XML text length overflowed")?; + if next_len > MAX_XML_TEXT_BYTES { + return Err("XML text exceeds size limit".to_string()); + } + entry.text.push_str(&text); + } + } + Ok(Event::Eof) => break, + Ok(_) => {} // Skip PI, Comment, Decl, DocType, CData + Err(e) => return Err(format!("invalid XML: {}", e)), + } + } + + if !stack.is_empty() { + return Err("unexpected end of XML input".to_string()); + } + + Ok(serde_json::Value::Object(root)) +} + +fn count_xml_node(nodes: &mut usize) -> Result<(), String> { + *nodes = nodes.checked_add(1).ok_or("XML node count overflowed")?; + if *nodes > MAX_XML_NODES { + return Err("XML contains too many nodes".to_string()); + } + Ok(()) +} + +fn record_output_value(output_count: &mut usize) -> Result<(), String> { + *output_count = output_count + .checked_add(1) + .ok_or("output count overflowed")?; + if *output_count > MAX_OUTPUT_VALUES { + return Err("too many output values".to_string()); + } + Ok(()) +} + +fn read_limited_string(reader: R) -> Result { + read_limited_source(reader, "stdin", MAX_INPUT_BYTES) +} + +fn read_limited_source(reader: R, label: &str, limit: usize) -> Result { + let mut input = String::new(); + reader + .take((limit + 1) as u64) + .read_to_string(&mut input) + .map_err(|e| format!("failed to read {}: {}", label, e))?; + if input.len() > limit { + return Err(format!("{} exceeds size limit", label)); + } + Ok(input) +} + +fn read_sources(opts: &YqOptions) -> Result, String> { + if opts.input_paths.is_empty() { + return Ok(vec![read_limited_string(io::stdin())?]); + } + + let mut total = 0usize; + let mut sources = Vec::new(); + for path in &opts.input_paths { + let remaining = MAX_INPUT_BYTES.saturating_sub(total); + let data = if path == "-" { + read_limited_source(io::stdin(), "stdin", remaining)? + } else { + let file = FsFile::open(path).map_err(|e| format!("failed to open {}: {}", path, e))?; + read_limited_source(file, path, remaining)? + }; + total = total + .checked_add(data.len()) + .ok_or_else(|| "input exceeds size limit".to_string())?; + if total > MAX_INPUT_BYTES { + return Err("input exceeds size limit".to_string()); + } + sources.push(data); + } + Ok(sources) +} + +fn insert_or_array( + map: &mut serde_json::Map, + key: String, + value: serde_json::Value, +) { + if let Some(existing) = map.get_mut(&key) { + match existing { + serde_json::Value::Array(arr) => arr.push(value), + _ => { + let prev = existing.clone(); + *existing = serde_json::Value::Array(vec![prev, value]); + } + } + } else { + map.insert(key, value); + } +} + +// --- XML output --- + +fn json_to_xml(val: &serde_json::Value) -> Result { + use quick_xml::Writer; + + let mut writer = Writer::new(LimitedBytes::new(MAX_FORMATTED_OUTPUT_BYTES)); + + match val { + serde_json::Value::Object(map) => { + for (key, value) in map { + write_xml_element(&mut writer, key, value) + .map_err(|e| format!("XML write error: {}", e))?; + } + } + _ => { + write_xml_element(&mut writer, "root", val) + .map_err(|e| format!("XML write error: {}", e))?; + } + } + + writer + .into_inner() + .into_string() + .map_err(|e| format!("XML encoding error: {}", e)) +} + +fn write_xml_element( + writer: &mut quick_xml::Writer, + name: &str, + val: &serde_json::Value, +) -> Result<(), quick_xml::Error> { + use quick_xml::events::{BytesEnd, BytesStart, BytesText, Event}; + + match val { + serde_json::Value::Object(map) => { + let mut elem = BytesStart::new(name); + for (k, v) in map { + if let Some(attr_name) = k.strip_prefix('@') { + if let serde_json::Value::String(s) = v { + elem.push_attribute((attr_name, s.as_str())); + } + } + } + writer.write_event(Event::Start(elem))?; + + if let Some(serde_json::Value::String(text)) = map.get("#text") { + writer.write_event(Event::Text(BytesText::new(text)))?; + } + + for (k, v) in map { + if k.starts_with('@') || k == "#text" { + continue; + } + match v { + serde_json::Value::Array(arr) => { + for item in arr { + write_xml_element(writer, k, item)?; + } + } + _ => write_xml_element(writer, k, v)?, + } + } + + writer.write_event(Event::End(BytesEnd::new(name)))?; + } + serde_json::Value::Array(arr) => { + for item in arr { + write_xml_element(writer, name, item)?; + } + } + serde_json::Value::String(s) => { + writer.write_event(Event::Start(BytesStart::new(name)))?; + writer.write_event(Event::Text(BytesText::new(s)))?; + writer.write_event(Event::End(BytesEnd::new(name)))?; + } + serde_json::Value::Number(n) => { + let s = n.to_string(); + writer.write_event(Event::Start(BytesStart::new(name)))?; + writer.write_event(Event::Text(BytesText::new(&s)))?; + writer.write_event(Event::End(BytesEnd::new(name)))?; + } + serde_json::Value::Bool(b) => { + let s = if *b { "true" } else { "false" }; + writer.write_event(Event::Start(BytesStart::new(name)))?; + writer.write_event(Event::Text(BytesText::new(s)))?; + writer.write_event(Event::End(BytesEnd::new(name)))?; + } + serde_json::Value::Null => { + writer.write_event(Event::Empty(BytesStart::new(name)))?; + } + } + Ok(()) +} + +// --- Output formatting --- + +fn format_val_output(val: &Val, opts: &YqOptions, out_format: Format) -> Result { + let mut compact = LimitedString::new(MAX_FORMATTED_OUTPUT_BYTES); + fmt::write(&mut compact, format_args!("{}", val)) + .map_err(|_| "formatted output exceeds size limit".to_string())?; + let compact_str = compact.into_string(); + + // Raw output: unquote strings + if opts.raw_output { + if compact_str.starts_with('"') && compact_str.ends_with('"') && compact_str.len() >= 2 { + if let Ok(unescaped) = serde_json::from_str::(&compact_str) { + ensure_formatted_output_limit(unescaped.len())?; + return Ok(unescaped); + } + } + } + + let json_val: serde_json::Value = + serde_json::from_str(&compact_str).unwrap_or(serde_json::Value::String(compact_str)); + + let output = format_json_as(out_format, &json_val, opts.compact)?; + ensure_formatted_output_limit(output.len())?; + Ok(output) +} + +fn ensure_formatted_output_limit(len: usize) -> Result<(), String> { + if len > MAX_FORMATTED_OUTPUT_BYTES { + return Err("formatted output exceeds size limit".to_string()); + } + Ok(()) +} + +struct LimitedString { + inner: String, + limit: usize, +} + +impl LimitedString { + fn new(limit: usize) -> Self { + Self { + inner: String::new(), + limit, + } + } + + fn into_string(self) -> String { + self.inner + } + + fn write_str(&mut self, s: &str) -> Result<(), String> { + let next_len = self + .inner + .len() + .checked_add(s.len()) + .ok_or("formatted output length overflowed")?; + if next_len > self.limit { + return Err("formatted output exceeds size limit".to_string()); + } + self.inner.push_str(s); + Ok(()) + } + + fn write_char(&mut self, ch: char) -> Result<(), String> { + let mut buf = [0u8; 4]; + self.write_str(ch.encode_utf8(&mut buf)) + } +} + +impl fmt::Write for LimitedString { + fn write_str(&mut self, s: &str) -> fmt::Result { + LimitedString::write_str(self, s).map_err(|_| fmt::Error) + } +} + +struct LimitedBytes { + inner: Vec, + limit: usize, +} + +impl LimitedBytes { + fn new(limit: usize) -> Self { + Self { + inner: Vec::new(), + limit, + } + } + + fn into_string(self) -> Result { + String::from_utf8(self.inner) + } +} + +impl io::Write for LimitedBytes { + fn write(&mut self, buf: &[u8]) -> io::Result { + let next_len = self + .inner + .len() + .checked_add(buf.len()) + .ok_or_else(|| io::Error::other("formatted output length overflowed"))?; + if next_len > self.limit { + return Err(io::Error::other("formatted output exceeds size limit")); + } + self.inner.extend_from_slice(buf); + Ok(buf.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +fn format_json_as( + format: Format, + val: &serde_json::Value, + compact: bool, +) -> Result { + match format { + Format::Json => { + let mut out = LimitedBytes::new(MAX_FORMATTED_OUTPUT_BYTES); + if compact { + serde_json::to_writer(&mut out, val) + .map_err(|e| format!("JSON output error: {}", e))?; + } else { + serde_json::to_writer_pretty(&mut out, val) + .map_err(|e| format!("JSON output error: {}", e))?; + } + out.into_string() + .map_err(|e| format!("JSON encoding error: {}", e)) + } + Format::Yaml => { + let mut out = LimitedBytes::new(MAX_FORMATTED_OUTPUT_BYTES); + serde_yaml::to_writer(&mut out, val) + .map_err(|e| format!("YAML output error: {}", e))?; + let s = out + .into_string() + .map_err(|e| format!("YAML encoding error: {}", e))?; + // Strip leading "---\n" and trailing newline for cleaner output + let s = s.strip_prefix("---\n").unwrap_or(&s); + let s = s.strip_suffix('\n').unwrap_or(s); + Ok(s.to_string()) + } + Format::Toml => json_to_toml_bounded(val), + Format::Xml => json_to_xml(val), + } +} + +fn json_to_toml_bounded(val: &serde_json::Value) -> Result { + let toml_val = json_to_toml(val)?; + let mut out = LimitedString::new(MAX_FORMATTED_OUTPUT_BYTES); + write_toml_document(&mut out, &toml_val)?; + let s = out.into_string(); + Ok(s.strip_suffix('\n').unwrap_or(&s).to_string()) +} + +fn write_toml_document(out: &mut LimitedString, val: &toml::Value) -> Result<(), String> { + match val { + toml::Value::Table(table) => write_toml_table(out, &mut Vec::new(), table), + other => write_toml_inline(out, other), + } +} + +fn write_toml_table( + out: &mut LimitedString, + path: &mut Vec, + table: &toml::map::Map, +) -> Result<(), String> { + for (key, value) in table { + if matches!(value, toml::Value::Table(_)) { + continue; + } + write_toml_key(out, key)?; + out.write_str(" = ")?; + write_toml_inline(out, value)?; + out.write_char('\n')?; + } + + for (key, value) in table { + let toml::Value::Table(child) = value else { + continue; + }; + if !path.is_empty() || table_has_scalar_entries(child) { + out.write_char('\n')?; + path.push(key.clone()); + out.write_char('[')?; + write_toml_path(out, path)?; + out.write_str("]\n")?; + write_toml_table(out, path, child)?; + path.pop(); + } else { + path.push(key.clone()); + write_toml_table(out, path, child)?; + path.pop(); + } + } + + Ok(()) +} + +fn table_has_scalar_entries(table: &toml::map::Map) -> bool { + table + .values() + .any(|value| !matches!(value, toml::Value::Table(_))) +} + +fn write_toml_path(out: &mut LimitedString, path: &[String]) -> Result<(), String> { + for (i, key) in path.iter().enumerate() { + if i > 0 { + out.write_char('.')?; + } + write_toml_key(out, key)?; + } + Ok(()) +} + +fn write_toml_key(out: &mut LimitedString, key: &str) -> Result<(), String> { + if !key.is_empty() + && key + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'_' || b == b'-') + { + out.write_str(key)?; + } else { + write_toml_string(out, key)?; + } + Ok(()) +} + +fn write_toml_inline(out: &mut LimitedString, val: &toml::Value) -> Result<(), String> { + match val { + toml::Value::String(s) => write_toml_string(out, s), + toml::Value::Integer(i) => out.write_str(&i.to_string()), + toml::Value::Float(f) => out.write_str(&f.to_string()), + toml::Value::Boolean(b) => out.write_str(if *b { "true" } else { "false" }), + toml::Value::Datetime(dt) => out.write_str(&dt.to_string()), + toml::Value::Array(arr) => { + out.write_char('[')?; + for (i, item) in arr.iter().enumerate() { + if i > 0 { + out.write_str(", ")?; + } + write_toml_inline(out, item)?; + } + out.write_char(']') + } + toml::Value::Table(table) => { + out.write_str("{ ")?; + for (i, (key, value)) in table.iter().enumerate() { + if i > 0 { + out.write_str(", ")?; + } + write_toml_key(out, key)?; + out.write_str(" = ")?; + write_toml_inline(out, value)?; + } + out.write_str(" }") + } + } +} + +fn write_toml_string(out: &mut LimitedString, s: &str) -> Result<(), String> { + out.write_char('"')?; + for ch in s.chars() { + match ch { + '"' => out.write_str("\\\"")?, + '\\' => out.write_str("\\\\")?, + '\n' => out.write_str("\\n")?, + '\r' => out.write_str("\\r")?, + '\t' => out.write_str("\\t")?, + '\u{08}' => out.write_str("\\b")?, + '\u{0c}' => out.write_str("\\f")?, + ch if ch.is_control() => out.write_str(&format!("\\u{:04X}", ch as u32))?, + ch => out.write_char(ch)?, + } + } + out.write_char('"') +} + +// --- Main logic --- + +fn run_yq(args: &[String]) -> Result { + let opts = parse_args(args)?; + + let sources = if opts.null_input { + Vec::new() + } else { + read_sources(&opts)? + }; + + let input_formats: Vec = if opts.null_input { + vec![opts.input_format.unwrap_or(Format::Yaml)] + } else { + sources + .iter() + .map(|source| opts.input_format.unwrap_or_else(|| detect_format(source))) + .collect() + }; + let default_input_format = input_formats.first().copied().unwrap_or(Format::Yaml); + + // Default output format: YAML for YAML input, otherwise matches input + let out_format = opts.output_format.unwrap_or(default_input_format); + + // Parse input to JSON, then convert to jaq Val + let inputs = if opts.null_input { + vec![Val::from(serde_json::Value::Null)] + } else { + let json_values: Result, _> = sources + .iter() + .zip(input_formats.iter().copied()) + .map(|(source, format)| parse_input(source, format)) + .collect(); + let mut json_values = json_values?; + if opts.slurp { + vec![Val::from(serde_json::Value::Array(json_values))] + } else { + json_values.drain(..).map(Val::from).collect() + } + }; + + // Compile jaq filter (same engine as jq) + let loader = Loader::new(jaq_std::defs().chain(jaq_json::defs())); + let arena = Arena::default(); + let program = File { + code: opts.filter.as_str(), + path: (), + }; + let modules = loader + .load(&arena, program) + .map_err(|errs| format!("parse error: {:?}", errs))?; + + let filter = Compiler::default() + .with_funs(jaq_std::funs().chain(jaq_json::funs())) + .compile(modules) + .map_err(|errs| format!("compile error: {:?}", errs))?; + + // Execute filter and output + let empty_inputs = RcIter::new(core::iter::empty()); + let stdout = io::stdout(); + let mut out = stdout.lock(); + let mut output_count = 0usize; + + for input in inputs { + let ctx = Ctx::new(core::iter::empty(), &empty_inputs); + let results = filter.run((ctx, input)); + + for result in results { + match result { + Ok(val) => { + record_output_value(&mut output_count)?; + let s = format_val_output(&val, &opts, out_format)?; + writeln!(out, "{}", s).map_err(|e| format!("failed to write stdout: {}", e))?; + } + Err(e) => { + eprintln!("yq: error: {}", e); + return Ok(5); + } + } + } + } + + out.flush() + .map_err(|e| format!("failed to flush stdout: {}", e))?; + + Ok(0) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn xml_depth_limit_rejects_deep_input() { + let mut input = String::new(); + for i in 0..=MAX_XML_DEPTH { + input.push_str(&format!("")); + } + for i in (0..=MAX_XML_DEPTH).rev() { + input.push_str(&format!("")); + } + + let err = xml_to_json(&input).unwrap_err(); + + assert!(err.contains("nesting depth")); + } + + #[test] + fn xml_depth_limit_rejects_deep_empty_input() { + let mut input = String::new(); + for i in 0..MAX_XML_DEPTH { + input.push_str(&format!("")); + } + input.push_str(""); + for i in (0..MAX_XML_DEPTH).rev() { + input.push_str(&format!("")); + } + + let err = xml_to_json(&input).unwrap_err(); + + assert!(err.contains("nesting depth")); + } + + #[test] + fn xml_node_limit_rejects_many_elements() { + let mut nodes = MAX_XML_NODES; + + let err = count_xml_node(&mut nodes).unwrap_err(); + + assert!(err.contains("too many nodes")); + } + + #[test] + fn xml_text_limit_rejects_large_text() { + let input = format!("{}", "x".repeat(MAX_XML_TEXT_BYTES + 1)); + + let err = xml_to_json(&input).unwrap_err(); + + assert!(err.contains("text exceeds")); + } + + #[test] + fn output_limit_rejects_too_many_values() { + let mut count = MAX_OUTPUT_VALUES; + + let err = record_output_value(&mut count).unwrap_err(); + + assert!(err.contains("too many output values")); + } + + #[test] + fn formatted_output_limit_rejects_large_output() { + let err = ensure_formatted_output_limit(MAX_FORMATTED_OUTPUT_BYTES + 1).unwrap_err(); + + assert!(err.contains("formatted output")); + } + + #[test] + fn limited_bytes_rejects_large_serializer_write() { + let mut output = LimitedBytes::new(4); + + let err = output.write_all(b"hello").unwrap_err(); + + assert!(err.to_string().contains("formatted output")); + } + + #[test] + fn limited_toml_writer_rejects_large_value() { + let mut output = LimitedString::new(4); + let value = toml::Value::String("hello".to_string()); + + let err = write_toml_inline(&mut output, &value).unwrap_err(); + + assert!(err.contains("formatted output")); + } + + #[test] + fn input_limit_rejects_oversized_reader() { + let input = vec![b'x'; MAX_INPUT_BYTES + 1]; + + let err = read_limited_string(&input[..]).unwrap_err(); + + assert!(err.contains("stdin exceeds")); + } + + #[test] + fn xml_rejects_unclosed_elements() { + let err = xml_to_json("").unwrap_err(); + + assert!(err.contains("unexpected end")); + } + + #[test] + fn xml_rejects_invalid_text_escape() { + let err = xml_to_json("&bogus;").unwrap_err(); + + assert!(err.contains("invalid XML text")); + } +} diff --git a/software/yq/package.json b/software/yq/package.json new file mode 100644 index 0000000000..bce99150e3 --- /dev/null +++ b/software/yq/package.json @@ -0,0 +1,33 @@ +{ + "name": "@agentos-software/yq", + "version": "0.3.3", + "type": "module", + "license": "Apache-2.0", + "description": "yq YAML/JSON processor for secure-exec VMs", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "files": [ + "dist", + "!dist/package", + "!dist/package.tar" + ], + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "scripts": { + "build": "agentos-toolchain stage --commands-dir ../../toolchain/target/wasm32-wasip1/release/commands --if-missing skip && tsc && agentos-toolchain build", + "check-types": "tsc --noEmit", + "test": "vitest run test/ --passWithNoTests" + }, + "devDependencies": { + "@agentos-software/manifest": "workspace:*", + "@rivet-dev/agentos-toolchain": "workspace:*", + "@types/node": "^22.10.2", + "typescript": "^5.9.2", + "@agentos/test-harness": "workspace:*", + "vitest": "^2.1.9" + } +} diff --git a/registry/software/wget/src/index.ts b/software/yq/src/index.ts similarity index 100% rename from registry/software/wget/src/index.ts rename to software/yq/src/index.ts diff --git a/software/yq/test/manifest.test.ts b/software/yq/test/manifest.test.ts new file mode 100644 index 0000000000..89139d33b8 --- /dev/null +++ b/software/yq/test/manifest.test.ts @@ -0,0 +1,19 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +const packageDir = new URL("..", import.meta.url).pathname; + +describe("package manifest", () => { + it("declares command binaries", () => { + const manifest = JSON.parse( + readFileSync(join(packageDir, "agentos-package.json"), "utf8"), + ); + + expect(manifest.commands?.length ?? 0).toBeGreaterThan(0); + for (const command of manifest.commands) { + expect(typeof command).toBe("string"); + expect(command.length).toBeGreaterThan(0); + } + }); +}); diff --git a/software/yq/test/yq.test.ts b/software/yq/test/yq.test.ts new file mode 100644 index 0000000000..c74fe1b1d7 --- /dev/null +++ b/software/yq/test/yq.test.ts @@ -0,0 +1,181 @@ +import { existsSync } from "node:fs"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { + NodeFileSystem, + createKernel, + createWasmVmRuntime, + describeIf, +} from "@agentos/test-harness"; +import type { Kernel } from "@agentos/test-harness"; +import { afterEach, expect, it } from "vitest"; + +const YQ_COMMAND_DIR = fileURLToPath(new URL("../bin", import.meta.url)); +const hasYqPackageBinary = existsSync(join(YQ_COMMAND_DIR, "yq")); + +let tempRoot: string | undefined; + +async function writeFixture(path: string, contents: string): Promise { + if (!tempRoot) throw new Error("fixture root not initialized"); + const hostPath = join(tempRoot, path.replace(/^\/+/, "")); + await mkdir(dirname(hostPath), { recursive: true }); + await writeFile(hostPath, contents); +} + +async function createTestVFS(): Promise { + tempRoot = await mkdtemp(join(tmpdir(), "agentos-yq-")); + await writeFixture( + "/project/services.yaml", + [ + "services:", + " - name: api", + " enabled: true", + " port: 8080", + " - name: worker", + " enabled: false", + " port: 9090", + ].join("\n") + "\n", + ); + await writeFixture( + "/project/services.json", + JSON.stringify({ services: [{ name: "api" }, { name: "worker" }] }) + "\n", + ); + await writeFixture( + "/project/config.toml", + ["[server]", 'name = "agentos"', "port = 7331", "enabled = true"].join("\n") + + "\n", + ); + await writeFixture( + "/project/inventory.xml", + 'hammernail\n', + ); + await writeFixture("/project/broken.yaml", "services:\n - name: ok\n bad"); + return new NodeFileSystem({ root: tempRoot }); +} + +function lines(stdout: string): string[] { + return stdout.split("\n").filter((line) => line.length > 0); +} + +describeIf(hasYqPackageBinary, "yq command", { timeout: 10_000 }, () => { + let kernel: Kernel | undefined; + + afterEach(async () => { + await kernel?.dispose(); + kernel = undefined; + if (tempRoot) { + await rm(tempRoot, { recursive: true, force: true }); + tempRoot = undefined; + } + }); + + async function mountFixture(): Promise { + const vfs = await createTestVFS(); + kernel = createKernel({ filesystem: vfs }); + await kernel.mount(createWasmVmRuntime({ commandDirs: [YQ_COMMAND_DIR] })); + } + + async function runYq(args: string[]) { + if (!kernel) throw new Error("kernel not mounted"); + let stdout = ""; + let stderr = ""; + const proc = kernel.spawn("yq", args, { + onStdout: (chunk) => { + stdout += Buffer.from(chunk).toString("utf8"); + }, + onStderr: (chunk) => { + stderr += Buffer.from(chunk).toString("utf8"); + }, + }); + const exitCode = await proc.wait(); + await new Promise((resolve) => setTimeout(resolve, 0)); + return { stdout, stderr, exitCode }; + } + + it("filters YAML files and emits raw strings", async () => { + await mountFixture(); + + const result = await runYq([ + "-r", + ".services[] | select(.enabled) | .name", + "/project/services.yaml", + ]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(lines(result.stdout)).toEqual(["api"]); + }); + + it("converts YAML query results to compact JSON", async () => { + await mountFixture(); + + const result = await runYq([ + "-o", + "json", + "-c", + "{names: [.services[].name], ports: [.services[].port]}", + "/project/services.yaml", + ]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(JSON.parse(result.stdout.trim())).toEqual({ + names: ["api", "worker"], + ports: [8080, 9090], + }); + }); + + it("reads JSON files explicitly", async () => { + await mountFixture(); + + const result = await runYq([ + "-p", + "json", + "-r", + ".services[].name", + "/project/services.json", + ]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(lines(result.stdout)).toEqual(["api", "worker"]); + }); + + it("reads TOML files explicitly", async () => { + await mountFixture(); + + const result = await runYq([ + "-p", + "toml", + "-o", + "json", + "-c", + ".server", + "/project/config.toml", + ]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(JSON.parse(result.stdout.trim())).toEqual({ + name: "agentos", + port: 7331, + enabled: true, + }); + }); + + it("reads XML files explicitly", async () => { + await mountFixture(); + + const result = await runYq([ + "-p", + "xml", + "-r", + '.inventory.item[]["#text"]', + "/project/inventory.xml", + ]); + expect(result.exitCode, result.stderr || result.stdout).toBe(0); + expect(lines(result.stdout)).toEqual(["hammer", "nail"]); + }); + + it("fails with a parse error for invalid YAML", async () => { + await mountFixture(); + + const result = await runYq([".", "/project/broken.yaml"]); + expect(result.exitCode).not.toBe(0); + expect(result.stderr).toContain("invalid YAML"); + }); +}); diff --git a/software/yq/tsconfig.json b/software/yq/tsconfig.json new file mode 100644 index 0000000000..03ce790ab7 --- /dev/null +++ b/software/yq/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*"] +} diff --git a/software/zip/agentos-package.json b/software/zip/agentos-package.json new file mode 100644 index 0000000000..b2f8cb34ab --- /dev/null +++ b/software/zip/agentos-package.json @@ -0,0 +1,11 @@ +{ + "commands": [ + "zip" + ], + "registry": { + "title": "zip", + "description": "Create zip archives.", + "priority": 12, + "category": "core" + } +} diff --git a/software/zip/package.json b/software/zip/package.json new file mode 100644 index 0000000000..c638954a3c --- /dev/null +++ b/software/zip/package.json @@ -0,0 +1,33 @@ +{ + "name": "@agentos-software/zip", + "version": "0.3.3", + "type": "module", + "license": "Apache-2.0", + "description": "zip archive creation for secure-exec VMs", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "files": [ + "dist", + "!dist/package", + "!dist/package.tar" + ], + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "scripts": { + "build": "agentos-toolchain stage --commands-dir ../../toolchain/target/wasm32-wasip1/release/commands --if-missing skip && tsc && agentos-toolchain build", + "check-types": "tsc --noEmit", + "test": "vitest run test/ --passWithNoTests" + }, + "devDependencies": { + "@agentos-software/manifest": "workspace:*", + "@rivet-dev/agentos-toolchain": "workspace:*", + "@types/node": "^22.10.2", + "typescript": "^5.9.2", + "@agentos/test-harness": "workspace:*", + "vitest": "^2.1.9" + } +} diff --git a/registry/software/yq/src/index.ts b/software/zip/src/index.ts similarity index 100% rename from registry/software/yq/src/index.ts rename to software/zip/src/index.ts diff --git a/software/zip/test/zip.test.ts b/software/zip/test/zip.test.ts new file mode 100644 index 0000000000..0318e5b088 --- /dev/null +++ b/software/zip/test/zip.test.ts @@ -0,0 +1,56 @@ +/** + * Package-owned integration tests for the zip command. + */ + +import { afterEach, expect, it } from "vitest"; +import { + C_BUILD_DIR, + COMMANDS_DIR, + createInMemoryFileSystem, + createKernel, + createWasmVmRuntime, + describeIf, + hasCWasmBinaries, + type Kernel, +} from "@agentos/test-harness"; + +describeIf(hasCWasmBinaries("zip"), "zip command", { timeout: 10_000 }, () => { + let kernel: Kernel; + + afterEach(async () => { + await kernel?.dispose(); + }); + + it("creates a zip archive for a single file", async () => { + const vfs = createInMemoryFileSystem(); + await vfs.writeFile("/hello.txt", "Hello, World!\n"); + + kernel = createKernel({ filesystem: vfs }); + await kernel.mount( + createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] }), + ); + + const result = await kernel.exec("zip /archive.zip /hello.txt"); + expect(result.exitCode, result.stderr).toBe(0); + + const archive = await vfs.readFile("/archive.zip"); + expect(archive.length).toBeGreaterThan(0); + expect(Array.from(archive.slice(0, 2))).toEqual([0x50, 0x4b]); + }); + + it("creates a zip archive recursively", async () => { + const vfs = createInMemoryFileSystem(); + await vfs.mkdir("/mydir"); + await vfs.writeFile("/mydir/a.txt", "file a\n"); + await vfs.writeFile("/mydir/b.txt", "file b\n"); + + kernel = createKernel({ filesystem: vfs }); + await kernel.mount( + createWasmVmRuntime({ commandDirs: [C_BUILD_DIR, COMMANDS_DIR] }), + ); + + const result = await kernel.exec("zip -r /dir.zip /mydir"); + expect(result.exitCode, result.stderr).toBe(0); + expect(await vfs.exists("/dir.zip")).toBe(true); + }); +}); diff --git a/software/zip/tsconfig.json b/software/zip/tsconfig.json new file mode 100644 index 0000000000..03ce790ab7 --- /dev/null +++ b/software/zip/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*"] +} diff --git a/test-harness/package.json b/test-harness/package.json new file mode 100644 index 0000000000..9bef79bc5a --- /dev/null +++ b/test-harness/package.json @@ -0,0 +1,21 @@ +{ + "name": "@agentos/test-harness", + "version": "0.0.0", + "private": true, + "type": "module", + "license": "Apache-2.0", + "main": "./src/index.ts", + "types": "./src/index.ts", + "exports": { + ".": "./src/index.ts" + }, + "scripts": { + "check-types": "tsc --noEmit" + }, + "dependencies": { + "@rivet-dev/agentos-vm-test-harness": "workspace:*" + }, + "devDependencies": { + "typescript": "^5.9.2" + } +} diff --git a/test-harness/src/index.ts b/test-harness/src/index.ts new file mode 100644 index 0000000000..cb139a8665 --- /dev/null +++ b/test-harness/src/index.ts @@ -0,0 +1 @@ +export * from "@rivet-dev/agentos-vm-test-harness"; diff --git a/test-harness/tsconfig.json b/test-harness/tsconfig.json new file mode 100644 index 0000000000..d47274867f --- /dev/null +++ b/test-harness/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../software/tsconfig.base.json", + "compilerOptions": { + "noEmit": true + }, + "include": ["src/**/*"] +} diff --git a/toolchain/.gitignore b/toolchain/.gitignore new file mode 100644 index 0000000000..819906a59b --- /dev/null +++ b/toolchain/.gitignore @@ -0,0 +1,6 @@ +# Reproducible codex WASI build scratch: the shallow fork clone, its cargo +# vendor tree, and its wasm32-wasip1 target/ all live here. Never snapshot it. +.codex-build/ + +# cargo vendor output for `make wasm` (the command build) and any codex vendor. +vendor/ diff --git a/registry/native/Cargo.lock b/toolchain/Cargo.lock similarity index 94% rename from registry/native/Cargo.lock rename to toolchain/Cargo.lock index fb7e69a723..7358c629ab 100644 --- a/registry/native/Cargo.lock +++ b/toolchain/Cargo.lock @@ -151,6 +151,17 @@ dependencies = [ "triomphe", ] +[[package]] +name = "argmax" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0144c58b55af0133ec3963ce5e4d07aad866e3bbcfdcddbf4590dbd7ad6ff557" +dependencies = [ + "libc", + "nix 0.30.1", + "once_cell", +] + [[package]] name = "arrayref" version = "0.3.9" @@ -601,9 +612,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrono" -version = "0.4.44" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", "js-sys", @@ -829,6 +840,13 @@ dependencies = [ "uu_echo", ] +[[package]] +name = "cmd-egrep" +version = "0.1.0" +dependencies = [ + "secureexec-shims", +] + [[package]] name = "cmd-env" version = "0.1.0" @@ -868,7 +886,14 @@ dependencies = [ name = "cmd-fd" version = "0.1.0" dependencies = [ - "secureexec-fd", + "fd-find", +] + +[[package]] +name = "cmd-fgrep" +version = "0.1.0" +dependencies = [ + "secureexec-shims", ] [[package]] @@ -882,7 +907,8 @@ dependencies = [ name = "cmd-find" version = "0.1.0" dependencies = [ - "secureexec-find", + "findutils", + "uucore 0.9.0", ] [[package]] @@ -899,20 +925,6 @@ dependencies = [ "uu_fold", ] -[[package]] -name = "cmd-git" -version = "0.1.0" -dependencies = [ - "secureexec-git", -] - -[[package]] -name = "cmd-grep" -version = "0.1.0" -dependencies = [ - "secureexec-grep", -] - [[package]] name = "cmd-gzip" version = "0.1.0" @@ -1113,7 +1125,7 @@ dependencies = [ name = "cmd-rg" version = "0.1.0" dependencies = [ - "secureexec-grep", + "ripgrep", ] [[package]] @@ -1319,13 +1331,6 @@ dependencies = [ "uu_tr", ] -[[package]] -name = "cmd-tree" -version = "0.1.0" -dependencies = [ - "secureexec-tree", -] - [[package]] name = "cmd-true" version = "0.1.0" @@ -1400,7 +1405,7 @@ dependencies = [ name = "cmd-xargs" version = "0.1.0" dependencies = [ - "secureexec-shims", + "findutils", ] [[package]] @@ -1576,6 +1581,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-deque" version = "0.8.6" @@ -1837,6 +1851,24 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "encoding_rs_io" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cc3c5651fb62ab8aa3103998dade57efdd028544bd300516baa31840c252a83" +dependencies = [ + "encoding_rs", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -1853,6 +1885,27 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "etcetera" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de48cc4d1c1d97a20fd819def54b890cadde72ed3ad0c614822a0a433361be96" +dependencies = [ + "cfg-if", + "windows-sys 0.61.2", +] + +[[package]] +name = "faccess" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ae66425802d6a903e268ae1a08b8c38ba143520f227a205edf4e9c7e3e26d5" +dependencies = [ + "bitflags 1.3.2", + "libc", + "winapi", +] + [[package]] name = "fancy-regex" version = "0.16.2" @@ -1881,6 +1934,32 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "fd-find" +version = "10.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b95ed7d1f53e0446a7d47715801f6bee95f816c4aa33e25b5d89a2734ab00436" +dependencies = [ + "aho-corasick", + "anyhow", + "argmax", + "clap", + "crossbeam-channel", + "ctrlc", + "etcetera", + "faccess", + "globset", + "ignore", + "jiff", + "libc", + "lscolors", + "nix 0.31.3", + "normpath", + "nu-ansi-term", + "regex", + "regex-syntax", +] + [[package]] name = "fd-lock" version = "4.0.4" @@ -1909,6 +1988,25 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "findutils" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "719cc0af0060da07d610f4faa20d2d46dca46ff6bdf778a4b4feec3a7c8af7ea" +dependencies = [ + "argmax", + "chrono", + "clap", + "faccess", + "itertools 0.14.0", + "nix 0.31.3", + "onig", + "regex", + "thiserror 2.0.18", + "uucore 0.9.0", + "walkdir", +] + [[package]] name = "fixed_decimal" version = "0.7.1" @@ -2188,6 +2286,98 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +[[package]] +name = "globset" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52dfc19153a48bde0cbd630453615c8151bce3a5adfac7a0aebfbf0a1e1f57e3" +dependencies = [ + "aho-corasick", + "bstr", + "log", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "grep" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "309217bc53e2c691c314389c7fa91f9cd1a998cda19e25544ea47d94103880c3" +dependencies = [ + "grep-cli", + "grep-matcher", + "grep-printer", + "grep-regex", + "grep-searcher", +] + +[[package]] +name = "grep-cli" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf32d263c5d5cc2a23ce587097f5ddafdb188492ba2e6fb638eaccdc22453631" +dependencies = [ + "bstr", + "globset", + "libc", + "log", + "termcolor", + "winapi-util", +] + +[[package]] +name = "grep-matcher" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36d7b71093325ab22d780b40d7df3066ae4aebb518ba719d38c697a8228a8023" +dependencies = [ + "memchr", +] + +[[package]] +name = "grep-printer" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd76035e87871f51c1ee5b793e32122b3ccf9c692662d9622ef1686ff5321acb" +dependencies = [ + "bstr", + "grep-matcher", + "grep-searcher", + "log", + "serde", + "serde_json", + "termcolor", +] + +[[package]] +name = "grep-regex" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce0c256c3ad82bcc07b812c15a45ec1d398122e8e15124f96695234db7112ef" +dependencies = [ + "bstr", + "grep-matcher", + "log", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "grep-searcher" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac63295322dc48ebb20a25348147905d816318888e64f531bfc2a2bc0577dc34" +dependencies = [ + "bstr", + "encoding_rs", + "encoding_rs_io", + "grep-matcher", + "log", + "memchr", + "memmap2", +] + [[package]] name = "half" version = "2.7.1" @@ -2570,6 +2760,22 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" +[[package]] +name = "ignore" +version = "0.4.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b915661dd01db3f05050265b2477bcc6527b3792388e2749b41623cc592be67d" +dependencies = [ + "crossbeam-deque", + "globset", + "log", + "memchr", + "regex-automata", + "same-file", + "walkdir", + "winapi-util", +] + [[package]] name = "indenter" version = "0.3.4" @@ -2851,11 +3057,17 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" +[[package]] +name = "lexopt" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "803ec87c9cfb29b9d2633f20cba1f488db3fd53f2158b1024cbefb47ba05d413" + [[package]] name = "libc" -version = "0.2.183" +version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "libm" @@ -3019,6 +3231,18 @@ dependencies = [ "libc", ] +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags 2.11.0", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "nom" version = "7.1.3" @@ -3044,6 +3268,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f5438dd2b2ff4c6df6e1ce22d825ed2fa93ee2922235cc45186991717f0a892d" +[[package]] +name = "normpath" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9985ef7269fa99f3b12437bb698381da2428743ab90f20393f399fa14cab21a" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "notify" version = "8.2.0" @@ -3191,6 +3424,28 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "onig" +version = "6.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc3cbf698f9438986c11a880c90a6d04b9de27575afd28bbf45b154b6c709e2" +dependencies = [ + "bitflags 2.11.0", + "libc", + "once_cell", + "onig_sys", +] + +[[package]] +name = "onig_sys" +version = "69.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e68317604e77e53b85896388e1a803c1d21b74c899ec9e5e1112db90735edd7" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "os_display" version = "0.1.4" @@ -3356,6 +3611,12 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + [[package]] name = "plain" version = "0.2.3" @@ -3659,6 +3920,24 @@ version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +[[package]] +name = "ripgrep" +version = "15.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f388c4955f85477c28a8667355819844a06614b083c23517f0e86bd1d6d82b73" +dependencies = [ + "anyhow", + "bstr", + "grep", + "ignore", + "lexopt", + "log", + "serde_json", + "termcolor", + "textwrap", + "tikv-jemallocator", +] + [[package]] name = "rlimit" version = "0.10.2" @@ -3788,13 +4067,6 @@ dependencies = [ "regex", ] -[[package]] -name = "secureexec-fd" -version = "0.1.0" -dependencies = [ - "regex", -] - [[package]] name = "secureexec-file-cmd" version = "0.1.0" @@ -3802,29 +4074,6 @@ dependencies = [ "infer", ] -[[package]] -name = "secureexec-find" -version = "0.1.0" -dependencies = [ - "regex", -] - -[[package]] -name = "secureexec-git" -version = "0.1.0" -dependencies = [ - "flate2", - "secureexec-wasi-http", - "sha1", -] - -[[package]] -name = "secureexec-grep" -version = "0.1.0" -dependencies = [ - "regex", -] - [[package]] name = "secureexec-gzip" version = "0.1.0" @@ -3869,10 +4118,6 @@ dependencies = [ "tar", ] -[[package]] -name = "secureexec-tree" -version = "0.1.0" - [[package]] name = "secureexec-wasi-http" version = "0.1.0" @@ -4279,6 +4524,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + [[package]] name = "terminal_size" version = "0.4.3" @@ -4301,6 +4555,12 @@ dependencies = [ "phf_codegen 0.11.3", ] +[[package]] +name = "textwrap" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c13547615a44dc9c452a8a534638acdf07120d4b6847c8178705da06306a3057" + [[package]] name = "thiserror" version = "1.0.69" @@ -4350,6 +4610,26 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "tikv-jemalloc-sys" +version = "0.6.1+5.3.0-1-ge13ca993e8ccb9ba9847cc330696e02839f328f7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd8aa5b2ab86a2cefa406d889139c162cbb230092f7d1d7cbc1716405d852a3b" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "tikv-jemallocator" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0359b4327f954e0567e69fb191cf1436617748813819c94b8cd4a431422d053a" +dependencies = [ + "libc", + "tikv-jemalloc-sys", +] + [[package]] name = "tinystr" version = "0.8.2" @@ -5669,6 +5949,30 @@ dependencies = [ "z85", ] +[[package]] +name = "uucore" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "069b34217c27f611e1589f540f58118dbf226a9e407d38ab472052ff075a1dc2" +dependencies = [ + "bstr", + "clap", + "dunce", + "fluent", + "fluent-syntax", + "libc", + "nix 0.31.3", + "os_display", + "rustc-hash", + "rustix 1.1.4", + "thiserror 2.0.18", + "unic-langid", + "uucore_procs 0.9.0", + "wild", + "winapi-util", + "windows-sys 0.61.2", +] + [[package]] name = "uucore_procs" version = "0.4.0" @@ -5689,6 +5993,16 @@ dependencies = [ "quote", ] +[[package]] +name = "uucore_procs" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34337da211e7abfff7189b794afb3b5018fe356fb36474b73645458fc1201350" +dependencies = [ + "proc-macro2", + "quote", +] + [[package]] name = "uuid" version = "1.22.0" diff --git a/toolchain/Cargo.toml b/toolchain/Cargo.toml new file mode 100644 index 0000000000..124085bce2 --- /dev/null +++ b/toolchain/Cargo.toml @@ -0,0 +1,30 @@ +[workspace] +resolver = "2" +members = [ + "crates/wasi-ext", + "crates/libs/*", + "crates/commands/*", + "stubs/codex-otel", + "stubs/ctrlc", + "stubs/hostname", + "../software/*/native/crates/*", +] + +[workspace.package] +version = "0.1.0" +edition = "2021" +license = "Apache-2.0" + +[profile.release] +opt-level = "z" +lto = true +codegen-units = 1 +strip = true + +# WASM-compatible stubs for crates that don't support wasm32-wasip1 +[patch.crates-io] +ctrlc = { path = "stubs/ctrlc" } +hostname = { path = "stubs/hostname" } +uucore = { path = "stubs/uucore" } +codex-network-proxy = { path = "stubs/codex-network-proxy" } +codex-otel = { path = "stubs/codex-otel" } diff --git a/toolchain/Makefile b/toolchain/Makefile new file mode 100644 index 0000000000..cff2c8ea86 --- /dev/null +++ b/toolchain/Makefile @@ -0,0 +1,365 @@ +WASM_TARGET := wasm32-wasip1 +RELEASE_DIR := target/$(WASM_TARGET)/release +WASI_C_SYSROOT := $(CURDIR)/c/sysroot +WASI_C_CC := $(CURDIR)/c/vendor/wasi-sdk/bin/clang +WASI_C_AR := $(CURDIR)/c/vendor/wasi-sdk/bin/llvm-ar + +# Standalone binary output directory (configurable) +COMMANDS_DIR ?= $(RELEASE_DIR)/commands + +# Detect patched std: if std-patches/ has .patch files and patch-std.sh exists, use patched sysroot +PATCHES := $(wildcard std-patches/*.patch) +PATCH_SCRIPT := scripts/patch-std.sh + +# Resolve WASI sysroot for CRT/libc needed by -Z build-std linking +# Uses rustc without +toolchain override so rust-toolchain.toml is respected +WASI_SYSROOT_SELF_CONTAINED := $(shell rustc --print sysroot)/lib/rustlib/$(WASM_TARGET)/lib/self-contained + +# Discover command binary names from colocated package crates plus toolchain +# helper crates. Keep known slow/heavy commands out of the default software +# gate; they can still be built explicitly +# with `make cmd/`. +SOFTWARE_COMMAND_CRATES := $(wildcard ../software/*/native/crates/cmd-*) +TOOLCHAIN_COMMAND_CRATES := $(wildcard crates/commands/*) +SKIPPED_BULK_COMMANDS := _stubs git codex codex-exec +COMMAND_NAMES := $(filter-out $(SKIPPED_BULK_COMMANDS),$(patsubst cmd-%,%,$(notdir $(SOFTWARE_COMMAND_CRATES))) $(notdir $(TOOLCHAIN_COMMAND_CRATES))) +COMMAND_PACKAGES := $(addprefix -p cmd-,$(COMMAND_NAMES)) + +# Alias symlinks: link_name:target_binary +ALIAS_SYMLINKS := \ + gunzip:gzip zcat:gzip \ + bash:sh \ + dir:ls vdir:ls \ + more:cat \ + [:test + +# Stub commands: each gets a symlink to _stubs +STUB_COMMANDS := \ + chcon runcon \ + chgrp chown chroot \ + df \ + groups id \ + hostname hostid \ + install \ + kill \ + mkfifo mknod \ + pinky who users uptime \ + stty sync tty + +.PHONY: all commands wasm wasm-opt-check host test clean patch-std patch-check vendor patch-vendor vendor-patch-check size-report install c-wasm codex + +all: wasm + +# Build the fast command set that lands in COMMANDS_DIR: Rust commands except +# known heavy/external ones, then the opted-in fast C commands. Heavy commands +# (git, duckdb, vim), generated helpers (_stubs), and the external codex build +# are intentionally excluded from the default software-build gate. +commands: wasm + $(MAKE) -C c programs install + +# Build the real wasm32-wasip1 `codex-exec` agent engine and install it where +# @agentos-software/codex-cli stages commands from. +# +# Two modes: +# - default (CODEX_REPO unset): REPRODUCIBLE build. clone-and-build-codex-wasi.sh +# shallow-clones the fork pinned in toolchain/codex-ref, rewrites its +# [patch.crates-io] to the toolchain stubs + vendored/patched tokio, and builds. +# No local checkout needed; deterministic from the pin. +# - CODEX_REPO=/path/to/codex-rs/codex-rs: DEV override. Builds that existing +# checkout in place via scripts/build-codex-wasi.sh (fast local iteration). +AGENTOS_ROOT := $(abspath $(CURDIR)/..) +CODEX_REF := $(shell cat $(CURDIR)/codex-ref 2>/dev/null) +CODEX_REPO ?= +CODEX_WASI_SDK_DIR := $(abspath $(CURDIR)/c/vendor/wasi-sdk) +codex: c/vendor/wasi-sdk/bin/clang +ifeq ($(strip $(CODEX_REPO)),) + @echo "codex: reproducible build from pin $(CODEX_REF)" + @WASI_SDK_DIR="$(CODEX_WASI_SDK_DIR)" \ + "$(CURDIR)/scripts/clone-and-build-codex-wasi.sh" +else + @if [ -x "$(CODEX_REPO)/scripts/build-wasi-codex-exec.sh" ]; then \ + echo "codex: CODEX_REPO override -> building $(CODEX_REPO) in place"; \ + CODEX_REPO="$(CODEX_REPO)" \ + WASI_SDK_DIR="$(CODEX_WASI_SDK_DIR)" \ + "$(CURDIR)/scripts/build-codex-wasi.sh"; \ + else \ + echo "codex: CODEX_REPO=$(CODEX_REPO) has no scripts/build-wasi-codex-exec.sh"; \ + echo " Unset CODEX_REPO to build reproducibly from the pin instead."; \ + exit 1; \ + fi +endif + +c/vendor/wasi-sdk/bin/clang: + $(MAKE) -C c wasi-sdk + +c/sysroot/lib/wasm32-wasi/libc.a: + $(MAKE) -C c sysroot + +# Strict variant for environments that require the codex artifact. With CODEX_REPO +# unset it builds reproducibly from the pin; with CODEX_REPO set it fails hard if +# that checkout lacks the fork build script. +.PHONY: codex-required +codex-required: +ifeq ($(strip $(CODEX_REPO)),) + @$(MAKE) codex +else + @test -x "$(CODEX_REPO)/scripts/build-wasi-codex-exec.sh" || { \ + echo "ERROR: codex fork build script not found at $(CODEX_REPO); unset CODEX_REPO to build from the pin"; exit 1; } + @$(MAKE) codex +endif + +# Apply std patches if they exist +patch-std: +ifneq ($(PATCHES),) + @if [ -x "$(PATCH_SCRIPT)" ]; then \ + echo "Applying std patches..."; \ + ./$(PATCH_SCRIPT); \ + else \ + echo "Warning: patches found but $(PATCH_SCRIPT) not executable or missing"; \ + fi +else + @echo "No patches found, using stock std" +endif + +# Resolve Rust std source for vendoring build-std deps +RUST_STD_SRC := $(shell rustc --print sysroot)/lib/rustlib/src/rust + +# Vendor dependencies from crates.io into vendor/ (includes std deps for -Z build-std) +vendor: + @if [ ! -d vendor ]; then \ + echo "Vendoring dependencies (including std library deps)..."; \ + mkdir -p .cargo; \ + cargo vendor \ + --sync "$(RUST_STD_SRC)/library/std/Cargo.toml" \ + --sync "$(RUST_STD_SRC)/library/test/Cargo.toml" \ + vendor > .cargo/config.toml; \ + else \ + echo "vendor/ exists — skipping (run 'make clean-vendor' to re-vendor)"; \ + fi + @# Source-replacement config so cargo builds from the patched vendor/ tree + @# instead of the crates.io registry cache. This file is gitignored, so a + @# fresh checkout (CI) has none; without it every std-patches/crates/*.patch + @# wasi fix is silently ignored and crates like uu_tac fail to compile for + @# wasm32-wasip1. `cargo vendor` prints the exact config (including the + @# synced std-dep sources) on stdout, so capture it above. + @test -s .cargo/config.toml || { \ + echo "ERROR: .cargo/config.toml missing or empty after vendor"; \ + echo "Run 'make clean-vendor && make vendor' to regenerate."; \ + exit 1; \ + } + +# Apply crate-level patches to vendored sources +patch-vendor: vendor + @if [ -x scripts/patch-vendor.sh ]; then \ + ./scripts/patch-vendor.sh; \ + else \ + echo "Warning: scripts/patch-vendor.sh not found or not executable"; \ + fi + +# Dry-run verification of crate patches +vendor-patch-check: vendor + @if [ -x scripts/patch-vendor.sh ]; then \ + ./scripts/patch-vendor.sh --check; \ + else \ + echo "Warning: scripts/patch-vendor.sh not found or not executable"; \ + fi + +# Ensure wasm-opt is installed (needed for post-build optimization) +wasm-opt-check: + @if ! command -v wasm-opt >/dev/null 2>&1; then \ + echo "wasm-opt not found — installing via cargo..."; \ + cargo install wasm-opt; \ + else \ + echo "wasm-opt found: $$(wasm-opt --version)"; \ + fi + +# Build all standalone command binaries, optimize, strip .wasm extension, create symlinks +wasm: c/sysroot/lib/wasm32-wasi/libc.a vendor patch-vendor patch-std wasm-opt-check + # --cfg tokio_unstable lifts tokio's wasm `compile_error!` so the patched + # tokio process/net modules (std-patches/crates/tokio) compile on wasm32-wasip1. + # Needed by codex (tokio::process/net). Harmless for commands not using tokio. + CC_wasm32_wasip1="$(WASI_C_CC)" \ + AR_wasm32_wasip1="$(WASI_C_AR)" \ + CFLAGS_wasm32_wasip1="--sysroot=$(WASI_C_SYSROOT)" \ + RUSTFLAGS="-C link-arg=-L$(WASI_SYSROOT_SELF_CONTAINED) --cfg tokio_unstable" \ + cargo build --target $(WASM_TARGET) \ + -Z build-std=std,panic_abort \ + --release \ + $(COMMAND_PACKAGES) + @mkdir -p $(COMMANDS_DIR) + @TOTAL=0; SKIPPED=0; \ + HAS_OPT=$$(command -v wasm-opt >/dev/null 2>&1 && echo 1 || echo 0); \ + for cmd in $(COMMAND_NAMES); do \ + wasm="$(RELEASE_DIR)/$$cmd.wasm"; \ + if [ -f "$$wasm" ]; then \ + TOTAL=$$((TOTAL + 1)); \ + if [ "$$HAS_OPT" = "1" ]; then \ + wasm-opt -O3 --strip-debug --all-features "$$wasm" -o "$(COMMANDS_DIR)/$$cmd"; \ + else \ + cp "$$wasm" "$(COMMANDS_DIR)/$$cmd"; \ + fi; \ + else \ + SKIPPED=$$((SKIPPED + 1)); \ + echo "Warning: $$wasm not found — skipping $$cmd"; \ + fi; \ + done; \ + echo ""; \ + echo "=== Standalone Binary Build Report ==="; \ + echo "Binaries: $$TOTAL built"; \ + if [ $$SKIPPED -gt 0 ]; then \ + echo "Skipped: $$SKIPPED (binary not found)"; \ + fi; \ + if [ "$$HAS_OPT" = "1" ]; then \ + echo "Optimized: wasm-opt -O3 --strip-debug"; \ + else \ + echo "Optimized: none (wasm-opt not found)"; \ + fi + @# Create alias symlinks (fall back to copy on platforms without symlink support) + @cd $(COMMANDS_DIR) && \ + for pair in $(ALIAS_SYMLINKS); do \ + link=$${pair%%:*}; target=$${pair##*:}; \ + if [ -f "$$target" ]; then \ + ln -sf "$$target" "$$link" 2>/dev/null || cp "$$target" "$$link"; \ + fi; \ + done + @# Create stub command symlinks + @cd $(COMMANDS_DIR) && \ + for stub in $(STUB_COMMANDS); do \ + if [ -f _stubs ]; then \ + ln -sf _stubs "$$stub" 2>/dev/null || cp _stubs "$$stub"; \ + fi; \ + done + @TOTAL_FILES=$$(ls -1 $(COMMANDS_DIR) 2>/dev/null | wc -l); \ + echo "Total: $$TOTAL_FILES entries in $(COMMANDS_DIR)"; \ + echo "=== Build complete ===" + +# --------------------------------------------------------------------------- +# Uniform per-binary entry point: `make cmd/` builds ONE command binary +# into $(COMMANDS_DIR), dispatching to the toolchain that owns it: +# Rust software/*/native/crates/cmd- or crates/commands/ +# (cargo package cmd-, via wasm-cmd) +# C the C opt-in set below (toolchain/c, patched sysroot) +# codex the codex fork build (codex, codex-exec) +# `just toolchain-cmd ` calls this. `make wasm` builds the fast +# Rust set; `make -C c programs install` builds/installs the fast C set. +C_COMMANDS := zip unzip envsubst sqlite3 curl wget grep git duckdb vim + +.PHONY: cmd/% +cmd/%: + @name="$*"; \ + crate_dir=$$(find ../software -path "*/native/crates/cmd-$$name" -type d -print -quit); \ + if [ -n "$$crate_dir" ] || [ -d "crates/commands/$$name" ]; then \ + $(MAKE) wasm-cmd CMD=$$name; \ + elif echo " $(C_COMMANDS) " | grep -q " $$name "; then \ + src=$$name; [ "$$name" = "sqlite3" ] && src=sqlite3_cli; \ + $(MAKE) -C c sysroot "build/$$src" && $(MAKE) -C c install COMMANDS="$$name"; \ + elif [ "$$name" = "codex" ] || [ "$$name" = "codex-exec" ]; then \ + $(MAKE) codex-required; \ + else \ + echo "ERROR: unknown command '$$name' (no software/*/native/crates/cmd-$$name or crates/commands/$$name; not in C_COMMANDS or codex)"; \ + exit 1; \ + fi + +# Build ONE command binary (cargo package cmd-$(CMD) -> $(COMMANDS_DIR)/$(CMD)). +# Usage: make wasm-cmd CMD=sh +.PHONY: wasm-cmd +wasm-cmd: c/sysroot/lib/wasm32-wasi/libc.a vendor patch-vendor patch-std wasm-opt-check + @test -n "$(CMD)" || { echo "ERROR: pass CMD= (a dir under crates/commands/)"; exit 1; } + @crate_dir=$$(find ../software -path "*/native/crates/cmd-$(CMD)" -type d -print -quit); \ + test -n "$$crate_dir" || test -d "crates/commands/$(CMD)" || { \ + echo "ERROR: no software/*/native/crates/cmd-$(CMD) or crates/commands/$(CMD)"; exit 1; } + @cargo_pkg="cmd-$(CMD)"; cargo_bin="$(CMD)"; \ + if [ "$(CMD)" = "fd" ]; then \ + cargo_pkg="fd-find"; \ + cargo_bin="fd"; \ + elif [ "$(CMD)" = "rg" ]; then \ + cargo_pkg="ripgrep"; \ + cargo_bin="rg"; \ + fi; \ + CC_wasm32_wasip1="$(WASI_C_CC)" \ + AR_wasm32_wasip1="$(WASI_C_AR)" \ + CFLAGS_wasm32_wasip1="--sysroot=$(WASI_C_SYSROOT)" \ + RUSTFLAGS="-C link-arg=-L$(WASI_SYSROOT_SELF_CONTAINED) --cfg tokio_unstable" \ + cargo build --target $(WASM_TARGET) \ + -Z build-std=std,panic_abort \ + --release \ + -p $$cargo_pkg \ + --bin $$cargo_bin + @mkdir -p $(COMMANDS_DIR) + @if command -v wasm-opt >/dev/null 2>&1; then \ + wasm-opt -O3 --strip-debug --all-features "$(RELEASE_DIR)/$(CMD).wasm" -o "$(COMMANDS_DIR)/$(CMD)"; \ + else \ + cp "$(RELEASE_DIR)/$(CMD).wasm" "$(COMMANDS_DIR)/$(CMD)"; \ + fi + @echo "Built $(COMMANDS_DIR)/$(CMD)" + +# Show per-command binary sizes in COMMANDS_DIR +size-report: + @if [ ! -d "$(COMMANDS_DIR)" ]; then \ + echo "Error: $(COMMANDS_DIR) not found. Run 'make wasm' first."; \ + exit 1; \ + fi + @echo "=== Standalone Binary Size Report ===" + @echo "" + @printf "%-10s %s\n" "SIZE" "COMMAND" + @printf "%-10s %s\n" "----" "-------" + @ls -lhS $(COMMANDS_DIR)/* 2>/dev/null | \ + grep -v "^total" | \ + while read line; do \ + size=$$(echo "$$line" | awk '{print $$5}'); \ + name=$$(echo "$$line" | awk '{print $$NF}'); \ + base=$$(basename "$$name"); \ + link=""; \ + if [ -L "$$name" ]; then \ + target=$$(readlink "$$name"); \ + link=" -> $$target"; \ + fi; \ + printf "%-10s %s%s\n" "$$size" "$$base" "$$link"; \ + done + @echo "" + @TOTAL_SIZE=$$(du -sh $(COMMANDS_DIR) 2>/dev/null | awk '{print $$1}'); \ + TOTAL_FILES=$$(ls -1 $(COMMANDS_DIR) 2>/dev/null | wc -l); \ + REAL_FILES=$$(find $(COMMANDS_DIR) -maxdepth 1 -type f 2>/dev/null | wc -l); \ + SYMLINKS=$$(find $(COMMANDS_DIR) -maxdepth 1 -type l 2>/dev/null | wc -l); \ + echo "Total: $$TOTAL_SIZE ($$TOTAL_FILES entries: $$REAL_FILES binaries, $$SYMLINKS symlinks)" + @echo "" + @echo "=== Per-Crate Size Contribution (rlib, top 30) ===" + @echo "Note: rlib sizes are pre-LTO; actual contribution after LTO/DCE may differ." + @echo "" + @printf "%-10s %s\n" "SIZE" "CRATE" + @printf "%-10s %s\n" "----" "-----" + @ls -lhS $(RELEASE_DIR)/deps/*.rlib 2>/dev/null | \ + awk '{split($$NF, a, "/"); name=a[length(a)]; gsub(/^lib/, "", name); gsub(/-[0-9a-f]+\.rlib$$/, "", name); printf "%-10s %s\n", $$5, name}' | \ + awk '!seen[$$2]++' | \ + head -30 + @echo "" + @echo "=== Dependency Count ===" + @TOTAL=$$(ls $(RELEASE_DIR)/deps/*.rlib 2>/dev/null | \ + sed 's/.*\/lib//' | sed 's/-[0-9a-f]*\.rlib//' | sort -u | wc -l); \ + echo "Unique crates: $$TOTAL" + +# Install standalone binaries to a custom directory + +# Build C programs to WASM (downloads wasi-sdk if needed) + + +test: + cargo test --workspace + +clean: + cargo clean + rm -rf host/dist + +clean-vendor: + rm -rf vendor .cargo/config.toml + +patch-check: +ifneq ($(PATCHES),) + @if [ -x "$(PATCH_SCRIPT)" ]; then \ + ./$(PATCH_SCRIPT) --check; \ + else \ + echo "Warning: $(PATCH_SCRIPT) not found or not executable"; \ + fi +else + @echo "No patches found" +endif diff --git a/toolchain/c/Makefile b/toolchain/c/Makefile new file mode 100644 index 0000000000..a47d91037e --- /dev/null +++ b/toolchain/c/Makefile @@ -0,0 +1,962 @@ +# wasmvm/c/Makefile — Build C programs to standalone WASM binaries +# +# Targets: +# wasi-sdk Download and cache wasi-sdk toolchain +# programs Compile package C commands and C test programs to WASM binaries +# sysroot Build patched wasi-libc sysroot (invokes patch-wasi-libc.sh) +# install Copy opted-in real commands to COMMANDS_DIR +# native Compile all .c files to native binaries (for parity testing) +# clean Remove build output +# +# The vanilla (unpatched) wasi-sdk sysroot is used by default. +# After US-025 adds patch-wasi-libc.sh, `make sysroot` builds a patched one. + +# wasi-sdk version and platform detection +WASI_SDK_VERSION := 25 +WASI_SDK_MAJOR := 25.0 + +UNAME_S := $(shell uname -s) +UNAME_M := $(shell uname -m) + +ifeq ($(UNAME_S),Linux) + WASI_SDK_PLATFORM := linux +else ifeq ($(UNAME_S),Darwin) + WASI_SDK_PLATFORM := macos +else + WASI_SDK_PLATFORM := linux +endif + +ifeq ($(UNAME_M),arm64) + WASI_SDK_ARCH := arm64 +else ifeq ($(UNAME_M),aarch64) + WASI_SDK_ARCH := arm64 +else + WASI_SDK_ARCH := x86_64 +endif + +WASI_SDK_TARBALL := wasi-sdk-$(WASI_SDK_MAJOR)-$(WASI_SDK_ARCH)-$(WASI_SDK_PLATFORM).tar.gz +WASI_SDK_URL := https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-$(WASI_SDK_VERSION)/$(WASI_SDK_TARBALL) + +# Directories +VENDOR_DIR := vendor +WASI_SDK_DIR := $(VENDOR_DIR)/wasi-sdk +BUILD_DIR := build +NATIVE_DIR := build/native +SOFTWARE_DIR := ../../software +TEST_PROGRAMS_DIR := ../test-programs +C_SOURCE_DIRS := $(TEST_PROGRAMS_DIR) $(wildcard $(SOFTWARE_DIR)/*/native/c) +c_source = $(firstword $(foreach dir,$(C_SOURCE_DIRS),$(wildcard $(dir)/$(1).c))) + +# Toolchain +CC := $(WASI_SDK_DIR)/bin/clang +NATIVE_CC := $(shell command -v cc 2>/dev/null || command -v gcc 2>/dev/null || command -v clang 2>/dev/null) + +# Sysroot: use patched sysroot if built, otherwise use wasi-sdk's vanilla sysroot +PATCHED_SYSROOT := sysroot +ifeq ($(wildcard $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a),) + SYSROOT := $(WASI_SDK_DIR)/share/wasi-sysroot +else + SYSROOT := $(PATCHED_SYSROOT) +endif + +# Compile flags +WASM_CFLAGS := --target=wasm32-wasip1 --sysroot=$(SYSROOT) -O2 -flto -I include/ +NATIVE_CFLAGS := -O0 -g -D_LARGEFILE64_SOURCE -I include/ + +# COMMANDS_DIR for install target (configurable, matches Rust binary output) +COMMANDS_DIR ?= ../target/wasm32-wasip1/release/commands + +# Fast real commands installed by the default software gate. Slow/heavy commands +# (duckdb, vim) remain available through explicit parent `make cmd/`. +COMMANDS := zip unzip envsubst sqlite3 curl wget grep git git-remote-http tree + +# Programs requiring patched sysroot (Tier 2+ custom host imports) +PATCHED_PROGRAMS := http_get_test isatty_test getpid_test getppid_test getppid_verify userinfo pipe_test dup_test closefrom_test spawn_child spawn_exit_code pipeline kill_child waitpid_return waitpid_edge syscall_coverage getpwuid_test signal_tests sigaction_self sigaction_behavior delayed_tcp_echo delayed_kill pipe_edge tcp_accept_spawn tcp_echo tcp_server http_server udp_echo unix_socket signal_handler dns_lookup sqlite3 curl wget grep tree zip unzip fs_probe + +# Discover all package command and test-program C source files. +ALL_SOURCES := $(foreach dir,$(C_SOURCE_DIRS),$(wildcard $(dir)/*.c)) +SKIPPED_BULK_PROGRAMS := + +# Exclude patched-sysroot programs when only vanilla sysroot is available +ifeq ($(wildcard $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a),) + EXCLUDED := $(foreach program,$(PATCHED_PROGRAMS),$(call c_source,$(program))) + SOURCES := $(filter-out $(EXCLUDED) $(foreach program,$(SKIPPED_BULK_PROGRAMS),$(call c_source,$(program))),$(ALL_SOURCES)) +else + SOURCES := $(filter-out $(foreach program,$(SKIPPED_BULK_PROGRAMS),$(call c_source,$(program))),$(ALL_SOURCES)) +endif + +# Program names (without .c extension) +ifeq ($(wildcard $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a),) + CUSTOM_WASM_PROG_NAMES := +else + CUSTOM_WASM_PROG_NAMES := curl wget grep sqlite3 tree zip unzip +endif + +WASM_PROG_NAMES := $(sort $(basename $(notdir $(SOURCES))) $(CUSTOM_WASM_PROG_NAMES)) +NATIVE_PROG_NAMES := $(basename $(notdir $(SOURCES))) +PROGRAM_REPORT_NAMES := $(WASM_PROG_NAMES) + +# Default WASM output targets +EXTRA_WASM_OUTPUTS := +WASM_OUTPUTS := $(addprefix $(BUILD_DIR)/,$(WASM_PROG_NAMES)) $(EXTRA_WASM_OUTPUTS) +# Native output targets +NATIVE_OUTPUTS := $(addprefix $(NATIVE_DIR)/,$(NATIVE_PROG_NAMES)) + +.SECONDEXPANSION: + +.PHONY: all wasi-sdk programs sysroot install native clean wasm-opt-check os-test os-test-native + +HAS_WASM_OPT := $(shell command -v wasm-opt >/dev/null 2>&1 && echo 1 || echo 0) + +all: programs + +# --- wasi-sdk download and cache --- + +wasi-sdk: $(WASI_SDK_DIR)/bin/clang + +$(WASI_SDK_DIR)/bin/clang: + @echo "=== Downloading wasi-sdk $(WASI_SDK_MAJOR) ===" + @mkdir -p $(VENDOR_DIR) + @if command -v curl >/dev/null 2>&1; then \ + curl -fSL "$(WASI_SDK_URL)" -o "$(VENDOR_DIR)/$(WASI_SDK_TARBALL)"; \ + elif command -v wget >/dev/null 2>&1; then \ + wget -q "$(WASI_SDK_URL)" -O "$(VENDOR_DIR)/$(WASI_SDK_TARBALL)"; \ + else \ + echo "Error: neither curl nor wget found"; exit 1; \ + fi + @echo "Extracting..." + @tar -xzf "$(VENDOR_DIR)/$(WASI_SDK_TARBALL)" -C "$(VENDOR_DIR)" + @mv "$(VENDOR_DIR)/wasi-sdk-$(WASI_SDK_MAJOR)-$(WASI_SDK_ARCH)-$(WASI_SDK_PLATFORM)" "$(WASI_SDK_DIR)" + @rm -f "$(VENDOR_DIR)/$(WASI_SDK_TARBALL)" + @echo "wasi-sdk $(WASI_SDK_MAJOR) installed to $(WASI_SDK_DIR)" + +# --- C library dependencies (downloaded at build time) --- +# Unmodified upstream libs are fetched from their official sources. +# Modified libs (curl with WASI patches) are fetched from rivet-dev forks. +# All downloads cached in libs/ — add libs/ to .gitignore. + +SQLITE3_URL := https://www.sqlite.org/2024/sqlite-amalgamation-3470200.zip +CJSON_URL := https://github.com/DaveGamble/cJSON/archive/refs/tags/v1.7.18.zip +CURL_COMMIT := main +CURL_FORK_REPO_PREFIX := secure +CURL_FORK_REPO_SUFFIX := exec-curl +CURL_FORK_REPO := $(CURL_FORK_REPO_PREFIX)-$(CURL_FORK_REPO_SUFFIX) +CURL_URL := https://github.com/rivet-dev/$(CURL_FORK_REPO)/archive/refs/heads/$(CURL_COMMIT).zip +CURL_TOOL_VERSION := 8_11_1 +CURL_TOOL_URL := https://github.com/curl/curl/archive/refs/tags/curl-$(CURL_TOOL_VERSION).tar.gz +CURL_RELEASE_VERSION := 8.11.1 +CURL_RELEASE_TAG := 8_11_1 +CURL_RELEASE_URL := https://github.com/curl/curl/releases/download/curl-$(CURL_RELEASE_TAG)/curl-$(CURL_RELEASE_VERSION).tar.xz +CURL_UPSTREAM_BUILD_DIR := $(BUILD_DIR)/curl-upstream +# Reusable libcurl artifact (headers + libcurl.a) installed by the curl build so +# git's git-remote-http links the same overlaid, mbedTLS-linked libcurl. +CURL_LIBCURL_PREFIX := $(CURL_UPSTREAM_BUILD_DIR)/install +CURL_LIBCURL_ARTIFACT := $(CURL_LIBCURL_PREFIX)/lib/libcurl.a +CURL_UPSTREAM_OVERLAY_DIR := ../../software/curl/native/c/overlay +CURL_UPSTREAM_OVERLAY_FILES := $(wildcard $(CURL_UPSTREAM_OVERLAY_DIR)/lib/*.c $(CURL_UPSTREAM_OVERLAY_DIR)/lib/*.h $(CURL_UPSTREAM_OVERLAY_DIR)/lib/vtls/*.c $(CURL_UPSTREAM_OVERLAY_DIR)/lib/vtls/*.h) +ZLIB_VERSION := 1.3.1 +ZLIB_URL := https://zlib.net/zlib-$(ZLIB_VERSION).tar.gz +ZLIB_BUILD_DIR := $(BUILD_DIR)/zlib +# mbedTLS 3.6 LTS — in-guest TLS for curl/wget/git (see +# docs-internal/networking-parity-spec.md). Use the prepared release tarball +# (`mbedtls-.tar.bz2`), which bundles the generated sources and the +# `framework` submodule; the bare GitHub source archive does not. Everything +# ships under library/ in the 3.6 LTS line (crypto is not yet split into a +# separate tf-psa-crypto tree as on 4.x/master), so `make lib` needs no +# submodule fetch. We cross-compile every library/*.c against the wasi-sdk +# sysroot, mirroring the zlib rule, plus a getentropy() entropy shim. +MBEDTLS_VERSION := 3.6.2 +MBEDTLS_URL := https://github.com/Mbed-TLS/mbedtls/releases/download/mbedtls-$(MBEDTLS_VERSION)/mbedtls-$(MBEDTLS_VERSION).tar.bz2 +# Deferred `=`: LIBS_CACHE is defined further down, so expand it lazily. +MBEDTLS_SRC_DIR = $(LIBS_CACHE)/mbedtls-$(MBEDTLS_VERSION) +MBEDTLS_BUILD_DIR := $(BUILD_DIR)/mbedtls +MBEDTLS_CONFIG_DIR := mbedtls +MBEDTLS_LIBS := $(MBEDTLS_BUILD_DIR)/libmbedtls.a $(MBEDTLS_BUILD_DIR)/libmbedx509.a $(MBEDTLS_BUILD_DIR)/libmbedcrypto.a +WGET_VERSION := 1.24.5 +WGET_URL := https://ftp.gnu.org/gnu/wget/wget-$(WGET_VERSION).tar.gz +WGET_UPSTREAM_BUILD_DIR := $(BUILD_DIR)/wget-upstream +WGET_UPSTREAM_OVERLAY_INCLUDE_DIR := overlays/wget/include +# mbedTLS backend overlay (mirrors the curl overlay layout): src/wasi_ssl.c +# is copied into the wget source tree at configure time. +WGET_UPSTREAM_OVERLAY_DIR := ../../software/wget/native/c/overlay +WGET_UPSTREAM_OVERLAY_FILES := $(wildcard $(WGET_UPSTREAM_OVERLAY_INCLUDE_DIR)/*.h) \ + $(shell find $(WGET_UPSTREAM_OVERLAY_DIR) -type f 2>/dev/null) +GIT_VERSION := 2.55.0 +GIT_URL := https://www.kernel.org/pub/software/scm/git/git-$(GIT_VERSION).tar.xz +GIT_UPSTREAM_BUILD_DIR := $(BUILD_DIR)/git-upstream +GIT_PATCH_DIR := patches/git +GREP_VERSION := 3.12 +GREP_URL := https://ftp.gnu.org/gnu/grep/grep-$(GREP_VERSION).tar.xz +GREP_UPSTREAM_BUILD_DIR := $(BUILD_DIR)/grep-upstream +INFOZIP_ZIP_VERSION := 30 +INFOZIP_ZIP_URL := https://downloads.sourceforge.net/infozip/zip$(INFOZIP_ZIP_VERSION).tar.gz +INFOZIP_ZIP_BUILD_DIR := $(BUILD_DIR)/infozip-zip-upstream +INFOZIP_UNZIP_VERSION := 60 +INFOZIP_UNZIP_URL := https://downloads.sourceforge.net/infozip/unzip$(INFOZIP_UNZIP_VERSION).tar.gz +INFOZIP_UNZIP_BUILD_DIR := $(BUILD_DIR)/infozip-unzip-upstream +TREE_VERSION := 2.3.2 +TREE_URL := https://gitlab.com/OldManProgrammer/unix-tree/-/archive/$(TREE_VERSION)/unix-tree-$(TREE_VERSION).tar.gz +# duckdb-wasm currently documents DuckDB v1.5.0 in its README. We build the +# matching upstream DuckDB tag ourselves with our patched WASI/POSIX sysroot. +DUCKDB_VERSION := v1.5.0 +DUCKDB_GIT_DESCRIBE := $(DUCKDB_VERSION)-0-g0123456789 +DUCKDB_URL := https://github.com/duckdb/duckdb/archive/refs/tags/$(DUCKDB_VERSION).tar.gz + +LIBS_DIR := libs +LIBS_CACHE := .cache/libs + +TREE_SOURCES := color.c file.c filter.c hash.c html.c info.c json.c list.c tree.c unix.c util.c xml.c strverscmp.c +TREE_FILES := $(addprefix $(LIBS_DIR)/tree/,$(TREE_SOURCES) tree.h) +TREE_WASM_FLAGS := -DLARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64 + +libs/sqlite3/sqlite3.c libs/sqlite3/sqlite3.h libs/sqlite3/shell.c: + @echo "Fetching sqlite3..." + @mkdir -p $(LIBS_CACHE) $(LIBS_DIR)/sqlite3 + @curl -fSL "$(SQLITE3_URL)" -o "$(LIBS_CACHE)/sqlite3.zip" + @cd $(LIBS_CACHE) && unzip -qo sqlite3.zip + @cp $(LIBS_CACHE)/sqlite-amalgamation-*/sqlite3.c \ + $(LIBS_CACHE)/sqlite-amalgamation-*/sqlite3.h \ + $(LIBS_CACHE)/sqlite-amalgamation-*/shell.c \ + $(LIBS_DIR)/sqlite3/ + +$(TREE_FILES): $(LIBS_DIR)/tree/.fetched + +$(LIBS_DIR)/tree/.fetched: + @echo "Fetching upstream tree $(TREE_VERSION)..." + @mkdir -p $(LIBS_CACHE) $(LIBS_DIR)/tree + @curl -fSL "$(TREE_URL)" -o "$(LIBS_CACHE)/tree-$(TREE_VERSION).tar.gz" + @rm -rf "$(LIBS_CACHE)/unix-tree-$(TREE_VERSION)" + @tar -xzf "$(LIBS_CACHE)/tree-$(TREE_VERSION).tar.gz" -C "$(LIBS_CACHE)" + @cp $(addprefix $(LIBS_CACHE)/unix-tree-$(TREE_VERSION)/,$(TREE_SOURCES) tree.h) $(LIBS_DIR)/tree/ + @touch $@ + +libs/cjson/cJSON.c: + @echo "Fetching cJSON..." + @mkdir -p $(LIBS_CACHE) $(LIBS_DIR)/cjson + @curl -fSL "$(CJSON_URL)" -o "$(LIBS_CACHE)/cjson.zip" + @cd $(LIBS_CACHE) && unzip -qo cjson.zip + @cp $(LIBS_CACHE)/cJSON-*/cJSON.c $(LIBS_CACHE)/cJSON-*/cJSON.h $(LIBS_DIR)/cjson/ + +libs/curl/lib/easy.c: + @echo "Fetching curl (rivet-dev/$(CURL_FORK_REPO))..." + @mkdir -p $(LIBS_CACHE) + @curl -fSL "$(CURL_URL)" -o "$(LIBS_CACHE)/curl.zip" + @cd $(LIBS_CACHE) && unzip -qo curl.zip + @rm -rf $(LIBS_DIR)/curl + @mv $(LIBS_CACHE)/$(CURL_FORK_REPO)-* $(LIBS_DIR)/curl + +libs/duckdb/CMakeLists.txt: + @echo "Fetching DuckDB ($(DUCKDB_VERSION))..." + @mkdir -p $(LIBS_CACHE) + @curl -fSL "$(DUCKDB_URL)" -o "$(LIBS_CACHE)/duckdb.tar.gz" + @rm -rf $(LIBS_DIR)/duckdb + @mkdir -p $(LIBS_DIR)/duckdb + @tar -xzf "$(LIBS_CACHE)/duckdb.tar.gz" --strip-components=1 -C $(LIBS_DIR)/duckdb + +.PHONY: fetch-libs fetch-fast-libs clean-libs +fetch-libs: libs/sqlite3/sqlite3.c libs/cjson/cJSON.c libs/curl/lib/easy.c libs/duckdb/CMakeLists.txt +fetch-fast-libs: libs/sqlite3/sqlite3.c libs/cjson/cJSON.c libs/curl/lib/easy.c + +clean-libs: + rm -rf $(LIBS_DIR) $(LIBS_CACHE) + +# --- os-test POSIX conformance suite (downloaded at build time) --- +# ISC-licensed test suite from the Sortix project (https://sortix.org/os-test/) + +OS_TEST_VERSION := main +OS_TEST_URL := https://gitlab.com/sortix/os-test/-/archive/$(OS_TEST_VERSION)/os-test-$(OS_TEST_VERSION).tar.gz +OS_TEST_DIR := os-test + +.PHONY: fetch-os-test clean-os-test + +fetch-os-test: $(OS_TEST_DIR)/include +$(OS_TEST_DIR)/include: + @echo "Fetching os-test $(OS_TEST_VERSION)..." + @mkdir -p $(LIBS_CACHE) + @curl -fSL "$(OS_TEST_URL)" -o "$(LIBS_CACHE)/os-test.tar.gz" + @mkdir -p $(OS_TEST_DIR) + @tar -xzf "$(LIBS_CACHE)/os-test.tar.gz" --strip-components=1 -C $(OS_TEST_DIR) + @echo "os-test $(OS_TEST_VERSION) extracted to $(OS_TEST_DIR)/" + @echo " include/ : $$(find $(OS_TEST_DIR)/include -name '*.h' 2>/dev/null | wc -l) headers" + @echo " src/ : $$(find $(OS_TEST_DIR)/src -name '*.c' 2>/dev/null | wc -l) test programs" + +clean-os-test: + rm -rf $(OS_TEST_DIR) $(LIBS_CACHE)/os-test.tar.gz + +# --- os-test POSIX conformance build targets --- +# Compile every os-test C file to WASM and native binaries. +# Individual compile failures are expected (missing headers, unsupported features) +# and do not abort the build. + +OS_TEST_BUILD := $(BUILD_DIR)/os-test +OS_TEST_NATIVE_BUILD := $(NATIVE_DIR)/os-test +OS_TEST_WASM_CFLAGS := --target=wasm32-wasip1 --sysroot=$(SYSROOT) -O0 \ + -D_GNU_SOURCE -D_BSD_SOURCE -D_ALL_SOURCE -D_DEFAULT_SOURCE +OS_TEST_WASM_LDFLAGS := -lc-printscan-long-double +# Stub main() for namespace/ tests (compile-only header conformance checks) +OS_TEST_NS_MAIN := os-test-overrides/namespace_main.c +OS_TEST_NATIVE_CFLAGS := -O0 -g \ + -D_GNU_SOURCE -D_BSD_SOURCE -D_ALL_SOURCE -D_DEFAULT_SOURCE +OS_TEST_NATIVE_LDFLAGS := -lm +ifeq ($(UNAME_S),Linux) + OS_TEST_NATIVE_LDFLAGS += -lpthread -lrt +endif +ifeq ($(UNAME_S),Darwin) + OS_TEST_NATIVE_LDFLAGS += -lpthread +endif + +.PHONY: os-test os-test-native + +os-test: wasi-sdk fetch-os-test + @total=0; pass=0; fail=0; \ + for src in $$(find $(OS_TEST_DIR) -name '*.c' -not -path '*/.expect/*' -not -path '*/misc/*' | sort); do \ + rel=$${src#$(OS_TEST_DIR)/}; \ + name=$${rel%.c}; \ + out=$(OS_TEST_BUILD)/$$name; \ + mkdir -p "$$(dirname "$$out")"; \ + total=$$((total + 1)); \ + extras=""; \ + case "$$rel" in namespace/*) extras="$(OS_TEST_NS_MAIN)";; esac; \ + if $(CC) $(OS_TEST_WASM_CFLAGS) -o "$$out" "$$src" $$extras $(OS_TEST_WASM_LDFLAGS) 2>/dev/null; then \ + pass=$$((pass + 1)); \ + else \ + echo "COMPILE FAILED: $$name"; \ + rm -f "$$out"; \ + fail=$$((fail + 1)); \ + fi; \ + done; \ + echo ""; \ + echo "=== os-test WASM Build Report ==="; \ + echo "Total: $$total"; \ + echo "Compiled: $$pass"; \ + echo "Failed: $$fail"; \ + echo "Output: $(OS_TEST_BUILD)/"; \ + echo "=== Build complete ==="; \ + if [ $$fail -gt 0 ]; then echo "ERROR: $$fail tests failed to compile" >&2; exit 1; fi + +os-test-native: fetch-os-test + @total=0; pass=0; fail=0; \ + for src in $$(find $(OS_TEST_DIR) -name '*.c' -not -path '*/.expect/*' -not -path '*/misc/*' | sort); do \ + rel=$${src#$(OS_TEST_DIR)/}; \ + out=$(OS_TEST_NATIVE_BUILD)/$${rel%.c}; \ + mkdir -p "$$(dirname "$$out")"; \ + total=$$((total + 1)); \ + extras=""; \ + case "$$rel" in namespace/*) extras="$(OS_TEST_NS_MAIN)";; esac; \ + if $(NATIVE_CC) $(OS_TEST_NATIVE_CFLAGS) -o "$$out" "$$src" $$extras $(OS_TEST_NATIVE_LDFLAGS) 2>/dev/null; then \ + pass=$$((pass + 1)); \ + else \ + rm -f "$$out"; \ + fail=$$((fail + 1)); \ + fi; \ + done; \ + echo ""; \ + echo "=== os-test Native Build Report ==="; \ + echo "Total: $$total"; \ + echo "Compiled: $$pass"; \ + echo "Failed: $$fail"; \ + echo "Output: $(OS_TEST_NATIVE_BUILD)/"; \ + echo "=== Build complete ===" + +# --- musl libc-test conformance suite (downloaded at build time) --- +# MIT-licensed test suite from musl (Bytecode Alliance mirror) + +LIBC_TEST_VERSION := master +LIBC_TEST_URL := https://github.com/bytecodealliance/libc-test/archive/refs/heads/$(LIBC_TEST_VERSION).tar.gz +LIBC_TEST_DIR := libc-test + +.PHONY: fetch-libc-test clean-libc-test + +fetch-libc-test: $(LIBC_TEST_DIR)/src +$(LIBC_TEST_DIR)/src: + @echo "Fetching musl libc-test $(LIBC_TEST_VERSION)..." + @mkdir -p $(LIBS_CACHE) + @curl -fSL "$(LIBC_TEST_URL)" -o "$(LIBS_CACHE)/libc-test.tar.gz" + @mkdir -p $(LIBC_TEST_DIR) + @tar -xzf "$(LIBS_CACHE)/libc-test.tar.gz" --strip-components=1 -C $(LIBC_TEST_DIR) + @echo "libc-test $(LIBC_TEST_VERSION) extracted to $(LIBC_TEST_DIR)/" + @echo " src/ : $$(find $(LIBC_TEST_DIR)/src -name '*.c' 2>/dev/null | wc -l) test programs" + +clean-libc-test: + rm -rf $(LIBC_TEST_DIR) $(LIBS_CACHE)/libc-test.tar.gz + +# --- musl libc-test build targets --- +# Compile functional/ and regression/ tests to WASM and native binaries. +# Tests requiring fork/exec/signals/pthreads will fail to compile for WASM — that's expected. +# The test.h framework (src/common/) is compiled into a static library first. + +LIBC_TEST_BUILD := $(BUILD_DIR)/libc-test +LIBC_TEST_NATIVE_BUILD := $(NATIVE_DIR)/libc-test +LIBC_TEST_WASM_CFLAGS := --target=wasm32-wasip1 --sysroot=$(SYSROOT) -O0 \ + -D_GNU_SOURCE -D_BSD_SOURCE -D_ALL_SOURCE -D_DEFAULT_SOURCE \ + -I$(LIBC_TEST_DIR)/src/common +LIBC_TEST_WASM_LDFLAGS := -lc-printscan-long-double -lm +LIBC_TEST_NATIVE_CFLAGS := -O0 -g \ + -D_GNU_SOURCE -D_BSD_SOURCE -D_ALL_SOURCE -D_DEFAULT_SOURCE \ + -I$(LIBC_TEST_DIR)/src/common +LIBC_TEST_NATIVE_LDFLAGS := -lm +ifeq ($(UNAME_S),Linux) + LIBC_TEST_NATIVE_LDFLAGS += -lpthread -lrt -lcrypt -lresolv -lutil -ldl +endif +ifeq ($(UNAME_S),Darwin) + LIBC_TEST_NATIVE_LDFLAGS += -lpthread +endif + +# Test framework sources compiled alongside each test +LIBC_TEST_COMMON_SRCS := $(LIBC_TEST_DIR)/src/common/print.c + +.PHONY: libc-test libc-test-native + +libc-test: wasi-sdk fetch-libc-test + @total=0; pass=0; fail=0; \ + for src in $$(find $(LIBC_TEST_DIR)/src/functional $(LIBC_TEST_DIR)/src/regression -name '*.c' 2>/dev/null | sort); do \ + rel=$${src#$(LIBC_TEST_DIR)/src/}; \ + name=$${rel%.c}; \ + out=$(LIBC_TEST_BUILD)/$$name; \ + mkdir -p "$$(dirname "$$out")"; \ + total=$$((total + 1)); \ + if $(CC) $(LIBC_TEST_WASM_CFLAGS) -o "$$out" "$$src" $(LIBC_TEST_COMMON_SRCS) $(LIBC_TEST_WASM_LDFLAGS) 2>/dev/null; then \ + pass=$$((pass + 1)); \ + else \ + rm -f "$$out"; \ + fail=$$((fail + 1)); \ + fi; \ + done; \ + echo ""; \ + echo "=== libc-test WASM Build Report ==="; \ + echo "Total: $$total"; \ + echo "Compiled: $$pass"; \ + echo "Failed: $$fail"; \ + echo "Output: $(LIBC_TEST_BUILD)/"; \ + echo "=== Build complete ===" + +libc-test-native: fetch-libc-test + @total=0; pass=0; fail=0; \ + for src in $$(find $(LIBC_TEST_DIR)/src/functional $(LIBC_TEST_DIR)/src/regression -name '*.c' 2>/dev/null | sort); do \ + rel=$${src#$(LIBC_TEST_DIR)/src/}; \ + out=$(LIBC_TEST_NATIVE_BUILD)/$${rel%.c}; \ + mkdir -p "$$(dirname "$$out")"; \ + total=$$((total + 1)); \ + if $(NATIVE_CC) $(LIBC_TEST_NATIVE_CFLAGS) -o "$$out" "$$src" $(LIBC_TEST_COMMON_SRCS) $(LIBC_TEST_NATIVE_LDFLAGS) 2>/dev/null; then \ + pass=$$((pass + 1)); \ + else \ + rm -f "$$out"; \ + fail=$$((fail + 1)); \ + fi; \ + done; \ + echo ""; \ + echo "=== libc-test Native Build Report ==="; \ + echo "Total: $$total"; \ + echo "Compiled: $$pass"; \ + echo "Failed: $$fail"; \ + echo "Output: $(LIBC_TEST_NATIVE_BUILD)/"; \ + echo "=== Build complete ===" + +# --- Patched sysroot (delegates to patch-wasi-libc.sh) --- + +WASI_LIBC_PATCHES := $(wildcard ../std-patches/wasi-libc/*.patch) +WASI_LIBC_OVERRIDES := $(wildcard ../std-patches/wasi-libc-overrides/*.c) +LLVM_RUNTIME_PATCHES := $(wildcard patches/llvm-project/*.patch) + +$(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a: wasi-sdk ../scripts/patch-wasi-libc.sh scripts/build-llvm-runtimes.sh $(WASI_LIBC_PATCHES) $(WASI_LIBC_OVERRIDES) $(LLVM_RUNTIME_PATCHES) + ../scripts/patch-wasi-libc.sh + +sysroot: $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a + +# --- wasm-opt check --- + +wasm-opt-check: + @if [ "$(HAS_WASM_OPT)" = "1" ]; then \ + echo "wasm-opt found: $$(wasm-opt --version | head -1)"; \ + else \ + echo "Warning: wasm-opt (binaryen) is not installed; skipping WASM optimization."; \ + fi + +# --- Compile all C programs to WASM --- + +programs: wasi-sdk fetch-fast-libs wasm-opt-check $(WASM_OUTPUTS) + @echo "" + @echo "=== C WASM Build Report ===" + @echo "Programs: $(words $(PROGRAM_REPORT_NAMES)) compiled" + @echo "Sysroot: $(SYSROOT)" + @echo "Output: $(BUILD_DIR)/" + @echo "=== Build complete ===" + +# Default rule: single-file programs +$(BUILD_DIR)/%: $$(call c_source,$$*) $(WASI_SDK_DIR)/bin/clang + @mkdir -p $(BUILD_DIR) + $(CC) $(WASM_CFLAGS) -o $@.wasm $< + @if [ "$(HAS_WASM_OPT)" = "1" ]; then \ + wasm-opt -O3 --strip-debug $@.wasm -o $@; \ + else \ + cp $@.wasm $@; \ + fi + @rm -f $@.wasm + +# --- Per-program override rules for multi-file compilation --- +# Add overrides here for programs that link vendored libraries. + +# json_parse: links cJSON library +$(BUILD_DIR)/json_parse: $(call c_source,json_parse) libs/cjson/cJSON.c $(WASI_SDK_DIR)/bin/clang + @mkdir -p $(BUILD_DIR) + $(CC) $(WASM_CFLAGS) -Ilibs/cjson -o $@.wasm $(call c_source,json_parse) libs/cjson/cJSON.c + @if [ "$(HAS_WASM_OPT)" = "1" ]; then \ + wasm-opt -O3 --strip-debug $@.wasm -o $@; \ + else \ + cp $@.wasm $@; \ + fi + @rm -f $@.wasm + +$(NATIVE_DIR)/json_parse: $(call c_source,json_parse) libs/cjson/cJSON.c + @mkdir -p $(NATIVE_DIR) + $(NATIVE_CC) $(NATIVE_CFLAGS) -Ilibs/cjson -o $@ $(call c_source,json_parse) libs/cjson/cJSON.c + +# sqlite3_mem: links SQLite amalgamation +SQLITE_COMMON := -DSQLITE_OMIT_WAL -DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_THREADSAFE=0 \ + -DSQLITE_OMIT_LOCALTIME +SQLITE_WASM := -DSQLITE_OS_OTHER $(SQLITE_COMMON) -lwasi-emulated-signal -lwasi-emulated-mman +SQLITE_NATIVE := $(SQLITE_COMMON) +SQLITE_SHELL_COMMON := -DSQLITE_OMIT_WAL -DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_THREADSAFE=0 \ + -DSQLITE_OMIT_LOCALTIME -DSQLITE_NOHAVE_SYSTEM -DHAVE_READLINE=0 +SQLITE_SHELL_WASM := $(SQLITE_SHELL_COMMON) -D_WASI_EMULATED_PROCESS_CLOCKS \ + -lwasi-emulated-signal -lwasi-emulated-mman -lwasi-emulated-process-clocks +SQLITE_SHELL_NATIVE := $(SQLITE_SHELL_COMMON) + +$(BUILD_DIR)/sqlite3_mem: $(call c_source,sqlite3_mem) libs/sqlite3/sqlite3.c $(WASI_SDK_DIR)/bin/clang + @mkdir -p $(BUILD_DIR) + $(CC) --target=wasm32-wasip1 --sysroot=$(SYSROOT) -Os -I include/ \ + $(SQLITE_WASM) -Ilibs/sqlite3 -Wl,--initial-memory=16777216 -o $@ \ + $(call c_source,sqlite3_mem) libs/sqlite3/sqlite3.c + +$(NATIVE_DIR)/sqlite3_mem: $(call c_source,sqlite3_mem) libs/sqlite3/sqlite3.c + @mkdir -p $(NATIVE_DIR) + $(NATIVE_CC) $(NATIVE_CFLAGS) $(SQLITE_NATIVE) -Ilibs/sqlite3 -o $@ \ + $(call c_source,sqlite3_mem) libs/sqlite3/sqlite3.c -lm -lpthread + +# sqlite3: full official CLI, links SQLite amalgamation. +# Uses -Os (not -O2) + no wasm-opt — higher optimization breaks SQLite's +# internal function pointer tables in WASM indirect call table. +# Uses --initial-memory=16777216 (16MB) for SQLite's malloc requirements. +$(BUILD_DIR)/sqlite3: libs/sqlite3/shell.c libs/sqlite3/sqlite3.c $(WASI_SDK_DIR)/bin/clang + @mkdir -p $(BUILD_DIR) + $(CC) --target=wasm32-wasip1 --sysroot=$(SYSROOT) -Os -I include/ \ + $(SQLITE_SHELL_WASM) -Ilibs/sqlite3 -Wl,--initial-memory=16777216 \ + -o $@ libs/sqlite3/shell.c libs/sqlite3/sqlite3.c + +$(BUILD_DIR)/sqlite3_cli: $(BUILD_DIR)/sqlite3 + cp $< $@ + +$(NATIVE_DIR)/sqlite3: libs/sqlite3/shell.c libs/sqlite3/sqlite3.c + @mkdir -p $(NATIVE_DIR) + $(NATIVE_CC) $(NATIVE_CFLAGS) $(SQLITE_SHELL_NATIVE) -Ilibs/sqlite3 -o $@ \ + libs/sqlite3/shell.c libs/sqlite3/sqlite3.c -lm -lpthread + +$(LIBS_CACHE)/zlib-$(ZLIB_VERSION)/zlib.h: + @echo "Fetching zlib $(ZLIB_VERSION)..." + @mkdir -p $(LIBS_CACHE) + @curl -fSL "$(ZLIB_URL)" -o "$(LIBS_CACHE)/zlib-$(ZLIB_VERSION).tar.gz" + @rm -rf "$(LIBS_CACHE)/zlib-$(ZLIB_VERSION)" + @tar -xzf "$(LIBS_CACHE)/zlib-$(ZLIB_VERSION).tar.gz" -C "$(LIBS_CACHE)" + +$(ZLIB_BUILD_DIR)/libz.a: $(LIBS_CACHE)/zlib-$(ZLIB_VERSION)/zlib.h $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a $(WASI_SDK_DIR)/bin/clang + @mkdir -p $(ZLIB_BUILD_DIR) + @for src in adler32.c crc32.c deflate.c infback.c inffast.c inflate.c inftrees.c trees.c zutil.c compress.c uncompr.c gzclose.c gzlib.c gzread.c gzwrite.c; do \ + $(CC) --target=wasm32-wasip1 --sysroot=$(SYSROOT) -O2 -DHAVE_UNISTD_H \ + -I$(LIBS_CACHE)/zlib-$(ZLIB_VERSION) \ + -c "$(LIBS_CACHE)/zlib-$(ZLIB_VERSION)/$$src" \ + -o "$(ZLIB_BUILD_DIR)/$${src%.c}.o"; \ + done + $(WASI_SDK_DIR)/bin/llvm-ar rcs $@ $(ZLIB_BUILD_DIR)/*.o + +# --- mbedTLS 3.6 LTS for wasm32-wasip1 --- +# Fetch/extract the prepared release tarball, then cross-compile every +# library/*.c (plus the getentropy entropy shim) against the wasi-sdk sysroot, +# splitting objects into the three canonical archives by filename the same way +# mbedTLS's own Makefile does (crypto / x509 / tls). Single-threaded; entropy +# via getentropy() (MBEDTLS_ENTROPY_HARDWARE_ALT); TLS 1.3 on (default in 3.6). +$(MBEDTLS_SRC_DIR)/library/ssl_tls.c: + @echo "Fetching mbedTLS $(MBEDTLS_VERSION)..." + @mkdir -p $(LIBS_CACHE) + @curl -fSL "$(MBEDTLS_URL)" -o "$(LIBS_CACHE)/mbedtls-$(MBEDTLS_VERSION).tar.bz2" + @rm -rf "$(MBEDTLS_SRC_DIR)" + @tar -xjf "$(LIBS_CACHE)/mbedtls-$(MBEDTLS_VERSION).tar.bz2" -C "$(LIBS_CACHE)" + +.PHONY: mbedtls +mbedtls: $(MBEDTLS_LIBS) + +$(MBEDTLS_LIBS): $(MBEDTLS_SRC_DIR)/library/ssl_tls.c \ + $(MBEDTLS_CONFIG_DIR)/wasi_user_config.h \ + $(MBEDTLS_CONFIG_DIR)/mbedtls_wasi_entropy.c \ + $(WASI_SDK_DIR)/bin/clang + @echo "=== Building mbedTLS $(MBEDTLS_VERSION) for wasm32-wasip1 ===" + @mkdir -p $(MBEDTLS_BUILD_DIR)/crypto $(MBEDTLS_BUILD_DIR)/x509 $(MBEDTLS_BUILD_DIR)/tls + @for src in $(MBEDTLS_SRC_DIR)/library/*.c $(MBEDTLS_CONFIG_DIR)/mbedtls_wasi_entropy.c; do \ + name=$$(basename "$${src%.c}"); \ + case "$$name" in \ + x509*|pkcs7) grp=x509 ;; \ + ssl_*|ssl|mps_*|debug|net_sockets) grp=tls ;; \ + *) grp=crypto ;; \ + esac; \ + $(CC) --target=wasm32-wasip1 --sysroot=$(SYSROOT) -O2 \ + -I$(MBEDTLS_SRC_DIR)/include -I$(MBEDTLS_CONFIG_DIR) \ + -DMBEDTLS_USER_CONFIG_FILE='"wasi_user_config.h"' \ + -c "$$src" -o "$(MBEDTLS_BUILD_DIR)/$$grp/$$name.o" || exit 1; \ + done + @rm -f $(MBEDTLS_LIBS) + $(WASI_SDK_DIR)/bin/llvm-ar rcs $(MBEDTLS_BUILD_DIR)/libmbedcrypto.a $(MBEDTLS_BUILD_DIR)/crypto/*.o + $(WASI_SDK_DIR)/bin/llvm-ar rcs $(MBEDTLS_BUILD_DIR)/libmbedx509.a $(MBEDTLS_BUILD_DIR)/x509/*.o + $(WASI_SDK_DIR)/bin/llvm-ar rcs $(MBEDTLS_BUILD_DIR)/libmbedtls.a $(MBEDTLS_BUILD_DIR)/tls/*.o + @echo "=== mbedTLS libs built ===" + @ls -l $(MBEDTLS_LIBS) + +# --- brotli 1.1.0 (decoder) for wasm32-wasip1 --- +# curl's --compressed brotli decode. Only the dependency-free portable C +# `common` + `dec` sources are needed (no encoder). Mirrors the zlib rule: +# cross-compile each .c against the wasi-sdk sysroot and archive into the two +# canonical libs (libbrotlicommon.a, libbrotlidec.a; dec depends on common). +BROTLI_VERSION := 1.1.0 +BROTLI_URL := https://github.com/google/brotli/archive/refs/tags/v$(BROTLI_VERSION).tar.gz +BROTLI_SRC_DIR = $(LIBS_CACHE)/brotli-$(BROTLI_VERSION) +BROTLI_BUILD_DIR := $(BUILD_DIR)/brotli +BROTLI_INCLUDE_DIR = $(BROTLI_SRC_DIR)/c/include +BROTLI_COMMON_SRCS := constants.c context.c dictionary.c platform.c shared_dictionary.c transform.c +BROTLI_DEC_SRCS := bit_reader.c decode.c huffman.c state.c +BROTLI_LIBS := $(BROTLI_BUILD_DIR)/libbrotlidec.a $(BROTLI_BUILD_DIR)/libbrotlicommon.a + +$(BROTLI_SRC_DIR)/c/dec/decode.c: + @echo "Fetching brotli $(BROTLI_VERSION)..." + @mkdir -p $(LIBS_CACHE) + @curl -fSL "$(BROTLI_URL)" -o "$(LIBS_CACHE)/brotli-$(BROTLI_VERSION).tar.gz" + @rm -rf "$(BROTLI_SRC_DIR)" + @tar -xzf "$(LIBS_CACHE)/brotli-$(BROTLI_VERSION).tar.gz" -C "$(LIBS_CACHE)" + +.PHONY: brotli +brotli: $(BROTLI_LIBS) + +$(BROTLI_LIBS): $(BROTLI_SRC_DIR)/c/dec/decode.c $(WASI_SDK_DIR)/bin/clang + @echo "=== Building brotli $(BROTLI_VERSION) (dec) for wasm32-wasip1 ===" + @mkdir -p $(BROTLI_BUILD_DIR)/common $(BROTLI_BUILD_DIR)/dec + @for src in $(BROTLI_COMMON_SRCS); do \ + $(CC) --target=wasm32-wasip1 --sysroot=$(SYSROOT) -O2 \ + -I$(BROTLI_INCLUDE_DIR) \ + -c "$(BROTLI_SRC_DIR)/c/common/$$src" \ + -o "$(BROTLI_BUILD_DIR)/common/$${src%.c}.o" || exit 1; \ + done + @for src in $(BROTLI_DEC_SRCS); do \ + $(CC) --target=wasm32-wasip1 --sysroot=$(SYSROOT) -O2 \ + -I$(BROTLI_INCLUDE_DIR) \ + -c "$(BROTLI_SRC_DIR)/c/dec/$$src" \ + -o "$(BROTLI_BUILD_DIR)/dec/$${src%.c}.o" || exit 1; \ + done + @rm -f $(BROTLI_LIBS) + $(WASI_SDK_DIR)/bin/llvm-ar rcs $(BROTLI_BUILD_DIR)/libbrotlicommon.a $(BROTLI_BUILD_DIR)/common/*.o + $(WASI_SDK_DIR)/bin/llvm-ar rcs $(BROTLI_BUILD_DIR)/libbrotlidec.a $(BROTLI_BUILD_DIR)/dec/*.o + @echo "=== brotli libs built ===" + @ls -l $(BROTLI_LIBS) + +# --- zstd 1.5.6 (decoder) for wasm32-wasip1 --- +# curl's --compressed zstd decode. Single-threaded (ZSTD_MULTITHREAD off), +# ZSTD_DISABLE_ASM=1 so the portable C huffman path is used instead of the +# amd64 .S. Only lib/common + lib/decompress are needed for ZSTD_*DStream. +ZSTD_VERSION := 1.5.6 +ZSTD_URL := https://github.com/facebook/zstd/releases/download/v$(ZSTD_VERSION)/zstd-$(ZSTD_VERSION).tar.gz +ZSTD_SRC_DIR = $(LIBS_CACHE)/zstd-$(ZSTD_VERSION) +ZSTD_BUILD_DIR := $(BUILD_DIR)/zstd +ZSTD_INCLUDE_DIR = $(ZSTD_SRC_DIR)/lib +ZSTD_LIB := $(ZSTD_BUILD_DIR)/libzstd.a +# NB: leave ZSTD_MULTITHREAD *undefined* (not =0). zstd's threading.h keys the +# pthread path off `#if defined(ZSTD_MULTITHREAD)`, so even =0 pulls in +# pthread_create/join, which wasi-libc static-asserts away without threads. +ZSTD_CFLAGS := -O2 -DZSTD_DISABLE_ASM=1 \ + -I$(ZSTD_SRC_DIR)/lib -I$(ZSTD_SRC_DIR)/lib/common + +$(ZSTD_SRC_DIR)/lib/zstd.h: + @echo "Fetching zstd $(ZSTD_VERSION)..." + @mkdir -p $(LIBS_CACHE) + @curl -fSL "$(ZSTD_URL)" -o "$(LIBS_CACHE)/zstd-$(ZSTD_VERSION).tar.gz" + @rm -rf "$(ZSTD_SRC_DIR)" + @tar -xzf "$(LIBS_CACHE)/zstd-$(ZSTD_VERSION).tar.gz" -C "$(LIBS_CACHE)" + +.PHONY: zstd +zstd: $(ZSTD_LIB) + +$(ZSTD_LIB): $(ZSTD_SRC_DIR)/lib/zstd.h $(WASI_SDK_DIR)/bin/clang + @echo "=== Building zstd $(ZSTD_VERSION) (decompress) for wasm32-wasip1 ===" + @mkdir -p $(ZSTD_BUILD_DIR)/obj + @for src in $(ZSTD_SRC_DIR)/lib/common/*.c $(ZSTD_SRC_DIR)/lib/decompress/*.c; do \ + name=$$(basename "$${src%.c}"); \ + $(CC) --target=wasm32-wasip1 --sysroot=$(SYSROOT) $(ZSTD_CFLAGS) \ + -c "$$src" -o "$(ZSTD_BUILD_DIR)/obj/$$name.o" || exit 1; \ + done + @rm -f $(ZSTD_LIB) + $(WASI_SDK_DIR)/bin/llvm-ar rcs $(ZSTD_LIB) $(ZSTD_BUILD_DIR)/obj/*.o + @echo "=== zstd lib built ===" + @ls -l $(ZSTD_LIB) + +$(NATIVE_DIR)/sqlite3_cli: $(NATIVE_DIR)/sqlite3 + cp $< $@ + +# tree: upstream Steve Baker tree compiled from pinned C source. +$(BUILD_DIR)/tree: $(TREE_FILES) $(WASI_SDK_DIR)/bin/clang $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a + @mkdir -p $(BUILD_DIR) + $(CC) --target=wasm32-wasip1 --sysroot=$(SYSROOT) -O2 -I include/ \ + $(TREE_WASM_FLAGS) -I$(LIBS_DIR)/tree -o $@ \ + $(addprefix $(LIBS_DIR)/tree/,$(TREE_SOURCES)) + +$(NATIVE_DIR)/tree: $(TREE_FILES) + @mkdir -p $(NATIVE_DIR) + $(NATIVE_CC) $(NATIVE_CFLAGS) $(TREE_WASM_FLAGS) -I$(LIBS_DIR)/tree -o $@ \ + $(addprefix $(LIBS_DIR)/tree/,$(TREE_SOURCES)) + +# zip/unzip: upstream Info-ZIP releases built against the patched C sysroot. +$(BUILD_DIR)/zip: scripts/build-infozip-upstream.sh $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a $(WASI_SDK_DIR)/bin/clang + @mkdir -p $(BUILD_DIR) + bash scripts/build-infozip-upstream.sh \ + --tool zip \ + --version "$(INFOZIP_ZIP_VERSION)" \ + --url "$(INFOZIP_ZIP_URL)" \ + --cache-dir "$(abspath $(LIBS_CACHE))" \ + --build-dir "$(abspath $(INFOZIP_ZIP_BUILD_DIR))" \ + --cc "$(abspath $(CC)) --target=wasm32-wasip1 --sysroot=$(abspath $(SYSROOT))" \ + --output "$(abspath $@)" + +$(BUILD_DIR)/unzip: scripts/build-infozip-upstream.sh $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a $(WASI_SDK_DIR)/bin/clang + @mkdir -p $(BUILD_DIR) + bash scripts/build-infozip-upstream.sh \ + --tool unzip \ + --version "$(INFOZIP_UNZIP_VERSION)" \ + --url "$(INFOZIP_UNZIP_URL)" \ + --cache-dir "$(abspath $(LIBS_CACHE))" \ + --build-dir "$(abspath $(INFOZIP_UNZIP_BUILD_DIR))" \ + --cc "$(abspath $(CC)) --target=wasm32-wasip1 --sysroot=$(abspath $(SYSROOT))" \ + --output "$(abspath $@)" + +$(BUILD_DIR)/git: scripts/build-git-upstream.sh $(CURL_LIBCURL_ARTIFACT) $(MBEDTLS_LIBS) $(BROTLI_LIBS) $(ZSTD_LIB) $(ZLIB_BUILD_DIR)/libz.a $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a $(WASI_SDK_DIR)/bin/clang + @mkdir -p $(BUILD_DIR) + bash scripts/build-git-upstream.sh \ + --version "$(GIT_VERSION)" \ + --url "$(GIT_URL)" \ + --cache-dir "$(abspath $(LIBS_CACHE))" \ + --build-dir "$(abspath $(GIT_UPSTREAM_BUILD_DIR))" \ + --patch-dir "$(abspath $(GIT_PATCH_DIR))" \ + --zlib-dir "$(abspath $(LIBS_CACHE)/zlib-$(ZLIB_VERSION))" \ + --zlib-build-dir "$(abspath $(ZLIB_BUILD_DIR))" \ + --curl-prefix "$(abspath $(CURL_LIBCURL_PREFIX))" \ + --mbedtls-libdir "$(abspath $(MBEDTLS_BUILD_DIR))" \ + --brotli-libdir "$(abspath $(BROTLI_BUILD_DIR))" \ + --zstd-libdir "$(abspath $(ZSTD_BUILD_DIR))" \ + --cc "$(abspath $(CC))" \ + --ar "$(abspath $(WASI_SDK_DIR)/bin/llvm-ar)" \ + --ranlib "$(abspath $(WASI_SDK_DIR)/bin/llvm-ranlib)" \ + --sysroot "$(abspath $(SYSROOT))" \ + --remote-http-output "$(abspath $(BUILD_DIR)/git-remote-http)" \ + --output "$(abspath $@)" + +# git-remote-http is produced by the same git build invocation. +$(BUILD_DIR)/git-remote-http: $(BUILD_DIR)/git + @test -f $@ || { echo "Error: git-remote-http missing at $@ (git build did not produce it)"; exit 1; } + +# curl_test: links libcurl (HTTP/HTTPS build for WASM via host_net + host_tls) +CURL_SRCS := $(wildcard libs/curl/lib/*.c) $(wildcard libs/curl/lib/vauth/*.c) \ + $(wildcard libs/curl/lib/vtls/*.c) $(wildcard libs/curl/lib/vquic/*.c) \ + $(wildcard libs/curl/lib/vssh/*.c) +CURL_INCLUDES := -Ilibs/curl/include -Ilibs/curl/lib -include libs/curl/lib/curl_setup.h -include libs/curl/lib/curl_printf.h +CURL_LIB_DEFS := -DHAVE_CONFIG_H -DBUILDING_LIBCURL -D_WASI_EMULATED_SIGNAL -DHAVE_BASENAME -DHAVE_LIBGEN_H + +$(BUILD_DIR)/curl: scripts/build-curl-upstream.sh $(CURL_UPSTREAM_OVERLAY_FILES) $(MBEDTLS_LIBS) $(ZLIB_BUILD_DIR)/libz.a $(BROTLI_LIBS) $(ZSTD_LIB) $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a $(WASI_SDK_DIR)/bin/clang + @mkdir -p $(BUILD_DIR) + bash scripts/build-curl-upstream.sh \ + --version "$(CURL_RELEASE_VERSION)" \ + --tag "$(CURL_RELEASE_TAG)" \ + --url "$(CURL_RELEASE_URL)" \ + --cache-dir "$(abspath $(LIBS_CACHE))" \ + --build-dir "$(abspath $(CURL_UPSTREAM_BUILD_DIR))" \ + --overlay-dir "$(abspath $(CURL_UPSTREAM_OVERLAY_DIR))" \ + --cc "$(abspath $(CC)) --target=wasm32-wasip1 --sysroot=$(abspath $(SYSROOT))" \ + --ar "$(abspath $(WASI_SDK_DIR)/bin/llvm-ar)" \ + --ranlib "$(abspath $(WASI_SDK_DIR)/bin/llvm-ranlib)" \ + --mbedtls-include "$(abspath $(MBEDTLS_SRC_DIR)/include)" \ + --mbedtls-libdir "$(abspath $(MBEDTLS_BUILD_DIR))" \ + --zlib-include "$(abspath $(LIBS_CACHE)/zlib-$(ZLIB_VERSION))" \ + --zlib-libdir "$(abspath $(ZLIB_BUILD_DIR))" \ + --brotli-include "$(abspath $(BROTLI_INCLUDE_DIR))" \ + --brotli-libdir "$(abspath $(BROTLI_BUILD_DIR))" \ + --zstd-include "$(abspath $(ZSTD_INCLUDE_DIR))" \ + --zstd-libdir "$(abspath $(ZSTD_BUILD_DIR))" \ + --ca-bundle "/etc/ssl/certs/ca-certificates.crt" \ + --install-prefix "$(abspath $(CURL_LIBCURL_PREFIX))" \ + --output "$(abspath $@)" + +# The libcurl install artifact is produced as a side effect of the curl build. +# Declare it so git can depend on it; verify it exists after the curl target ran. +$(CURL_LIBCURL_ARTIFACT): $(BUILD_DIR)/curl + @test -f $@ || { echo "Error: libcurl artifact missing at $@ (curl build did not install it)"; exit 1; } + +$(BUILD_DIR)/wget: scripts/build-wget-upstream.sh $(WGET_UPSTREAM_OVERLAY_FILES) $(MBEDTLS_LIBS) $(ZLIB_BUILD_DIR)/libz.a $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a $(WASI_SDK_DIR)/bin/clang + @mkdir -p $(BUILD_DIR) + bash scripts/build-wget-upstream.sh \ + --version "$(WGET_VERSION)" \ + --url "$(WGET_URL)" \ + --cache-dir "$(abspath $(LIBS_CACHE))" \ + --build-dir "$(abspath $(WGET_UPSTREAM_BUILD_DIR))" \ + --overlay-include-dir "$(abspath $(WGET_UPSTREAM_OVERLAY_INCLUDE_DIR))" \ + --overlay-dir "$(abspath $(WGET_UPSTREAM_OVERLAY_DIR))" \ + --mbedtls-include "$(abspath $(MBEDTLS_SRC_DIR)/include)" \ + --mbedtls-libdir "$(abspath $(MBEDTLS_BUILD_DIR))" \ + --zlib-include "$(abspath $(LIBS_CACHE)/zlib-$(ZLIB_VERSION))" \ + --zlib-libdir "$(abspath $(ZLIB_BUILD_DIR))" \ + --cc "$(abspath $(CC)) --target=wasm32-wasip1 --sysroot=$(abspath $(SYSROOT))" \ + --ar "$(abspath $(WASI_SDK_DIR)/bin/llvm-ar)" \ + --ranlib "$(abspath $(WASI_SDK_DIR)/bin/llvm-ranlib)" \ + --output "$(abspath $@)" + +$(BUILD_DIR)/grep: scripts/build-grep-upstream.sh $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a $(WASI_SDK_DIR)/bin/clang + @mkdir -p $(BUILD_DIR) + bash scripts/build-grep-upstream.sh \ + --version "$(GREP_VERSION)" \ + --url "$(GREP_URL)" \ + --cache-dir "$(abspath $(LIBS_CACHE))" \ + --build-dir "$(abspath $(GREP_UPSTREAM_BUILD_DIR))" \ + --cc "$(abspath $(CC)) --target=wasm32-wasip1 --sysroot=$(abspath $(SYSROOT))" \ + --ar "$(abspath $(WASI_SDK_DIR)/bin/llvm-ar)" \ + --ranlib "$(abspath $(WASI_SDK_DIR)/bin/llvm-ranlib)" \ + --output "$(abspath $@)" + +# duckdb: upstream DuckDB CLI built from source with our patched WASI/POSIX sysroot +$(BUILD_DIR)/duckdb: libs/duckdb/CMakeLists.txt $(PATCHED_SYSROOT)/lib/wasm32-wasi/libc.a $(WASI_SDK_DIR)/bin/clang $(WASI_SDK_DIR)/bin/clang++ cmake/FindThreads.cmake scripts/build-duckdb.sh include/fcntl.h include/ifaddrs.h include/net/if.h include/sys/ioctl.h + @mkdir -p $(BUILD_DIR) + DUCKDB_SRC_DIR="$(abspath $(LIBS_DIR)/duckdb)" \ + DUCKDB_BUILD_DIR="$(abspath $(BUILD_DIR)/duckdb-cmake)" \ + DUCKDB_OUTPUT="$(abspath $(BUILD_DIR)/duckdb)" \ + WASI_SDK_DIR="$(abspath $(WASI_SDK_DIR))" \ + SYSROOT_DIR="$(abspath $(PATCHED_SYSROOT))" \ + MODULE_PATH="$(abspath cmake)" \ + OVERLAY_INCLUDE_DIR="$(abspath include)" \ + DUCKDB_GIT_DESCRIBE="$(DUCKDB_GIT_DESCRIBE)" \ + PATCH_DIR="$(abspath ../../software/duckdb/native/c/patches)" \ + bash scripts/build-duckdb.sh + +# --- Compile all C programs to native binaries (for parity testing) --- + +native: $(NATIVE_OUTPUTS) + @echo "" + @echo "=== Native Build Report ===" + @echo "Programs: $(words $(NATIVE_PROG_NAMES)) compiled" + @echo "Output: $(NATIVE_DIR)/" + @echo "=== Build complete ===" + +$(NATIVE_DIR)/%: $$(call c_source,$$*) + @mkdir -p $(NATIVE_DIR) + $(NATIVE_CC) $(NATIVE_CFLAGS) -o $@ $< + +# --- Install opted-in commands to COMMANDS_DIR --- + +install: + @if [ -z "$(COMMANDS)" ]; then \ + echo "No C commands opted in for install (COMMANDS is empty)"; \ + exit 0; \ + fi + @if [ ! -d "$(BUILD_DIR)" ]; then \ + echo "Error: $(BUILD_DIR) not found. Run 'make programs' first."; \ + exit 1; \ + fi + @mkdir -p $(COMMANDS_DIR) + @INSTALLED=0; \ + for cmd in $(COMMANDS); do \ + src="$$cmd"; \ + if [ -f "$(BUILD_DIR)/$$src" ]; then \ + cp "$(BUILD_DIR)/$$src" "$(COMMANDS_DIR)/$$cmd"; \ + INSTALLED=$$((INSTALLED + 1)); \ + else \ + echo "Warning: $(BUILD_DIR)/$$src not found — skipping"; \ + fi; \ + done; \ + echo "Installed $$INSTALLED C command(s) to $(COMMANDS_DIR)" + @# Git helper entry points. upload-pack/receive-pack/upload-archive ARE git + @# builtins, so they alias the git binary. The HTTPS transport lives in the + @# real git-remote-http binary (libcurl in-process); git-remote-https is an + @# alias to it (same binary, ftp/ftps schemes unsupported and unused). + @if [ -f "$(COMMANDS_DIR)/git" ]; then \ + for alias in git-upload-pack git-receive-pack git-upload-archive; do \ + ln -sf git "$(COMMANDS_DIR)/$$alias"; \ + done; \ + fi + @if [ -f "$(COMMANDS_DIR)/git-remote-http" ]; then \ + ln -sf git-remote-http "$(COMMANDS_DIR)/git-remote-https"; \ + fi + +# --- Clean --- + +clean: + rm -rf $(BUILD_DIR) + +# --- vim: real vim 9.2 -> wasm32-wasip1 against the full-OS libc --- +# +# vim compiles as a normal terminal program against the PATCHED sysroot plus a +# small bridge library (vim/): a termcap stub (builtin termcaps), and POSIX +# stubs the full-OS libc still lacks. Termios/process spawning live in sysroot. +# Recipe recovered from the original hand build (see vim/ headers). The result +# lands in $(BUILD_DIR)/vim; `make -C .. cmd/vim` installs it into the shared +# commands dir. NOTE: run as `vim -n ` docs-wise is no longer needed — +# swap files work; kept here for archaeology. +VIM_VERSION ?= v9.2.0780 +VIM_SRC := libs/vim +VIM_BRIDGE_SRC := ../../software/vim/native/c/vim-bridge +VIM_BRIDGE_BUILD := $(BUILD_DIR)/vim-bridge +VIM_BRIDGE_OBJS := $(VIM_BRIDGE_BUILD)/termcap_stub.o $(VIM_BRIDGE_BUILD)/posix_stubs.o +VIM_WCC := $(VIM_BRIDGE_BUILD)/wcc + +.PHONY: fetch-vim +fetch-vim: $(VIM_SRC)/src/vim.h +$(VIM_SRC)/src/vim.h: + git clone --depth 1 --branch $(VIM_VERSION) https://github.com/vim/vim $(VIM_SRC) + +$(VIM_BRIDGE_BUILD)/%.o: $(VIM_BRIDGE_SRC)/%.c $(WASI_SDK_DIR)/bin/clang sysroot + @mkdir -p $(VIM_BRIDGE_BUILD) + $(CC) --target=wasm32-wasip1 --sysroot=$(PATCHED_SYSROOT) -O2 -I $(VIM_BRIDGE_SRC) -c -o $@ $< + +$(VIM_BRIDGE_BUILD)/libtermcap.a: $(VIM_BRIDGE_OBJS) + rm -f $@ + $(WASI_SDK_DIR)/bin/llvm-ar rcs $@ $(VIM_BRIDGE_OBJS) + +# Wrapper compiler: every vim TU gets the sysroot, the bridge headers, and the +# forced compat include, so vim's own Makefile needs no flag surgery. +$(VIM_WCC): $(WASI_SDK_DIR)/bin/clang + @mkdir -p $(VIM_BRIDGE_BUILD) + @printf '#!/bin/sh\nexec %s --target=wasm32-wasip1 --sysroot=%s -O2 -I %s -include %s "$$@"\n' \ + "$(abspath $(WASI_SDK_DIR)/bin/clang)" \ + "$(abspath $(PATCHED_SYSROOT))" \ + "$(abspath $(VIM_BRIDGE_SRC))" \ + "$(abspath $(VIM_BRIDGE_SRC)/compat.h)" > $@ + @chmod +x $@ + +$(BUILD_DIR)/vim: fetch-vim $(VIM_BRIDGE_BUILD)/libtermcap.a $(VIM_WCC) wasm-opt-check + rm -f $(VIM_SRC)/src/auto/config.cache + cd $(VIM_SRC)/src && \ +CC="$(abspath $(VIM_WCC))" \ +LDFLAGS="-L$(abspath $(VIM_BRIDGE_BUILD))" \ + vim_cv_toupper_broken=no \ + vim_cv_terminfo=no \ + vim_cv_tgetent=zero \ + vim_cv_getcwd_broken=no \ + vim_cv_stat_ignores_slash=no \ + vim_cv_memmove_handles_overlap=yes \ +vim_cv_bcopy_handles_overlap=yes \ +vim_cv_memcpy_handles_overlap=yes \ +vim_cv_tty_group=world \ +vim_cv_tty_mode=0620 \ +ac_cv_header_dlfcn_h=no \ +./configure --host=wasm32-wasip1 --build=$$(cc -dumpmachine 2>/dev/null || echo x86_64-pc-linux-gnu) \ + --with-features=normal --with-tlib=termcap \ + --disable-gui --without-x --disable-nls --disable-channel \ + --without-wayland \ + --disable-netbeans --disable-selinux --disable-gpm --disable-sysmouse \ + --disable-icon-cache-update --disable-desktop-database-update + $(MAKE) -C $(VIM_SRC)/src vim + @mkdir -p $(BUILD_DIR) + @if [ "$(HAS_WASM_OPT)" = "1" ]; then \ + wasm-opt -O3 --strip-debug --all-features $(VIM_SRC)/src/vim -o $(BUILD_DIR)/vim; \ + else \ + cp $(VIM_SRC)/src/vim $(BUILD_DIR)/vim; \ + fi + @echo "Built $(BUILD_DIR)/vim" diff --git a/registry/native/c/cmake/FindThreads.cmake b/toolchain/c/cmake/FindThreads.cmake similarity index 100% rename from registry/native/c/cmake/FindThreads.cmake rename to toolchain/c/cmake/FindThreads.cmake diff --git a/registry/native/c/include/fcntl.h b/toolchain/c/include/fcntl.h similarity index 100% rename from registry/native/c/include/fcntl.h rename to toolchain/c/include/fcntl.h diff --git a/registry/native/c/include/ifaddrs.h b/toolchain/c/include/ifaddrs.h similarity index 100% rename from registry/native/c/include/ifaddrs.h rename to toolchain/c/include/ifaddrs.h diff --git a/registry/native/c/include/net/if.h b/toolchain/c/include/net/if.h similarity index 100% rename from registry/native/c/include/net/if.h rename to toolchain/c/include/net/if.h diff --git a/registry/native/c/include/posix_spawn_compat.h b/toolchain/c/include/posix_spawn_compat.h similarity index 92% rename from registry/native/c/include/posix_spawn_compat.h rename to toolchain/c/include/posix_spawn_compat.h index 6186535302..38c6333bfc 100644 --- a/registry/native/c/include/posix_spawn_compat.h +++ b/toolchain/c/include/posix_spawn_compat.h @@ -23,6 +23,11 @@ typedef struct { int __dummy; } posix_spawnattr_t; /* --- posix_spawn ------------------------------------------------------ */ +int posix_spawn(pid_t *restrict, const char *restrict, + const posix_spawn_file_actions_t *, + const posix_spawnattr_t *restrict, + char *const[restrict], char *const[restrict]); + int posix_spawnp(pid_t *restrict, const char *restrict, const posix_spawn_file_actions_t *, const posix_spawnattr_t *restrict, diff --git a/registry/native/c/include/sched.h b/toolchain/c/include/sched.h similarity index 100% rename from registry/native/c/include/sched.h rename to toolchain/c/include/sched.h diff --git a/registry/native/c/include/sys/ioctl.h b/toolchain/c/include/sys/ioctl.h similarity index 100% rename from registry/native/c/include/sys/ioctl.h rename to toolchain/c/include/sys/ioctl.h diff --git a/toolchain/c/mbedtls/mbedtls_wasi_entropy.c b/toolchain/c/mbedtls/mbedtls_wasi_entropy.c new file mode 100644 index 0000000000..ac76210218 --- /dev/null +++ b/toolchain/c/mbedtls/mbedtls_wasi_entropy.c @@ -0,0 +1,60 @@ +/* + * Hardware entropy shim for the wasm32-wasip1 (WASI) mbedTLS build. + * + * The WASI build sets MBEDTLS_NO_PLATFORM_ENTROPY (no /dev/urandom fopen path) + * and MBEDTLS_ENTROPY_HARDWARE_ALT, which routes the entropy module through + * this mbedtls_hardware_poll() implementation. We satisfy it with getentropy(), + * the WASI random primitive (backed by the kernel's random device layer) that + * the repo's host-side wasi_tls.c already uses. getentropy() fills at most 256 + * bytes per call, so we loop for larger requests. + */ + +#include +#include /* clock_gettime(), CLOCK_MONOTONIC */ +#include /* getentropy() — declared by wasi-libc */ + +#include "mbedtls/build_info.h" +#include "mbedtls/platform_util.h" /* mbedtls_ms_time_t */ + +/* + * WASI mbedtls_ms_time() over clock_gettime(CLOCK_MONOTONIC). Enabled by + * MBEDTLS_PLATFORM_MS_TIME_ALT in the WASI user config (see wasi_user_config.h), + * which suppresses mbedTLS's own platform-detected definition. + */ +#if defined(MBEDTLS_PLATFORM_MS_TIME_ALT) +mbedtls_ms_time_t mbedtls_ms_time(void) +{ + struct timespec tv; + if (clock_gettime(CLOCK_MONOTONIC, &tv) != 0) { + return (mbedtls_ms_time_t) time(NULL) * 1000; + } + return (mbedtls_ms_time_t) tv.tv_sec * 1000 + + (mbedtls_ms_time_t) (tv.tv_nsec / 1000000); +} +#endif /* MBEDTLS_PLATFORM_MS_TIME_ALT */ + +int mbedtls_hardware_poll(void *data, unsigned char *output, size_t len, + size_t *olen) +{ + (void)data; + + size_t filled = 0; + while (filled < len) { + size_t chunk = len - filled; + if (chunk > 256) { + chunk = 256; /* getentropy() rejects requests larger than 256. */ + } + if (getentropy(output + filled, chunk) != 0) { + if (olen != NULL) { + *olen = filled; + } + return -1; /* MBEDTLS_ERR_ENTROPY_SOURCE_FAILED at the call site. */ + } + filled += chunk; + } + + if (olen != NULL) { + *olen = len; + } + return 0; +} diff --git a/toolchain/c/mbedtls/wasi_user_config.h b/toolchain/c/mbedtls/wasi_user_config.h new file mode 100644 index 0000000000..88b22cb823 --- /dev/null +++ b/toolchain/c/mbedtls/wasi_user_config.h @@ -0,0 +1,44 @@ +/* + * mbedTLS user-config overrides for the wasm32-wasip1 (WASI) build. + * + * This file is included via -DMBEDTLS_USER_CONFIG_FILE after mbedTLS's own + * default `mbedtls_config.h`, so it only needs to describe the deltas required + * to cross-compile cleanly against the wasi-sdk sysroot. The default 3.6 config + * already enables TLS 1.2 + TLS 1.3, X.509, PK, PSA crypto, and MBEDTLS_FS_IO + * (needed for curl/wget's mbedtls_x509_crt_parse_file on /etc/ssl/certs). + * + * Deltas: + * - Single-threaded: MBEDTLS_THREADING_C stays off (default) — no pthreads. + * - Entropy: WASI has no /dev/urandom fopen path, so disable platform entropy + * and route the entropy module through mbedtls_hardware_poll(), which we + * implement over getentropy() (see mbedtls_wasi_entropy.c). getentropy() is + * the same primitive the repo's host-side wasi_tls.c already relies on. + * - Networking: MBEDTLS_NET_C pulls in BSD select()/socket() + * that wasi-libc does not fully provide. curl/wget own the transport and + * only need the TLS record + X.509 layers, so drop mbedTLS's own sockets. + * - Timing: MBEDTLS_TIMING_C uses gettimeofday()/select() hardware timers that + * are unnecessary here (no DTLS retransmission timers in the curl path). + */ + +#ifndef MBEDTLS_WASI_USER_CONFIG_H +#define MBEDTLS_WASI_USER_CONFIG_H + +/* Entropy: replace the POSIX /dev/urandom source with a getentropy() poll. */ +#undef MBEDTLS_PLATFORM_ENTROPY +#define MBEDTLS_NO_PLATFORM_ENTROPY +#define MBEDTLS_ENTROPY_HARDWARE_ALT + +/* + * mbedtls_ms_time(): mbedTLS's built-in POSIX branch only compiles when it can + * see _POSIX_VERSION, but it guards the include behind unix/__unix + * macros that clang does not define for wasm32, so the platform detection falls + * through to `#error "No mbedtls_ms_time available"`. Provide it ourselves over + * clock_gettime(CLOCK_MONOTONIC) (available in wasi-libc) via the ALT hook. + */ +#define MBEDTLS_PLATFORM_MS_TIME_ALT + +/* Drop mbedTLS's own BSD-sockets and hardware-timing modules for WASI. */ +#undef MBEDTLS_NET_C +#undef MBEDTLS_TIMING_C + +#endif /* MBEDTLS_WASI_USER_CONFIG_H */ diff --git a/toolchain/c/overlays/wget/include/unistd.h b/toolchain/c/overlays/wget/include/unistd.h new file mode 100644 index 0000000000..7ff50df4d5 --- /dev/null +++ b/toolchain/c/overlays/wget/include/unistd.h @@ -0,0 +1,14 @@ +#ifndef AGENTOS_WGET_UNISTD_OVERLAY_H +#define AGENTOS_WGET_UNISTD_OVERLAY_H + +#include_next + +/* + * GNU Wget's ptimer fallback only needs to know that Linux-style POSIX timer + * IDs are not compile-time portable here. The underlying sysroot still exposes + * WASI clocks for libc++ and other consumers that use the WASI clockid_t ABI. + */ +#undef _POSIX_TIMERS +#define _POSIX_TIMERS 0 + +#endif diff --git a/toolchain/c/patches/git/0001-wasi-posix-spawn-run-command.patch b/toolchain/c/patches/git/0001-wasi-posix-spawn-run-command.patch new file mode 100644 index 0000000000..aa954de840 --- /dev/null +++ b/toolchain/c/patches/git/0001-wasi-posix-spawn-run-command.patch @@ -0,0 +1,201 @@ +Use posix_spawn for Git child processes on WASI. + +AgentOS provides POSIX process creation through the sysroot's posix_spawn +implementation, backed by the host_process broker. WASI cannot provide fork() +semantics, so route Git's central child-process path through posix_spawn while +leaving the rest of upstream Git's command machinery intact. + +POSIX spawn does not execute Git's fork-child close calls. Add explicit, +deduplicated close actions after the descriptor mappings so children cannot +retain unused pipe ends and suppress EOF. + +--- a/run-command.c ++++ b/run-command.c +@@ -18,6 +18,10 @@ + #include "packfile.h" + #include "compat/nonblock.h" + ++#ifdef __wasi__ ++#include ++#endif ++ + void child_process_init(struct child_process *child) + { + struct child_process blank = CHILD_PROCESS_INIT; +@@ -747,12 +751,14 @@ + + #ifndef GIT_WINDOWS_NATIVE + { ++#ifndef __wasi__ + int notify_pipe[2]; ++ struct child_err cerr; ++ struct atfork_state as; ++#endif + int null_fd = -1; + char **childenv; + struct strvec argv = STRVEC_INIT; +- struct child_err cerr; +- struct atfork_state as; + + if (prepare_cmd(&argv, cmd) < 0) { + failed_errno = errno; +@@ -764,8 +770,10 @@ + + trace_argv_printf(&argv.v[1], "trace: start_command:"); + ++#ifndef __wasi__ + if (pipe(notify_pipe)) + notify_pipe[0] = notify_pipe[1] = -1; ++#endif + + if (cmd->no_stdin || cmd->no_stdout || cmd->no_stderr) { + null_fd = xopen("/dev/null", O_RDWR | O_CLOEXEC); +@@ -773,6 +781,140 @@ + } + + childenv = prep_childenv(cmd->env.v); ++#ifdef __wasi__ ++ { ++ posix_spawn_file_actions_t actions; ++ int spawn_err; ++ ++ spawn_err = posix_spawn_file_actions_init(&actions); ++ if (spawn_err) { ++ failed_errno = spawn_err; ++ cmd->pid = -1; ++ errno = spawn_err; ++ goto wasi_spawn_done; ++ } ++ ++ if (cmd->no_stdin) ++ spawn_err = posix_spawn_file_actions_adddup2(&actions, null_fd, 0); ++ else if (need_in) ++ spawn_err = posix_spawn_file_actions_adddup2(&actions, fdin[0], 0); ++ else if (cmd->in) ++ spawn_err = posix_spawn_file_actions_adddup2(&actions, cmd->in, 0); ++ if (spawn_err) ++ goto wasi_spawn_failed_action; ++ ++ if (cmd->no_stderr) ++ spawn_err = posix_spawn_file_actions_adddup2(&actions, null_fd, 2); ++ else if (need_err) ++ spawn_err = posix_spawn_file_actions_adddup2(&actions, fderr[1], 2); ++ else if (cmd->err > 1) ++ spawn_err = posix_spawn_file_actions_adddup2(&actions, cmd->err, 2); ++ if (spawn_err) ++ goto wasi_spawn_failed_action; ++ ++ if (cmd->no_stdout) ++ spawn_err = posix_spawn_file_actions_adddup2(&actions, null_fd, 1); ++ else if (cmd->stdout_to_stderr) ++ spawn_err = posix_spawn_file_actions_adddup2(&actions, 2, 1); ++ else if (need_out) ++ spawn_err = posix_spawn_file_actions_adddup2(&actions, fdout[1], 1); ++ else if (cmd->out > 1) ++ spawn_err = posix_spawn_file_actions_adddup2(&actions, cmd->out, 1); ++ if (spawn_err) ++ goto wasi_spawn_failed_action; ++ ++ if (cmd->dir) { ++ spawn_err = posix_spawn_file_actions_addchdir(&actions, cmd->dir); ++ if (spawn_err) ++ goto wasi_spawn_failed_action; ++ } ++ ++ /* ++ * Unlike fork(), posix_spawn() does not run Git's child_close() ++ * calls. Close every non-stdio pipe end and caller-supplied source ++ * after the dup2 actions, otherwise a child can retain the last ++ * writer and prevent its sibling from ever observing EOF. ++ */ ++ { ++ int close_fds[8]; ++ size_t close_nr = 0; ++ ++#define ADD_SPAWN_CLOSE(fd) do { \ ++ int close_fd = (fd); \ ++ int already_added = 0; \ ++ for (size_t close_i = 0; close_i < close_nr; close_i++) \ ++ if (close_fds[close_i] == close_fd) \ ++ already_added = 1; \ ++ if (close_fd > STDERR_FILENO && !already_added) \ ++ close_fds[close_nr++] = close_fd; \ ++} while (0) ++ ++ if (need_in) { ++ ADD_SPAWN_CLOSE(fdin[0]); ++ ADD_SPAWN_CLOSE(fdin[1]); ++ } else { ++ ADD_SPAWN_CLOSE(cmd->in); ++ } ++ if (need_out) { ++ ADD_SPAWN_CLOSE(fdout[0]); ++ ADD_SPAWN_CLOSE(fdout[1]); ++ } else { ++ ADD_SPAWN_CLOSE(cmd->out); ++ } ++ if (need_err) { ++ ADD_SPAWN_CLOSE(fderr[0]); ++ ADD_SPAWN_CLOSE(fderr[1]); ++ } else { ++ ADD_SPAWN_CLOSE(cmd->err); ++ } ++ ADD_SPAWN_CLOSE(null_fd); ++#undef ADD_SPAWN_CLOSE ++ ++ for (size_t close_i = 0; close_i < close_nr; close_i++) { ++ spawn_err = posix_spawn_file_actions_addclose( ++ &actions, close_fds[close_i]); ++ if (spawn_err) ++ goto wasi_spawn_failed_action; ++ } ++ } ++ ++ /* ++ * Match the fork path's close_fd_above_stderr contract. Helpers ++ * that communicate exclusively through stdio must not inherit an ++ * unrelated pipe end and keep a sibling from observing EOF. ++ */ ++ if (cmd->close_fd_above_stderr) { ++ spawn_err = posix_spawn_file_actions_addclosefrom_np( ++ &actions, STDERR_FILENO + 1); ++ if (spawn_err) ++ goto wasi_spawn_failed_action; ++ } ++ ++ spawn_err = posix_spawn(&cmd->pid, argv.v[1], &actions, NULL, ++ (char *const *)argv.v + 1, childenv); ++ if (spawn_err == ENOEXEC) ++ spawn_err = posix_spawn(&cmd->pid, argv.v[0], &actions, NULL, ++ (char *const *)argv.v, childenv); ++ ++wasi_spawn_failed_action: ++ posix_spawn_file_actions_destroy(&actions); ++ if (spawn_err) { ++ failed_errno = spawn_err; ++ cmd->pid = -1; ++ errno = spawn_err; ++ if (!cmd->silent_exec_failure || errno != ENOENT) ++ error_errno("cannot spawn %s", cmd->args.v[0]); ++ } else if (cmd->clean_on_exit) { ++ mark_child_for_cleanup(cmd->pid, cmd); ++ } ++ ++wasi_spawn_done: ++ if (null_fd >= 0) ++ close(null_fd); ++ strvec_clear(&argv); ++ free(childenv); ++ } ++#else + atfork_prepare(&as); + + /* +@@ -905,6 +1047,7 @@ + close(null_fd); + strvec_clear(&argv); + free(childenv); ++#endif + } + end_of_spawn: + diff --git a/toolchain/c/patches/git/0002-wasi-synchronous-sideband-demux.patch b/toolchain/c/patches/git/0002-wasi-synchronous-sideband-demux.patch new file mode 100644 index 0000000000..a27a7d5454 --- /dev/null +++ b/toolchain/c/patches/git/0002-wasi-synchronous-sideband-demux.patch @@ -0,0 +1,213 @@ +Stream Git sidebands through a spawned multicall helper on WASI. + +Native Git runs recv_sideband() in an async worker while index-pack or +pack-objects runs in parallel. The old WASI workaround ran recv_sideband() +synchronously: fetch packs and push status reports were first written to +objects/pack/tmp_sideband_* and only consumed after the network phase. +Besides doubling storage for fetched packs, that changed full-duplex push +semantics and could deadlock when a server needed its response drained while +the client was still uploading. + +The AgentOS POSIX layer supports posix_spawn(), so preserve Git's native +process topology with a hidden multicall applet. The applet performs the +same recv_sideband() call as the native async worker. fetch-pack pipes its +band-1 output directly into index-pack/unpack-objects; send-pack starts it +before pack-objects and reads the remote status from its bounded pipe. +Band-2 progress and band-3 errors continue to go directly to stderr. The +pack failure path also drains and reports whatever remote status arrives, +matching native Git. + +--- a/builtin.h ++++ b/builtin.h +@@ -252,6 +252,7 @@ + int cmd_revert(int argc, const char **argv, const char *prefix, struct repository *repo); + int cmd_rm(int argc, const char **argv, const char *prefix, struct repository *repo); + int cmd_send_pack(int argc, const char **argv, const char *prefix, struct repository *repo); ++int cmd_sideband__helper(int argc, const char **argv, const char *prefix, struct repository *repo); + int cmd_shortlog(int argc, const char **argv, const char *prefix, struct repository *repo); + int cmd_show(int argc, const char **argv, const char *prefix, struct repository *repo); + int cmd_show_branch(int argc, const char **argv, const char *prefix, struct repository *repo); +--- a/git.c ++++ b/git.c +@@ -649,6 +649,7 @@ + { "revert", cmd_revert, RUN_SETUP | NEED_WORK_TREE }, + { "rm", cmd_rm, RUN_SETUP }, + { "send-pack", cmd_send_pack, RUN_SETUP }, ++ { "sideband--helper", cmd_sideband__helper, NO_PARSEOPT }, + { "shortlog", cmd_shortlog, RUN_SETUP_GENTLY | USE_PAGER }, + { "show", cmd_show, RUN_SETUP }, + { "show-branch", cmd_show_branch, RUN_SETUP }, +--- a/fetch-pack.c ++++ b/fetch-pack.c +@@ -892,6 +892,7 @@ + return retval; + } + ++#ifndef __wasi__ + static int sideband_demux(int in UNUSED, int out, void *data) + { + int *xd = data; +@@ -901,6 +902,7 @@ + close(out); + return ret; + } ++#endif + + static void create_promisor_file(const char *keep_name, + struct ref **sought, int nr_sought) +@@ -969,9 +971,13 @@ + struct child_process cmd = CHILD_PROCESS_INIT; + int fsck_objects = 0; + int ret; ++#ifdef __wasi__ ++ struct child_process demux_process = CHILD_PROCESS_INIT; ++#endif + + memset(&demux, 0, sizeof(demux)); + if (use_sideband) { ++#ifndef __wasi__ + /* xd[] is talking with upload-pack; subprocess reads from + * xd[0], spits out band#2 to stderr, and feeds us band#1 + * through demux->out. +@@ -982,6 +988,28 @@ + demux.isolate_sigpipe = 1; + if (start_async(&demux)) + die(_("fetch-pack: unable to fork off sideband demultiplexer")); ++#else ++ /* ++ * The WASI runtime cannot execute an in-process async function, ++ * but it can spawn another Git multicall applet. Keep the native ++ * streaming topology: the helper drains the remote concurrently ++ * and pipes band#1 directly to index-pack/unpack-objects. ++ */ ++ strvec_pushl(&demux_process.args, ++ "sideband--helper", "demux", "fetch-pack", NULL); ++ strvec_pushf(&demux_process.args, "%d", ++ sideband_get_allow_control_characters()); ++ demux_process.git_cmd = 1; ++ demux_process.clean_on_exit = 1; ++ demux_process.in = dup(xd[0]); ++ if (demux_process.in < 0) ++ die_errno(_("fetch-pack: unable to duplicate sideband input")); ++ demux_process.out = -1; ++ demux_process.close_fd_above_stderr = 1; ++ if (start_command(&demux_process)) ++ die(_("fetch-pack: unable to spawn sideband demultiplexer")); ++ demux.out = demux_process.out; ++#endif + } + else + demux.out = xd[0]; +@@ -1098,8 +1126,13 @@ + ret == 0; + else + die(_("%s failed"), cmd_name); ++#ifndef __wasi__ + if (use_sideband && finish_async(&demux)) + die(_("error in sideband demultiplexer")); ++#else ++ if (use_sideband && finish_command(&demux_process)) ++ die(_("error in sideband demultiplexer")); ++#endif + + sigchain_pop(SIGPIPE); + +--- a/send-pack.c ++++ b/send-pack.c +@@ -281,6 +281,7 @@ + return ret; + } + ++#ifndef __wasi__ + static int sideband_demux(int in UNUSED, int out, void *data) + { + int *fd = data, ret; +@@ -290,6 +291,7 @@ + close(out); + return ret; + } ++#endif + + static int advertise_shallow_grafts_cb(const struct commit_graft *graft, void *cb) + { +@@ -538,6 +540,9 @@ + char *push_cert_nonce = NULL; + struct packet_reader reader; + int use_bitmaps; ++#ifdef __wasi__ ++ struct child_process demux_process = CHILD_PROCESS_INIT; ++#endif + + if (!remote_refs) { + fprintf(stderr, "No refs in common and none specified; doing nothing.\n" +@@ -729,6 +734,7 @@ + + if (use_sideband && cmds_sent) { + memset(&demux, 0, sizeof(demux)); ++#ifndef __wasi__ + demux.proc = sideband_demux; + demux.data = fd; + demux.out = -1; +@@ -736,6 +742,28 @@ + if (start_async(&demux)) + die("send-pack: unable to fork off sideband demultiplexer"); + in = demux.out; ++#endif ++ /* ++ * Spawn the hidden Git helper on WASI so the response is drained ++ * while pack-objects is still writing the request, matching the ++ * native async worker's full-duplex process topology. ++ */ ++#ifdef __wasi__ ++ strvec_pushl(&demux_process.args, ++ "sideband--helper", "demux", "send-pack", NULL); ++ strvec_pushf(&demux_process.args, "%d", ++ sideband_get_allow_control_characters()); ++ demux_process.git_cmd = 1; ++ demux_process.clean_on_exit = 1; ++ demux_process.in = dup(fd[0]); ++ if (demux_process.in < 0) ++ die_errno("send-pack: unable to duplicate sideband input"); ++ demux_process.out = -1; ++ demux_process.close_fd_above_stderr = 1; ++ if (start_command(&demux_process)) ++ die("send-pack: unable to spawn sideband demultiplexer"); ++ in = demux_process.out; ++#endif + } + + packet_reader_init(&reader, in, NULL, 0, +@@ -759,8 +787,12 @@ + receive_status(r, &reader, remote_refs); + + if (use_sideband) { +- close(demux.out); ++ close(in); ++#ifndef __wasi__ + finish_async(&demux); ++#else ++ finish_command(&demux_process); ++#endif + } + fd[1] = -1; + +@@ -782,11 +814,19 @@ + packet_flush(out); + + if (use_sideband && cmds_sent) { ++#ifndef __wasi__ + close(demux.out); + if (finish_async(&demux)) { + error("error in sideband demultiplexer"); + ret = -1; + } ++#else ++ close(in); ++ if (finish_command(&demux_process)) { ++ error("error in sideband demultiplexer"); ++ ret = -1; ++ } ++#endif + } + + if (ret < 0) diff --git a/toolchain/c/patches/git/0003-wasi-stream-receive-sideband.patch b/toolchain/c/patches/git/0003-wasi-stream-receive-sideband.patch new file mode 100644 index 0000000000..f633f2e90f --- /dev/null +++ b/toolchain/c/patches/git/0003-wasi-stream-receive-sideband.patch @@ -0,0 +1,290 @@ +Stream receive-pack sideband muxers through a spawned helper on WASI. + +receive-pack uses the same in-process async facility for hook stderr, +connectivity progress, and index-pack progress. On WASI, async startup falls +back to unavailable fork(); most callers then lose progress and the unpack +path incorrectly returns success without consuming the pack. + +Reuse the hidden sideband multicall helper as a streaming mux process. The +helper reads bounded 128-byte chunks exactly like upstream copy_to_sideband(), +including keepalives, while its stdin pipe provides backpressure. Native builds +retain the existing async implementation. Treat mux startup or exit failure as +an unpack failure instead of silently skipping the pack. + +--- a/builtin/receive-pack.c ++++ b/builtin/receive-pack.c +@@ -560,35 +560,103 @@ + return 0; + } + ++int cmd_sideband__helper(int argc, ++ const char **argv, ++ const char *prefix UNUSED, ++ struct repository *repo UNUSED) ++{ ++ if (argc == 3 && !strcmp(argv[1], "demux")) ++ return recv_sideband(argv[2], STDIN_FILENO, STDOUT_FILENO); ++ ++ if (argc == 5 && !strcmp(argv[1], "mux")) { ++ use_sideband = atoi(argv[2]); ++ use_keepalive = atoi(argv[3]); ++ keepalive_in_sec = atoi(argv[4]); ++ if (use_sideband <= 0 || use_keepalive < KEEPALIVE_NEVER || ++ use_keepalive > KEEPALIVE_ALWAYS) ++ return error("invalid sideband mux parameters"); ++ return copy_to_sideband(STDIN_FILENO, STDOUT_FILENO, NULL); ++ } ++ ++ return error("usage: git sideband--helper demux