diff --git a/.dockerignore b/.dockerignore index 891c644f7..c0bc46de7 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,7 +7,8 @@ databases .env.* .git .github -docs +docs/* +!docs/serviceTemplates src/test *.md *.log diff --git a/.env.example b/.env.example index 5b151e342..73b8ab494 100644 --- a/.env.example +++ b/.env.example @@ -65,13 +65,14 @@ export P2P_BOOTSTRAP_NODES= export P2P_FILTER_ANNOUNCED_ADDRESSES= ## compute -# Each environment defines its own resources (CPU, RAM, disk, GPUs) with full configuration. -# CPU, RAM, and disk are per-env exclusive: inUse tracked only within the environment where the job runs. -# A global check ensures the aggregate usage across all environments does not exceed physical capacity. -# GPUs are shared-exclusive: if a job on envA uses gpu0, it shows as in-use on envB too. -# CPU cores are automatically partitioned across environments based on each env's cpu.total. -# CPU and RAM defaults are auto-detected from the system when not configured. -# export DOCKER_COMPUTE_ENVIRONMENTS='[{"socketPath":"/var/run/docker.sock","environments":[{"id":"envA","storageExpiry":604800,"maxJobDuration":3600,"minJobDuration":60,"resources":[{"id":"cpu","total":4,"max":4,"min":1,"type":"cpu"},{"id":"ram","total":16,"max":16,"min":1,"type":"ram"},{"id":"disk","total":500,"max":500,"min":10,"type":"disk"},{"id":"gpu0","total":1,"max":1,"min":0,"type":"gpu","init":{"deviceRequests":{"Driver":"nvidia","DeviceIDs":["0"],"Capabilities":[["gpu"]]}}}],"fees":{"1":[{"feeToken":"0x123","prices":[{"id":"cpu","price":1},{"id":"ram","price":0.1},{"id":"disk","price":0.01},{"id":"gpu0","price":5}]}]}}]}]' +# Resources are defined at the Docker-connection level (socketPath) and shared across all environments. +# cpu, ram, and disk are auto-detected from the host — omit them to use all available capacity, +# or include them to cap/reserve (e.g. limit an 8-core host to 6 cores for compute). +# GPUs and other hardware go in the connection-level "resources" array with kind:"discrete". +# Each environment references pool resources by id using lightweight refs {id, total?, min?, max?}. +# Dual-gate tracking for fungible resources: per-env ceiling (Gate 1) + engine-wide pool (Gate 2). +# Discrete resources (GPUs) are tracked globally — a GPU in use on envA shows as in-use on envB too. +# export DOCKER_COMPUTE_ENVIRONMENTS='[{"socketPath":"/var/run/docker.sock","resources":[{"id":"disk","total":500},{"id":"gpu0","kind":"discrete","type":"gpu","total":1,"description":"NVIDIA A100","platform":"nvidia","driverVersion":"570.195.03","init":{"deviceRequests":{"Driver":"nvidia","DeviceIDs":["GPU-uuid-a"],"Capabilities":[["gpu"]]}}}],"environments":[{"id":"envA","storageExpiry":604800,"maxJobDuration":3600,"minJobDuration":60,"resources":[{"id":"cpu"},{"id":"ram"},{"id":"disk","max":500},{"id":"gpu0"}],"fees":{"1":[{"feeToken":"0x123","prices":[{"id":"cpu","price":1},{"id":"ram","price":0.1},{"id":"disk","price":0.01},{"id":"gpu0","price":5}]}]}}]}]' export DOCKER_COMPUTE_ENVIRONMENTS= diff --git a/.eslintrc b/.eslintrc index 7f439ccba..e38e991f2 100644 --- a/.eslintrc +++ b/.eslintrc @@ -25,6 +25,7 @@ "jest": true }, "globals": { - "NodeJS": true + "NodeJS": true, + "RequestInit": true } } diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 05b8495f5..6de9e951d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: - node-version: 'v22.15.0' + node-version: 'v22.22.2' - name: Cache node_modules uses: actions/cache@v3 env: @@ -42,7 +42,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest] - node: ['18.20.4', 'v20.19.0', 'v22.15.0'] + node: ['v22.22.2'] steps: - uses: actions/checkout@v4 @@ -66,7 +66,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: - node-version: 'v22.15.0' + node-version: 'v22.22.2' - name: Cache node_modules uses: actions/cache@v3 env: @@ -102,7 +102,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: - node-version: 'v22.15.0' + node-version: 'v22.22.2' - name: Cache node_modules uses: actions/cache@v3 env: @@ -118,6 +118,7 @@ jobs: with: repository: 'oceanprotocol/barge' path: 'barge' + ref: '43cfdfd21154a2bae00770b779e7c39390ff5043' - name: Login to Docker Hub if: ${{ env.DOCKERHUB_PASSWORDNONO && env.DOCKERHUB_USERNAMENONO }} run: | @@ -129,6 +130,8 @@ jobs: working-directory: ${{ github.workspace }}/barge run: | bash -x start_ocean.sh --no-node --with-typesense 2>&1 > start_ocean.log & + env: + CONTRACTS_VERSION: '2.9.0' - run: npm ci - run: npm run build - run: docker image ls @@ -194,7 +197,7 @@ jobs: - name: Set up Node.js uses: actions/setup-node@v4 with: - node-version: 'v22.15.0' + node-version: 'v22.22.2' - name: Cache node_modules uses: actions/cache@v3 @@ -226,6 +229,8 @@ jobs: working-directory: ${{ github.workspace }}/barge run: | bash -x start_ocean.sh --no-node --with-typesense 2>&1 > start_ocean.log & + env: + CONTRACTS_VERSION: '2.9.0' - run: docker image ls - name: Delete default runner images run: | diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 9a81f4571..76e7c324f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -9,6 +9,7 @@ on: pull_request: branches: - 'main' + - 'next-4' env: DOCKERHUB_IMAGE: ${{ 'oceanprotocol/ocean-node' }} diff --git a/.nvmrc b/.nvmrc index 42126c054..a937bb366 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1,2 +1,2 @@ -22 +22.22.2 diff --git a/CLAUDE.md b/CLAUDE.md old mode 100755 new mode 100644 index 036f57125..2365df094 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -2,7 +2,6 @@ This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. - Ocean Node is the all-in-one backend for the Ocean Protocol stack. A single Node process replaces three legacy components: **Provider** (data access / encryption / compute), **Aquarius** (metadata cache) and the **subgraph** (on-chain event indexing). It is a @@ -13,12 +12,12 @@ of which dispatch to the same set of command handlers. ## 1. Environment & tooling prerequisites -- **Node.js 22 is required** (`.nvmrc` pins `22`). Always run `nvm use` (or - `source ~/.nvm/nvm.sh && nvm use`) before any `npm`, build, or test command. The wrong - Node version fails with errors like `Unexpected token 'with'` or missing `GLIBC_2.38`. +- **Node.js ≥ 22.13 is required** (`.nvmrc` pins `22.22.2`, matching the Dockerfile and CI; + `package.json` `engines` requires `>=22.13.0`). Always run `nvm use` (or `source ~/.nvm/nvm.sh && nvm use`) before + any `npm`, build, or test command. The wrong Node version fails with errors like + `Unexpected token 'with'`, missing `GLIBC_2.38`, or — since the SQLite layer uses the + built-in `node:sqlite` module — `ERR_UNKNOWN_BUILTIN_MODULE: node:sqlite` on Node < 22.13. This is enforced by `.cursor/rules/tests-nvm.mdc` and the in-repo `CLAUDE.md`. -- If `sqlite3` native bindings break after switching to Node 22, rebuild from source: - `npm_config_build_from_source=true npm rebuild sqlite3`. - **`postinstall` runs `scripts/fix-libp2p-http-utils.js`** — a patch applied to a libp2p dependency. Expect it to run on every `npm install`; don't remove it. - **Docker + docker-compose** are needed for the metadata database (Typesense or @@ -77,6 +76,7 @@ no semicolons, single quotes, `printWidth: 90`, no trailing commas, 2-space tabs **Tests run against compiled JS in `dist/test/`, not the TypeScript source.** The `test:*` scripts all call `npm run build-tests` first, which: + - compiles `src/` (incl. `src/test`) into `dist/`, - copies `src/test/.env.test` and `.env.test2` into `dist/test`, - copies `src/test/config.json` to `$HOME/config.json`. @@ -189,7 +189,7 @@ the command, looks up the handler by `task.command`, and calls `handler.handle(t Two front doors, one dispatcher: - **HTTP `POST /directCommand`** (`src/components/httpRoutes/commands.ts`): validates the - body, then decides *local vs remote*. If the command targets this node (or no P2P), it + body, then decides _local vs remote_. If the command targets this node (or no P2P), it calls `oceanNode.handleDirectProtocolCommand(...)`. If it targets another peer and P2P is enabled, it forwards via `oceanNode.getP2PNode().sendTo(node, msg, multiAddrs)`. Responses are streamed back to the client (binary or text). @@ -225,6 +225,7 @@ register it in the `CoreHandlersRegistry` constructor; add param validation; opt REST route in `src/components/httpRoutes/` and mount it in `httpRoutes/index.ts`. Handler source is grouped under `src/components/core/`: + - `handler/` — general handlers (ddo, download, encrypt, fees, nonce, query, status, p2p, auth, accessList, escrow, fileInfo, persistentStorage, policyServer, getJobs). - `compute/` — C2D command handlers: `initialize`, `startCompute` (paid), `freeStartCompute`, @@ -263,13 +264,14 @@ Handler source is grouped under `src/components/core/`: `getComputeResult` (+ `getComputeStreamableLogs`) → `stopCompute`. Paid compute settles via the `Escrow` component; `serviceResourceMatching.ts` maps requested cpu/ram/disk/gpu against environment pools (dual-gate: per-env ceiling + engine-wide pool; GPUs tracked globally). - See `docs/compute-pricing.md`, `docs/GPU.md`. + See `docs/compute.md`. - **database/** — `Database.init()` factory (`index.ts`, `DatabaseFactory.ts`). The metadata DB backend is pluggable: **Typesense or Elasticsearch** (chosen by `DB_TYPE`) for DDOs, indexer state, logs, orders, ddoState, access lists, escrow events — behind the `Abstract*Database` interfaces in `BaseDatabase.ts`. **SQLite** is always used for the nonce DB, config DB, C2D job DB, and auth-token DB (works even with no metadata DB - configured). See `docs/database.md`. + configured) — via Node's built-in `node:sqlite` module (no native addon), wrapped by + `SqliteClient` in `src/components/database/sqliteClient.ts`. See `docs/database.md`. - **KeyManager/** — provider-abstraction over the node key (`docs/KeyManager.md`). Currently `RawPrivateKeyProvider` (from `PRIVATE_KEY`); derives the libp2p peerId/keys and the EVM address, and caches the ethers signer. Designed to add KMS providers (GCP/AWS) later. @@ -316,5 +318,5 @@ can talk to `/var/run/docker.sock`. Deployment options (Docker, local Docker bui `Arhitecture.md` (note the spelling), `API.md` (full HTTP API reference — very large, plus a Postman collection), `env.md` (authoritative env-var reference), `database.md`, `Storage.md` / `persistentStorage.md`, `KeyManager.md`, `PolicyServer.md`, `services.md` -(Service-on-Demand), `compute-pricing.md` / `GPU.md` (C2D), `networking.md`, `Logs.md`, +(Service-on-Demand), `compute.md` (C2D configuration: resources, GPUs, constraints, pricing), `networking.md`, `Logs.md`, `Publishing.md`, `testing.md`, `dockerDeployment.md`. diff --git a/Dockerfile b/Dockerfile index 1567fa7e6..b8e4b14a2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -43,6 +43,9 @@ COPY --chown=node:node --from=builder /usr/src/app/node_modules ./node_modules COPY --chown=node:node --from=builder /usr/src/app/schemas ./schemas COPY --chown=node:node --from=builder /usr/src/app/package.json ./ COPY --chown=node:node --from=builder /usr/src/app/config.json ./ +# Ship the operator service-on-demand templates so SERVICE_TEMPLATES_PATH=docs/serviceTemplates/ +# resolves inside the image (the rest of docs/ stays excluded via .dockerignore). +COPY --chown=node:node --from=builder /usr/src/app/docs/serviceTemplates ./docs/serviceTemplates RUN mkdir -p databases c2d_storage logs diff --git a/README.md b/README.md index c3d1d33ba..6a5233b23 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ This command will run you through the process of setting up the environmental va > [!NOTE] > The quickstart script attempts to automatically detect GPUs (NVIDIA via `nvidia-smi`, others via `lspci`) and appends them to your `DOCKER_COMPUTE_ENVIRONMENTS`. +> Detected GPUs are added to `DOCKER_COMPUTE_ENVIRONMENTS[0].resources` (the connection-level resource pool), and a lightweight ref is added to `DOCKER_COMPUTE_ENVIRONMENTS[0].environments[0].resources`. > If you choose to manually configure `DOCKER_COMPUTE_ENVIRONMENTS` before running the script (e.g. via environment variable), be aware that auto-detected GPUs will be **merged** into your configuration, which could lead to duplication if you already manually defined them. > For most users, it is recommended to let the script handle GPU detection automatically. @@ -153,5 +154,5 @@ Your node is now running. To start additional nodes, repeat these steps in a new - [Network Configuration](docs/networking.md) - [Logging & accessing logs](docs/networking.md) - [Docker Deployment Guide](docs/dockerDeployment.md) -- [C2D GPU Guide](docs/GPU.md) -- [Compute pricing](docs/compute-pricing.md) +- [Compute (C2D) Configuration — resources, GPUs, constraints, pricing](docs/compute.md) +- [Services (Service-on-Demand)](docs/services.md) diff --git a/config.json b/config.json index 42d364183..35b084a61 100644 --- a/config.json +++ b/config.json @@ -94,18 +94,31 @@ "validateUnsignedDDO": true, "jwtSecret": "ocean-node-secret", "enableBenchmark": false, + "serviceTemplatesPath": "docs/serviceTemplates/", "dockerComputeEnvironments": [ { "socketPath": "/var/run/docker.sock", + "resources": [ + { + "id": "disk", + "total": 1 + } + ], "environments": [ { "storageExpiry": 604800, "maxJobDuration": 3600, "minJobDuration": 60, "resources": [ + { + "id": "cpu" + }, + { + "id": "ram" + }, { "id": "disk", - "total": 1 + "max": 1 } ], "access": { diff --git a/docs/API.md b/docs/API.md index dd5475852..f7c22db36 100644 --- a/docs/API.md +++ b/docs/API.md @@ -1819,3 +1819,362 @@ Delete a file from a bucket. ```json { "success": true } ``` + +--- + +## Service on Demand + +Service-on-Demand lets a consumer launch a long-running Docker container (e.g. JupyterLab, a +vLLM inference server, VS Code) on a compute environment, pay up front via on-chain escrow for +a requested `duration`, and reach it over forwarded network endpoints +(`http://:`) while it runs. Unlike a compute job, a service stays up until +it expires, is stopped, or is extended. See [`services.md`](./services.md) for the full design +and security model. + +All routes live under `/api/services`. Every command except `serviceTemplates` is +authenticated by a signature over `consumerAddress` + `nonce` + `command` (or an auth-token +`Authorization` header). Cost is computed only from the environment's server-side pricing and +charged to the authenticated `consumerAddress`. + +> **Note:** service containers run hardened (`no-new-privileges`, `CapDrop: ['ALL']`), so a +> process inside the container cannot bind to a port below 1024 — have your service listen on a +> **high port** (the published host port is allocated by the node regardless). + +### Service object definitions + +#### `ServiceTemplatePublic` (returned by `serviceTemplates`) + +Operator-published blueprint. Secret `envVars` values are never returned — only their keys via +`envVarKeys`. + +| property | type | description | +| ------------------------- | ----------------------------- | ----------- | +| id | string | template id (`[a-z0-9][a-z0-9_-]{0,63}`) | +| name / description | string | human-readable labels | +| image | string | base image | +| tag / checksum / dockerfile | string | image spec — exactly one | +| exposedPorts | number[] | container ports to forward | +| envVarKeys | string[] | keys of operator-set env vars (values never returned) | +| userConfigurableEnvVars | object[] | `{ key, validation?, sensitive? }` passed via `userData` | +| command / entrypoint | string[] | Docker CMD / ENTRYPOINT overrides | +| requiredResources | object[] | resources the service MUST have to run | +| recommendedResources | object[] | resources for best performance | + +#### `ServiceJob` (returned by start / status / extend / restart / stop) + +The encrypted `userData` is never returned. Key fields: + +| property | type | description | +| ------------- | -------- | ----------- | +| serviceId | string | unique id of the running service | +| environment | string | envId the service runs on | +| owner | string | consumerAddress | +| status | number | `10` Starting, `20` Locking, `11` PullImage, `13` BuildImage, `30` Claiming, `40` Running, `12` PullImageFailed, `14` BuildImageFailed, `15` VulnerableImage, `50` Stopping, `70` Stopped, `75` Expired, `99` Error | +| statusText | string | human-readable status | +| dateCreated | string | ISO timestamp | +| expiresAt | number | Unix ms timestamp when the paid window ends | +| duration | number | requested seconds | +| endpoints | object[] | `{ containerPort, hostPort, url }` per exposed port | +| resources | object[] | `{ id, amount, price }` | +| payment | object | initial start payment record | +| extendPayments | object[] | one entry per successful extend | + +--- + +### `HTTP` GET /api/services/serviceTemplates + +### `P2P` command: serviceGetTemplates + +#### Description + +List the operator-published service templates (sanitized). Not authenticated. + +#### Query Parameters + +| name | type | required | description | +| ------- | ------ | -------- | ----------- | +| chainId | number | | filter to templates whose envs price on this chain | + +#### Response (200) + +```json +[ + { + "id": "jupyter-cpu", + "name": "JupyterLab (CPU)", + "image": "quay.io/jupyter/datascience-notebook", + "tag": "latest", + "exposedPorts": [8888], + "userConfigurableEnvVars": [{ "key": "JUPYTER_TOKEN", "sensitive": true }], + "requiredResources": [{ "id": "cpu", "min": 1 }, { "id": "ram", "min": 2 }] + } +] +``` + +--- + +### `HTTP` POST /api/services/serviceStart + +### `P2P` command: serviceStart + +#### Description + +Validate the request, persist the job, and **return immediately** with the `serviceId` — the +response does **not** wait for escrow or the image pull/build. The consumer supplies the +container spec directly (an `image` referenced by `tag`/`checksum`, or an inline `dockerfile` +when the operator allows building). + +The returned job has `status: 10` (`Starting`) and no `endpoints` yet. A background loop then +advances it: `Starting → Locking` (escrow lock) `→ PullImage`/`BuildImage` (image + scan) `→ +Claiming` (claim on success, or refund/cancel the lock on failure) `→ Running`. **Poll +`serviceStatus`** until `status` is `40` (`Running`, with `endpoints` populated) or a terminal +`*Failed` / `Error` status. Note that `Running` is not a final resting state for a poller to stop +at: the same background loop keeps checking the container's health afterward, and can move an +already-`Running` service to `Error` later if the container dies on its own — long-lived clients +should keep watching `serviceStatus`, not just stop once they first see `Running`. + +#### Request Body + +```json +{ + "consumerAddress": "0x...", + "nonce": "123", + "signature": "0x...", + "environment": "env-1", + "image": "nginxinc/nginx-unprivileged", + "tag": "alpine", + "exposedPorts": [8080], + "resources": [{ "id": "cpu", "amount": 1 }, { "id": "ram", "amount": 1 }], + "duration": 3600, + "userData": "", + "payment": { "chainId": 8996, "token": "0x..." } +} +``` + +| field | type | required | description | +| --------------------- | -------- | -------- | ----------- | +| environment | string | v | envId to run on (services must be enabled on it) | +| image | string | v | base image | +| tag / checksum / dockerfile | string | | image spec — at most one; `dockerfile` requires `allowImageBuild` | +| additionalDockerFiles | object | | filename → content; only with `dockerfile` | +| dockerCmd / dockerEntrypoint | string[] | | container CMD / ENTRYPOINT overrides | +| exposedPorts | number[] | | container ports to publish | +| resources | object[] | | `{ id, amount }` requested resources | +| duration | number | v | seconds; capped by `serviceOnDemand.maxDurationSeconds` | +| userData | string | | ECIES-encrypted (to the node pubkey) JSON of env vars | +| payment | object | v | `{ chainId, token }` | + +#### Response (200) + +The immediate response — `Starting`, no endpoints yet. Poll `serviceStatus` for the rest. + +```json +[ + { + "serviceId": "0x...", + "environment": "env-1", + "owner": "0x...", + "status": 10, + "statusText": "Starting", + "expiresAt": 1735689600000, + "duration": 3600, + "endpoints": [], + "payment": { "chainId": 8996, "token": "0x...", "cost": 10 } + } +] +``` + +Errors: `403` services disabled on the env / access denied, `400` invalid params (bad address, +duration, image spec, unavailable resources, or no pricing for the token). Escrow lock/claim now +happens in the background, so escrow failures surface as the job ending in an `Error` / `*Failed` +status (observed via `serviceStatus`), not as a synchronous `402`. + +--- + +### `HTTP` GET /api/services/serviceStatus + +### `P2P` command: serviceGetStatus + +#### Description + +Read service job status and endpoints. **Authenticated and owner-scoped** — only services owned +by the authenticated `consumerAddress` are returned. + +#### Query Parameters + +| name | type | required | description | +| --------------- | ------ | -------- | ----------- | +| consumerAddress | string | v | owner address | +| nonce | string | v | request nonce | +| signature | string | v | signed message (or use an `Authorization` auth-token header) | +| serviceId | string | | filter to a single service; omit to list all owned services | + +#### Response (200) + +Array of `ServiceJob` (with `userData` stripped). + +--- + +### `HTTP` GET /api/services/serviceList + +### `P2P` command: serviceList + +#### Description + +Node-wide service listing. **Authenticated but NOT owner-scoped** — any authenticated +consumer identity sees every owner's services. By default it returns only the services +**currently holding a resource reservation** (exactly what the engines count against the +shared pools): `Running`/`Restarting`/`Stopping`, the mid-start pipeline states, paid +`Error` (container died, restartable), and explicitly `Stopped` within the paid window. +`Expired` and never-paid jobs hold nothing and are not listed by default. + +#### Query Parameters + +| name | type | required | description | +| ----------------- | ------- | -------- | ----------- | +| consumerAddress | string | v | caller identity (any consumer) | +| nonce | string | v | request nonce | +| signature | string | v | signed message (or use an `Authorization` auth-token header) | +| status | number | | filter to ONE specific `ServiceStatusNumber` (any status, incl. `75` Expired); takes precedence over `includeAllStatuses` | +| includeAllStatuses | boolean | | `true` returns services in every status instead of only the resource-holding set | +| fromTimestamp | string | | only services created at/after this moment — ISO date (`2026-01-15T00:00:00Z`) or Unix timestamp (seconds or milliseconds) | + +#### Response (200) + +Array of `ServiceJob`, **listing-sanitized**: `userData`, `dockerCmd`, `dockerEntrypoint`, +`dockerfile` and `additionalDockerFiles` are stripped (identity, status, resources, +endpoints and payment metadata are kept). Use the owner-scoped `serviceStatus` to see a +service's own configuration. + +--- + +### `HTTP` POST /api/services/serviceExtend + +### `P2P` command: serviceExtend + +#### Description + +Pay to push the service expiry further out. The total remaining duration must not exceed +`maxDurationSeconds`. Re-checks the environment access list. + +#### Request Body + +```json +{ + "consumerAddress": "0x...", + "nonce": "123", + "signature": "0x...", + "serviceId": "0x...", + "additionalDuration": 1800, + "payment": { "chainId": 8996, "token": "0x..." } +} +``` + +`additionalDuration` must be a positive number of seconds. + +#### Response (200) + +The updated `ServiceJob` (advanced `expiresAt`, new entry in `extendPayments`). + +--- + +### `HTTP` POST /api/services/serviceRestart + +### `P2P` command: serviceRestart + +#### Description + +Recreate the service container (no extra charge), keeping the same `expiresAt` and host ports. +Re-checks the environment service gate and access list; rejected if the service has expired. +Optionally pass `userData` to replace the stored env vars, and/or `dockerCmd`/`dockerEntrypoint` +to replace the stored CMD/ENTRYPOINT overrides — each is independent: supply it (even as `[]`) +to replace the stored value, or omit it to reuse what the service already has. This is the +recommended recovery path after a service lands in `Error` because its container died on its own +(the background health check leaves host ports/network/container record reserved specifically so +restart can reuse them), in addition to recovering from an explicit `serviceStop` or any other +terminal failure. + +#### Request Body + +```json +{ + "consumerAddress": "0x...", + "nonce": "123", + "signature": "0x...", + "serviceId": "0x...", + "userData": "", + "dockerCmd": [""], + "dockerEntrypoint": [""] +} +``` + +#### Response (200) + +The `ServiceJob` with a new `containerId` (same `hostPort` and `expiresAt`). + +--- + +### `HTTP` POST /api/services/serviceStop + +### `P2P` command: serviceStop + +#### Description + +Tear down the service container and network and release its resources. Owner-gated. + +#### Request Body + +```json +{ + "consumerAddress": "0x...", + "nonce": "123", + "signature": "0x...", + "serviceId": "0x..." +} +``` + +#### Response (200) + +The `ServiceJob` with `status: 70` (Stopped). + +--- + +### `HTTP` GET /api/services/serviceStreamableLogs + +### `P2P` command: serviceGetStreamableLogs + +#### Description + +Stream the service container's stdout/stderr logs live. **Authenticated and owner-scoped** +— only the service's owner (`consumerAddress`, proven by signature/nonce or auth token) can +read its logs. Available while the service is `Running` (`40`) or `Error` (`99`) — a crashed +container is kept around until `stop`/`restart`, so its logs remain fetchable for diagnosis. + +#### Query Parameters + +| name | type | required | description | +| --------------- | ------ | -------- | ----------- | +| consumerAddress | string | v | owner address | +| nonce | string | v | request nonce | +| signature | string | v | signed message (or use an `Authorization` auth-token header) | +| serviceId | string | v | the service to stream logs for | +| since | string | | lower time bound for returned logs. Either a Unix timestamp in seconds (e.g. `1735689600`), or a relative duration counted back from now (e.g. `30s`, `45m`, `2h`, `7d`). Omit to get the full history since container start, then follow live. | + +#### Response (200) + +Raw `stdout`/`stderr` byte stream from the container, connection kept open and followed live. +With `since` set, historical output before that point is skipped — useful for a long-lived +service where fetching the full history would otherwise dump days/weeks of buffered logs +before reaching the live tail (e.g. `since=1h` for just the last hour). + +#### Response (400) + +`since` is present but not a valid Unix timestamp or duration (``). + +#### Response (404) + +Service not found, or not `Running`/`Error`. + +#### Response (401) + +Missing/invalid auth, or `consumerAddress` is not the service owner. diff --git a/docs/GPU.md b/docs/GPU.md deleted file mode 100644 index b89a7664f..000000000 --- a/docs/GPU.md +++ /dev/null @@ -1,730 +0,0 @@ -Supporting GPUs for c2d jobs comes down to: - -- define gpu list for each c2d env -- pass docker args for each gpu -- set a price for each gpu (see [Compute pricing](compute-pricing.md) for pricing units and examples) - -## Nvidia GPU Example - -Start by installing nvidia cuda drivers (ie:https://docs.nvidia.com/cuda/cuda-installation-guide-linux/), then install nvidia container toolkit (https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html) - -Once that is done, check if you can get gpu details by running 'nvidia-smi': - -``` -root@gpu-1:/repos/ocean/ocean-node# nvidia-smi -Fri Apr 25 06:00:34 2025 -+-----------------------------------------------------------------------------------------+ -| NVIDIA-SMI 550.163.01 Driver Version: 550.163.01 CUDA Version: 12.4 | -|-----------------------------------------+------------------------+----------------------+ -| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC | -| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. | -| | | MIG M. | -|=========================================+========================+======================| -| 0 NVIDIA GeForce GTX 1060 3GB Off | 00000000:01:00.0 Off | N/A | -| 0% 39C P8 6W / 120W | 2MiB / 3072MiB | 0% Default | -| | | N/A | -+-----------------------------------------+------------------------+----------------------+ - -+-----------------------------------------------------------------------------------------+ -| Processes: | -| GPU GI CI PID Type Process name GPU Memory | -| ID ID Usage | -|=========================================================================================| -| No running processes found | -+-----------------------------------------------------------------------------------------+ -``` - -Now, time to get the id of the gpu: - -```bash -root@gpu-1:/repos/ocean/ocean-node# nvidia-smi --query-gpu=name,uuid,driver_version,memory.total --format=csv -name, uuid, driver version, memory total -NVIDIA GeForce GTX 1060 3GB, GPU-294c6802-bb2f-fedb-f9e0-a26b9142dd81, 570.195.03, 3072 MiB -``` - -Now, we can define the gpu for node: - -```json -{ - "id": "myGPU", - "description": "NVIDIA GeForce GTX 1060 3GB", - "type": "gpu", - "total": 1, - "init": { - "deviceRequests": { - "Driver": "nvidia", - "DeviceIDs": ["GPU-294c6802-bb2f-fedb-f9e0-a26b9142dd81"], - "Capabilities": [["gpu"]] - } - }, - "driverVersion": "570.195.03", - "memoryTotal": "3072 MiB" -} -``` - -Don't forget to add it to fees definition and free definition (if desired). - -Here is the full definition of DOCKER_COMPUTE_ENVIRONMENTS: - -```json -[ - { - "socketPath": "/var/run/docker.sock", - "resources": [ - { - "id": "myGPU", - "description": "NVIDIA GeForce GTX 1060 3GB", - "type": "gpu", - "total": 1, - "init": { - "deviceRequests": { - "Driver": "nvidia", - "DeviceIDs": ["GPU-294c6802-bb2f-fedb-f9e0-a26b9142dd81"], - "Capabilities": [["gpu"]] - } - }, - "driverVersion": "570.195.03", - "memoryTotal": "3072 MiB" - }, - { "id": "disk", "total": 1 } - ], - "storageExpiry": 604800, - "maxJobDuration": 3600, - "minJobDuration": 60, - "fees": { - "1": [ - { - "feeToken": "0x123", - "prices": [ - { "id": "cpu", "price": 1 }, - { "id": "myGPU", "price": 3 } - ] - } - ] - }, - "free": { - "maxJobDuration": 60, - "minJobDuration": 10, - "maxJobs": 3, - "resources": [ - { "id": "cpu", "max": 1 }, - { "id": "ram", "max": 1 }, - { "id": "disk", "max": 1 }, - { "id": "myGPU", "max": 1 } - ] - } - } -] -``` - -And you should have it in your compute envs: - -```bash -root@gpu-1:/repos/ocean/ocean-node# curl http://localhost:8000/api/services/computeEnvironments -``` - -```json -[ - { - "id": "0xd6b10b27aab01a72070a5164c07d0517755838b9cb9857e2d5649287ec3aaaa2-0x66073c81f833deaa2f8e2a508f69cf78f8a99b17ba1a64f369af921750f93914", - "runningJobs": 0, - "consumerAddress": "0x00", - "platform": { "architecture": "x86_64", "os": "Ubuntu 22.04.3 LTS" }, - "fees": { - "1": [ - { - "feeToken": "0x123", - "prices": [ - { "id": "cpu", "price": 1 }, - { "id": "myGPU", "price": 3 } - ] - } - ] - }, - "storageExpiry": 604800, - "maxJobDuration": 3600, - "minJobDuration": 60, - "resources": [ - { "id": "cpu", "total": 8, "max": 8, "min": 1, "inUse": 0 }, - { - "id": "ram", - "total": 23, - "max": 23, - "min": 1, - "inUse": 0 - }, - { - "id": "myGPU", - "description": "NVIDIA GeForce GTX 1060 3GB", - "type": "gpu", - "total": 1, - "init": { - "deviceRequests": { - "Driver": "nvidia", - "DeviceIDs": ["GPU-294c6802-bb2f-fedb-f9e0-a26b9142dd81"], - "Capabilities": [["gpu"]] - } - }, - "driverVersion": "570.195.03", - "memoryTotal": "3072 MiB", - "max": 1, - "min": 0, - "inUse": 0 - }, - { "id": "disk", "total": 1, "max": 1, "min": 0, "inUse": 0 } - ], - "free": { - "maxJobDuration": 60, - "minJobDuration": 10, - "maxJobs": 3, - "resources": [ - { "id": "cpu", "max": 1, "inUse": 0 }, - { "id": "ram", "max": 1, "inUse": 0 }, - { "id": "disk", "max": 1, "inUse": 0 }, - { "id": "myGPU", "max": 1, "inUse": 0 } - ] - }, - "runningfreeJobs": 0 - } -] -``` - -Start a free job using: - -```json -{ - "command": "freeStartCompute", - "algorithm": { - "meta": { - "container": { - "image": "tensorflow/tensorflow", - "tag": "2.17.0-gpu", - "entrypoint": "python $ALGO" - }, - "rawcode": "import tensorflow as tf\nsess = tf.compat.v1.Session(config=tf.compat.v1.ConfigProto(log_device_placement=True))\nprint(\"Num GPUs Available: \", len(tf.config.list_physical_devices('GPU')))\ngpus = tf.config.list_physical_devices('GPU')\nfor gpu in gpus:\n\tprint('Name:', gpu.name, ' Type:', gpu.device_type)" - } - }, - "consumerAddress": "0x00", - "signature": "123", - "nonce": 1, - "environment": "0xd6b10b27aab01a72070a5164c07d0517755838b9cb9857e2d5649287ec3aaaa2-0x66073c81f833deaa2f8e2a508f69cf78f8a99b17ba1a64f369af921750f93914", - "resources": [ - { - "id": "cpu", - "amount": 1 - }, - { - "id": "myGPU", - "amount": 1 - } - ] -} -``` - -And the output of `getComputeResult` should look like: - -```bash -2025-04-25 06:18:20.890217: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:485] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered -2025-04-25 06:18:21.192330: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:8454] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered -2025-04-25 06:18:21.292230: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1452] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered -WARNING: All log messages before absl::InitializeLog() is called are written to STDERR -I0000 00:00:1745561915.985558 1 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355 -I0000 00:00:1745561915.993514 1 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355 -I0000 00:00:1745561915.993799 1 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355 -Num GPUs Available: 1 -Name: /physical_device:GPU:0 Type: GPU -``` - -## AMD Radeon 9070 XT ON WSL2 - -First, install ROCm (https://rocm.docs.amd.com/projects/radeon/en/latest/docs/install/wsl/install-radeon.html) - -Then define DOCKER_COMPUTE_ENVIRONMENTS with - -```json -[ - { - "socketPath": "/var/run/docker.sock", - "resources": [ - { - "id": "myGPU", - "description": "AMD Radeon RX 9070 XT", - "type": "gpu", - "total": 1, - "init": { - "advanced": { - "IpcMode": "host", - "ShmSize": 8589934592, - "CapAdd": ["SYS_PTRACE"], - "Devices": ["/dev/dxg", "/dev/dri/card0"], - "Binds": [ - "/usr/lib/wsl/lib/libdxcore.so:/usr/lib/libdxcore.so", - "/opt/rocm/lib/libhsa-runtime64.so.1:/opt/rocm/lib/libhsa-runtime64.so.1" - ], - "SecurityOpt": { - "seccomp": "unconfined" - } - } - }, - "driverVersion": "26.2.2", - "memoryTotal": "16384 MiB" - }, - { - "id": "disk", - "total": 1 - } - ], - "storageExpiry": 604800, - "maxJobDuration": 3600, - "minJobDuration": 60, - "fees": { - "1": [ - { - "feeToken": "0x123", - "prices": [ - { - "id": "cpu", - "price": 1 - }, - { - "id": "nyGPU", - "price": 3 - } - ] - } - ] - }, - "free": { - "maxJobDuration": 60, - "minJobDuration": 10, - "maxJobs": 3, - "resources": [ - { - "id": "cpu", - "max": 1 - }, - { - "id": "ram", - "max": 1 - }, - { - "id": "disk", - "max": 1 - }, - { - "id": "myGPU", - "max": 1 - } - ] - } - } -] -``` - -aka - -```bash -export DOCKER_COMPUTE_ENVIRONMENTS='[{"socketPath":"/var/run/docker.sock","resources":[{"id":"myGPU","description":"AMD Radeon RX 9070 XT","type":"gpu","total":1,"init":{"advanced":{"IpcMode":"host","ShmSize":8589934592,"CapAdd":["SYS_PTRACE"],"Devices":["/dev/dxg","/dev/dri/card0"],"Binds":["/usr/lib/wsl/lib/libdxcore.so:/usr/lib/libdxcore.so","/opt/rocm/lib/libhsa-runtime64.so.1:/opt/rocm/lib/libhsa-runtime64.so.1"],"SecurityOpt":{"seccomp":"unconfined"}}},"driverVersion":"26.2.2","memoryTotal":"16384 MiB"},{"id":"disk","total":1}],"storageExpiry":604800,"maxJobDuration":3600,"minJobDuration":60,"fees":{"1":[{"feeToken":"0x123","prices":[{"id":"cpu","price":1},{"id":"nyGPU","price":3}]}]},"free":{"maxJobDuration":60,"minJobDuration":10,"maxJobs":3,"resources":[{"id":"cpu","max":1},{"id":"ram","max":1},{"id":"disk","max":1},{"id":"myGPU","max":1}]}}]' -``` - -you should have it in your compute envs: - -```bash -root@gpu-1:/repos/ocean/ocean-node# curl http://localhost:8000/api/services/computeEnvironments -``` - -```json -[ - { - "id": "0xbb5773e734e1b188165dac88d9a3dc8ac28bc9f5624b45fa8bbd8fca043de7c1-0x2c2761f938cf186eeb81f71dee06ad7edb299493e39c316c390d0c0691e6585c", - "runningJobs": 0, - "consumerAddress": "0x00", - "platform": { - "architecture": "x86_64", - "os": "Ubuntu 24.04.2 LTS" - }, - "fees": { - "1": [ - { - "feeToken": "0x123", - "prices": [ - { - "id": "cpu", - "price": 1 - }, - { - "id": "nyGPU", - "price": 3 - } - ] - } - ] - }, - "storageExpiry": 604800, - "maxJobDuration": 3600, - "minJobDuration": 60, - "resources": [ - { - "id": "cpu", - "total": 16, - "max": 16, - "min": 1, - "inUse": 0 - }, - { - "id": "ram", - "total": 31, - "max": 31, - "min": 1, - "inUse": 0 - }, - { - "id": "myGPU", - "description": "AMD Radeon RX 9070 XT", - "type": "gpu", - "total": 1, - "init": { - "advanced": { - "IpcMode": "host", - "CapAdd": ["CAP_SYS_PTRACE"], - "Devices": ["/dev/dxg", "/dev/dri/card0"], - "Binds": [ - "/usr/lib/wsl/lib/libdxcore.so:/usr/lib/libdxcore.so", - "/opt/rocm/lib/libhsa-runtime64.so.1:/opt/rocm/lib/libhsa-runtime64.so.1" - ], - "SecurityOpt": { - "seccomp": "unconfined" - } - } - }, - "driverVersion": "26.2.2", - "memoryTotal": "16384 MiB", - "max": 1, - "min": 0, - "inUse": 0 - }, - { - "id": "disk", - "total": 10, - "max": 10, - "min": 0, - "inUse": 0 - } - ], - "free": { - "maxJobDuration": 60, - "minJobDuration": 10, - "maxJobs": 3, - "resources": [ - { - "id": "cpu", - "max": 1, - "inUse": 0 - }, - { - "id": "ram", - "max": 1, - "inUse": 0 - }, - { - "id": "disk", - "max": 1, - "inUse": 0 - }, - { - "id": "myGPU", - "max": 1, - "inUse": 0 - } - ] - }, - "runningfreeJobs": 0 - } -] -``` - -Start a free job with - -```json -{ - "command": "freeStartCompute", - "datasets": [ - { - "fileObject": { - "type": "url", - "url": "https://raw.githubusercontent.com/oceanprotocol/test-algorithm/master/javascript/algo.js", - "method": "get" - } - } - ], - "algorithm": { - "meta": { - "container": { - "image": "rocm/tensorflow", - "tag": "rocm6.4-py3.12-tf2.18-dev", - "entrypoint": "python $ALGO" - }, - "rawcode": "import tensorflow as tf\nsess = tf.compat.v1.Session(config=tf.compat.v1.ConfigProto(log_device_placement=True))\nprint(\"Num GPUs Available: \", len(tf.config.list_physical_devices('GPU')))\ngpus = tf.config.list_physical_devices('GPU')\nfor gpu in gpus:\n\tprint('Name:', gpu.name, ' Type:', gpu.device_type)" - } - }, - "consumerAddress": "0x00", - "signature": "123", - "nonce": 1, - "environment": "0xbb5773e734e1b188165dac88d9a3dc8ac28bc9f5624b45fa8bbd8fca043de7c1-0x2c2761f938cf186eeb81f71dee06ad7edb299493e39c316c390d0c0691e6585c", - "resources": [ - { - "id": "cpu", - "amount": 1 - }, - { - "id": "myGPU", - "amount": 1 - } - ] -} -``` - -and get the results - -```bash -2025-04-25 15:16:15.218050: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. -To enable the following instructions: SSE3 SSE4.1 SSE4.2 AVX AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. -WARNING: All log messages before absl::InitializeLog() is called are written to STDERR -I0000 00:00:1745594260.720023 1 gpu_device.cc:2022] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 2874 MB memory: -> device: 0, name: AMD Radeon RX 9070 XT, pci bus id: 0000:0d:00.0 -2025-04-25 15:17:44.018225: I tensorflow/core/common_runtime/direct_session.cc:378] Device mapping: -/job:localhost/replica:0/task:0/device:GPU:0 -> device: 0, name: AMD Radeon RX 9070 XT, pci bus id: 0000:0d:00.0 - -Num GPUs Available: 1 -Name: /physical_device:GPU:0 Type: GPU -Warning: Resource leak detected by SharedSignalPool, 385 Signals leaked. -pid:1 tid:0x7f4476ac1740 [~VaMgr] frag_map_ size is not 1. -``` - -## Intel Arc GPU Example - -First, install Intel GPU drivers (https://dgpu-docs.intel.com/driver/installation.html), then install Intel container toolkit (https://github.com/intel/intel-device-plugins-for-kubernetes/tree/main/cmd/gpu_plugin) - -Once that is done, check if you can get gpu details by running `clinfo`: - -```bash -root@gpu-1:/repos/ocean/ocean-node# clinfo -Number of platforms: 1 - Platform #0: Intel(R) OpenCL Graphics - Number of devices: 1 - Device #0: Intel(R) Arc(TM) A770M Graphics - Board name: Intel Arc Graphics - Vendor ID: 0x8086 - Device ID: 0x56a0 - Device Topology (NV12): PCI[ B#3 D#0 F#0 ] - Max compute units: 32 - Max clock frequency: 2400 MHz - Device extensions: cl_khr_fp64 cl_khr_fp16 cl_intel_subgroups ... -``` - -Now, get the device UUID: - -```bash -root@gpu-1:/repos/ocean/ocean-node# lspci -D | grep VGA -0000:03:00.0 VGA compatible controller: Intel Corporation Arc Graphics -``` - -For container runtime, Intel Arc GPUs use `/dev/dri/renderD128` or similar: - -```bash -root@gpu-1:/repos/ocean/ocean-node# ls -la /dev/dri/ -crw-rw---- 1 root render 226, 0 Apr 25 10:00 card0 -crw-rw---- 1 root render 226, 128 Apr 25 10:00 renderD128 -``` - -Now, we can define the GPU for the node: - -```json -{ - "id": "intelGPU", - "description": "Intel Arc A770M Graphics", - "type": "gpu", - "total": 1, - "init": { - "advanced": { - "Devices": ["/dev/dri/renderD128", "/dev/dri/card0"], - "GroupAdd": ["video", "render"], - "CapAdd": ["SYS_ADMIN"] - } - }, - "driverVersion": "32.0.101.8531", - "memoryTotal": "16384 MiB" -} -``` - -Here is the full definition of DOCKER_COMPUTE_ENVIRONMENTS with Intel GPU: - -```json -[ - { - "socketPath": "/var/run/docker.sock", - "resources": [ - { - "id": "intelGPU", - "description": "Intel Arc A770M Graphics", - "type": "gpu", - "total": 1, - "init": { - "advanced": { - "Devices": ["/dev/dri/renderD128", "/dev/dri/card0"], - "GroupAdd": ["video", "render"], - "CapAdd": ["SYS_ADMIN"] - } - }, - "driverVersion": "32.0.101.8531", - "memoryTotal": "16384 MiB" - }, - { "id": "disk", "total": 1 } - ], - "storageExpiry": 604800, - "maxJobDuration": 3600, - "minJobDuration": 60, - "fees": { - "1": [ - { - "feeToken": "0x123", - "prices": [ - { "id": "cpu", "price": 1 }, - { "id": "intelGPU", "price": 2 } - ] - } - ] - }, - "free": { - "maxJobDuration": 60, - "minJobDuration": 10, - "maxJobs": 3, - "resources": [ - { "id": "cpu", "max": 1 }, - { "id": "ram", "max": 1 }, - { "id": "disk", "max": 1 }, - { "id": "intelGPU", "max": 1 } - ] - } - } -] -``` - -Verify you have it in your compute environments: - -```bash -root@gpu-1:/repos/ocean/ocean-node# curl http://localhost:8000/api/services/computeEnvironments -``` - -```json -[ - { - "id": "0xaa1234567890abcdef1234567890abcdef1234567890abcdef1234567890ab-0xbb0987654321fedcba0987654321fedcba0987654321fedcba0987654321fed", - "runningJobs": 0, - "consumerAddress": "0x00", - "platform": { "architecture": "x86_64", "os": "Ubuntu 22.04.3 LTS" }, - "fees": { - "1": [ - { - "feeToken": "0x123", - "prices": [ - { "id": "cpu", "price": 1 }, - { "id": "intelGPU", "price": 2 } - ] - } - ] - }, - "storageExpiry": 604800, - "maxJobDuration": 3600, - "minJobDuration": 60, - "resources": [ - { "id": "cpu", "total": 16, "max": 16, "min": 1, "inUse": 0 }, - { - "id": "ram", - "total": 32, - "max": 32, - "min": 1, - "inUse": 0 - }, - { - "id": "intelGPU", - "description": "Intel Arc A770M Graphics", - "type": "gpu", - "total": 1, - "init": { - "advanced": { - "Devices": ["/dev/dri/renderD128", "/dev/dri/card0"], - "GroupAdd": ["video", "render"], - "CapAdd": ["SYS_ADMIN"] - } - }, - "driverVersion": "32.0.101.8531", - "memoryTotal": "16384 MiB" - "max": 1, - "min": 0, - "inUse": 0 - }, - { "id": "disk", "total": 1, "max": 1, "min": 0, "inUse": 0 } - ], - "free": { - "maxJobDuration": 60, - "minJobDuration": 10, - "maxJobs": 3, - "resources": [ - { "id": "cpu", "max": 1, "inUse": 0 }, - { "id": "ram", "max": 1, "inUse": 0 }, - { "id": "disk", "max": 1, "inUse": 0 }, - { "id": "intelGPU", "max": 1, "inUse": 0 } - ] - }, - "runningfreeJobs": 0 - } -] -``` - -Start a free job using Intel GPU with: - -```json -{ - "command": "freeStartCompute", - "algorithm": { - "meta": { - "container": { - "image": "intel/oneapi-runtime", - "tag": "2024.0-devel-ubuntu22.04", - "entrypoint": "python $ALGO" - }, - "rawcode": "import os\nprint('GPU device available:')\nos.system('clinfo')" - } - }, - "consumerAddress": "0x00", - "signature": "123", - "nonce": 1, - "environment": "0xaa1234567890abcdef1234567890abcdef1234567890abcdef1234567890ab-0xbb0987654321fedcba0987654321fedcba0987654321fedcba0987654321fed", - "resources": [ - { - "id": "cpu", - "amount": 1 - }, - { - "id": "intelGPU", - "amount": 1 - } - ] -} -``` - -And the output of `getComputeResult` should look like: - -```bash -Number of platforms: 1 - Platform #0: Intel(R) OpenCL Graphics - Number of devices: 1 - Device #0: Intel(R) Arc(TM) A770M Graphics - Board name: Intel Arc Graphics - Vendor ID: 0x8086 - Device ID: 0x56a0 - Device Topology (NV12): PCI[ B#3 D#0 F#0 ] - Max compute units: 32 - Max clock frequency: 2400 MHz - Device extensions: cl_khr_fp64 cl_khr_fp16 cl_intel_subgroups ... -``` diff --git a/docs/Ocean Node.postman_collection.json b/docs/Ocean Node.postman_collection.json index 8f5f7a128..aab3d465c 100644 --- a/docs/Ocean Node.postman_collection.json +++ b/docs/Ocean Node.postman_collection.json @@ -1,193 +1,2426 @@ { - "info": { - "_postman_id": "ff8f2614-8d77-40e4-9031-9ca2ed9f7973", - "name": "Ocean Node", - "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" - }, - "item": [ - { - "name": "8000 - getP2pPeers", - "request": { - "method": "GET", - "header": [] - }, - "response": [] - }, - { - "name": "8001 - getP2pPeers", - "request": { - "method": "GET", - "header": [] - }, - "response": [] - }, - { - "name": "8000 - getOceanPeers", - "request": { - "method": "GET", - "header": [] - }, - "response": [] - }, - { - "name": "8001 - getOceanPeers", - "request": { - "method": "GET", - "header": [] - }, - "response": [] - }, - { - "name": "8000 - getPeer", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "http://127.0.0.1:8000/getP2pPeer?peerId=16Uiu2HAmQU8YmsACkFjkaFqEECLN3Csu6JgoU3hw9EsPmk7i9TFL", - "protocol": "http", - "host": ["127", "0", "0", "1"], - "port": "8000", - "path": ["getP2pPeer"], - "query": [ - { - "key": "peerId", - "value": "16Uiu2HAmQU8YmsACkFjkaFqEECLN3Csu6JgoU3hw9EsPmk7i9TFL" - } - ] - } - } - }, - { - "name": "8000 - advertiseDid", - "request": { - "method": "GET", - "header": [] - }, - "response": [] - }, - { - "name": "8001 - advertiseDid", - "request": { - "method": "GET", - "header": [] - }, - "response": [] - }, - { - "name": "8000 - getProvidersForDid", - "request": { - "method": "GET", - "header": [] - }, - "response": [] - }, - { - "name": "8001 - getProvidersForDid", - "request": { - "method": "GET", - "header": [] - }, - "response": [] - }, - { - "name": "8000 - directCommand (findDDO)", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "type": "default" - } - ] - }, - "body": { - "mode": "raw", - "raw": "{\n \"command\": \"findDDO\",\n \"id\": \"did:op:0ebed8226ada17fde24b6bf2b95d27f8f05fcce09139ff5cec31f6d81a7cd2ea\"\n}" - }, - "url": { - "raw": "http://127.0.0.1:8000/directCommand", - "protocol": "http", - "host": ["127", "0", "0", "1"], - "port": "8000", - "path": ["directCommand"] - }, - "response": [] - }, - { - "name": "8001 - directCommand (findDDO)", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "type": "default" - } - ] - }, - "body": { - "mode": "raw", - "raw": "{\n \"command\": \"findDDO\",\n \"id\": \"did:op:0ebed8226ada17fde24b6bf2b95d27f8f05fcce09139ff5cec31f6d81a7cd2ea\", \"node\": \"16Uiu2HAkvfXgYiFhsHRJvcdtmMs3aopgoRphb5xnXMh3dxCRuuX\"\n}" - }, - "url": { - "raw": "http://127.0.0.1:8001/directCommand", - "protocol": "http", - "host": ["127", "0", "0", "1"], - "port": "8001", - "path": ["directCommand"] - }, - "response": [] - }, - { - "name": "8000 - directCommand", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "type": "default" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"command\":\"downloadURL\",\n \"node\": \"16Uiu2HAkxiemC25d2iZWTkVRQmZr9L9h3RNGnhiUWXEonmsPEC8y\",\n \"url\": \"http://example.com\",\n \"aes_encrypted_key\": \"0x1234567890abcdef\"\n}" - }, - "url": { - "raw": "http://127.0.0.1:8000/directCommand", - "protocol": "http", - "host": ["127", "0", "0", "1"], - "port": "8000", - "path": ["directCommand"] - } - }, - "response": [] - }, - { - "name": "8001 - directCommand", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "type": "default" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"command\":\"downloadURL\",\n \"node\": \"16Uiu2HAkvfXgYiFhsHRJvcdtmMs3aopgoRphb5xnXMh3dxCRuuX\",\n \"url\": \"http://example.com\",\n \"aes_encrypted_key\": \"0x1234567890abcdef\"\n}" - }, - "url": { - "raw": "http://127.0.0.1:8001/directCommand", - "protocol": "http", - "host": ["127", "0", "0", "1"], - "port": "8001", - "path": ["directCommand"] - } - }, - "response": [] - } - ] -} + "info": { + "name": "Ocean Node API", + "_postman_id": "ocean-node-http-api", + "description": "Complete collection of HTTP endpoints exposed by an Ocean Node's HTTP server.\n\nSet the `baseUrl` collection variable to your node (default `http://localhost:8000`). Many endpoints are authenticated with a signature over `consumerAddress` + `nonce` + `command`, or with an auth-token `Authorization` header. Convenience variables (`consumerAddress`, `signature`, `nonce`, `chainId`, `did`, `serviceId`, `jobId`, `token`, `bucketId`, `wallet`, `owner`) are provided as placeholders.\n\nGenerated from the route definitions in `src/components/httpRoutes/` and their handler implementations.", + "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" + }, + "variable": [ + { + "key": "baseUrl", + "value": "http://localhost:8000", + "type": "string" + }, + { + "key": "consumerAddress", + "value": "0x0000000000000000000000000000000000000000", + "type": "string" + }, + { + "key": "signature", + "value": "0x", + "type": "string" + }, + { + "key": "nonce", + "value": "1", + "type": "string" + }, + { + "key": "chainId", + "value": "8996", + "type": "string" + }, + { + "key": "did", + "value": "did:op:0000000000000000000000000000000000000000000000000000000000000000", + "type": "string" + }, + { + "key": "serviceId", + "value": "", + "type": "string" + }, + { + "key": "jobId", + "value": "", + "type": "string" + }, + { + "key": "token", + "value": "", + "type": "string" + }, + { + "key": "bucketId", + "value": "", + "type": "string" + }, + { + "key": "wallet", + "value": "0x0000000000000000000000000000000000000000", + "type": "string" + }, + { + "key": "owner", + "value": "0x0000000000000000000000000000000000000000", + "type": "string" + } + ], + "item": [ + { + "name": "Node Info", + "item": [ + { + "name": "Get Node Info", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "" + ] + }, + "description": "Returns node identity and the list of available service endpoints (nodeId, chainIds, providerAddress, nodePublicKey, serviceEndpoints, software, version)." + } + } + ] + }, + { + "name": "Aquarius (DDO)", + "item": [ + { + "name": "Get DDO by DID", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/api/aquarius/assets/ddo/{{did}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "aquarius", + "assets", + "ddo", + "{{did}}" + ] + }, + "description": "Retrieve the full DDO for a DID. Append an optional `/true` path segment to force a fresh lookup." + } + }, + { + "name": "Get DDO Metadata by DID", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/api/aquarius/assets/metadata/{{did}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "aquarius", + "assets", + "metadata", + "{{did}}" + ] + }, + "description": "Retrieve DDO metadata for a DID. Optional trailing `/true` forces a fresh lookup." + } + }, + { + "name": "Query Assets", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"query\": {\n \"match_all\": {}\n },\n \"from\": 0,\n \"size\": 10\n}" + }, + "url": { + "raw": "{{baseUrl}}/api/aquarius/assets/metadata/query", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "aquarius", + "assets", + "metadata", + "query" + ] + }, + "description": "Query indexed assets. Body is a search query object (filter/query/sort/from/size)." + } + }, + { + "name": "Get DDO State", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/api/aquarius/state/ddo?did={{did}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "aquarius", + "state", + "ddo" + ], + "query": [ + { + "key": "did", + "value": "{{did}}" + }, + { + "key": "nft", + "value": "", + "disabled": true + }, + { + "key": "txId", + "value": "", + "disabled": true + } + ] + }, + "description": "Query DDO state by `did`, `nft`, or `txId` (at least one required)." + } + }, + { + "name": "Validate DDO", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "{{token}}", + "disabled": true + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"ddo\": {},\n \"publisherAddress\": \"{{consumerAddress}}\",\n \"signature\": \"{{signature}}\",\n \"nonce\": \"{{nonce}}\"\n}" + }, + "url": { + "raw": "{{baseUrl}}/api/aquarius/assets/ddo/validate", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "aquarius", + "assets", + "ddo", + "validate" + ] + }, + "description": "Validate a DDO's schema and signature before publishing." + } + } + ] + }, + { + "name": "Provider", + "item": [ + { + "name": "Get Nonce", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/api/services/nonce?userAddress={{consumerAddress}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "nonce" + ], + "query": [ + { + "key": "userAddress", + "value": "{{consumerAddress}}" + } + ] + }, + "description": "Get the current nonce for a user address (used when signing requests)." + } + }, + { + "name": "Initialize (get fees)", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/api/services/initialize?documentId={{did}}&serviceId=&consumerAddress={{consumerAddress}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "initialize" + ], + "query": [ + { + "key": "documentId", + "value": "{{did}}" + }, + { + "key": "serviceId", + "value": "" + }, + { + "key": "consumerAddress", + "value": "{{consumerAddress}}" + }, + { + "key": "validUntil", + "value": "", + "disabled": true + }, + { + "key": "policyServer", + "value": "", + "disabled": true + } + ] + }, + "description": "Get the fees required to access an asset service." + } + }, + { + "name": "Encrypt", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/octet-stream" + } + ], + "body": { + "mode": "raw", + "raw": "hello world" + }, + "url": { + "raw": "{{baseUrl}}/api/services/encrypt?nonce={{nonce}}&consumerAddress={{consumerAddress}}&signature={{signature}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "encrypt" + ], + "query": [ + { + "key": "nonce", + "value": "{{nonce}}" + }, + { + "key": "consumerAddress", + "value": "{{consumerAddress}}" + }, + { + "key": "signature", + "value": "{{signature}}" + } + ] + }, + "description": "ECIES-encrypt the raw request body. Returns application/octet-stream. 25MB limit." + } + }, + { + "name": "Encrypt File", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"files\": {\n \"type\": \"url\",\n \"url\": \"https://example.com/file.txt\",\n \"method\": \"GET\"\n }\n}" + }, + "url": { + "raw": "{{baseUrl}}/api/services/encryptFile?nonce={{nonce}}&consumerAddress={{consumerAddress}}&signature={{signature}}&encryptMethod=ECIES", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "encryptFile" + ], + "query": [ + { + "key": "nonce", + "value": "{{nonce}}" + }, + { + "key": "consumerAddress", + "value": "{{consumerAddress}}" + }, + { + "key": "signature", + "value": "{{signature}}" + }, + { + "key": "encryptMethod", + "value": "ECIES" + } + ] + }, + "description": "Encrypt a file (AES or ECIES). Accepts a StorageObject JSON, raw binary, or multipart. Returns encrypted bytes with X-Encrypted-By / X-Encrypted-Method headers." + } + }, + { + "name": "Decrypt", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"decrypterAddress\": \"{{consumerAddress}}\",\n \"chainId\": {{chainId}},\n \"nonce\": \"{{nonce}}\",\n \"signature\": \"{{signature}}\",\n \"transactionId\": \"\",\n \"dataNftAddress\": \"\",\n \"encryptedDocument\": \"\",\n \"flags\": 0,\n \"documentHash\": \"\"\n}" + }, + "url": { + "raw": "{{baseUrl}}/api/services/decrypt", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "decrypt" + ] + }, + "description": "Decrypt a DDO document. Returns the decrypted payload as text/plain." + } + }, + { + "name": "Download", + "request": { + "method": "GET", + "header": [ + { + "key": "Authorization", + "value": "{{token}}", + "disabled": true + } + ], + "url": { + "raw": "{{baseUrl}}/api/services/download?fileIndex=0&documentId={{did}}&serviceId=&transferTxId=&nonce={{nonce}}&consumerAddress={{consumerAddress}}&signature={{signature}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "download" + ], + "query": [ + { + "key": "fileIndex", + "value": "0" + }, + { + "key": "documentId", + "value": "{{did}}" + }, + { + "key": "serviceId", + "value": "" + }, + { + "key": "transferTxId", + "value": "" + }, + { + "key": "nonce", + "value": "{{nonce}}" + }, + { + "key": "consumerAddress", + "value": "{{consumerAddress}}" + }, + { + "key": "signature", + "value": "{{signature}}" + }, + { + "key": "userdata", + "value": "", + "disabled": true + }, + { + "key": "policyServer", + "value": "", + "disabled": true + } + ] + }, + "description": "Download an asset file after a valid transfer (order). Streams the file." + } + } + ] + }, + { + "name": "File Info", + "item": [ + { + "name": "File Info", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"did\": \"{{did}}\",\n \"serviceId\": \"\",\n \"consumerAddress\": \"{{consumerAddress}}\"\n}" + }, + "url": { + "raw": "{{baseUrl}}/api/services/fileInfo", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "fileInfo" + ] + }, + "description": "Get file info for an asset (by `did`+`serviceId`) or by a raw file `type` descriptor. `consumerAddress` is required for NODE_PERSISTENT_STORAGE ACL gating." + } + } + ] + }, + { + "name": "Compute", + "item": [ + { + "name": "Get Compute Environments", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/api/services/computeEnvironments?chainId={{chainId}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "computeEnvironments" + ], + "query": [ + { + "key": "chainId", + "value": "{{chainId}}" + }, + { + "key": "node", + "value": "", + "disabled": true + } + ] + }, + "description": "List available compute environments (optionally filtered by chain)." + } + }, + { + "name": "Initialize Compute", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"algorithm\": { \"documentId\": \"{{did}}\", \"serviceId\": \"\" },\n \"datasets\": [\n { \"documentId\": \"{{did}}\", \"serviceId\": \"\" }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/api/services/initializeCompute", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "initializeCompute" + ] + }, + "description": "Validate algorithm + datasets and return a price/initialization quote." + } + }, + { + "name": "Start Compute (paid)", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "{{token}}", + "disabled": true + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"consumerAddress\": \"{{consumerAddress}}\",\n \"signature\": \"{{signature}}\",\n \"nonce\": \"{{nonce}}\",\n \"environment\": \"\",\n \"algorithm\": { \"documentId\": \"{{did}}\", \"serviceId\": \"\" },\n \"datasets\": [ { \"documentId\": \"{{did}}\", \"serviceId\": \"\" } ],\n \"maxJobDuration\": 3600,\n \"resources\": [ { \"id\": \"cpu\", \"amount\": 1 }, { \"id\": \"ram\", \"amount\": 1 } ],\n \"payment\": { \"chainId\": {{chainId}}, \"token\": \"0x...\" }\n}" + }, + "url": { + "raw": "{{baseUrl}}/api/services/compute", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "compute" + ] + }, + "description": "Start a paid compute job. Optional fields: policyServer, metadata, additionalViewers, queueMaxWaitTime, encryptedDockerRegistryAuth, output, outputBucketId." + } + }, + { + "name": "Start Free Compute", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"consumerAddress\": \"{{consumerAddress}}\",\n \"signature\": \"{{signature}}\",\n \"nonce\": \"{{nonce}}\",\n \"environment\": \"\",\n \"algorithm\": { \"documentId\": \"{{did}}\", \"serviceId\": \"\" },\n \"datasets\": [],\n \"resources\": [ { \"id\": \"cpu\", \"amount\": 1 }, { \"id\": \"ram\", \"amount\": 1 } ],\n \"maxJobDuration\": 600\n}" + }, + "url": { + "raw": "{{baseUrl}}/api/services/freeCompute", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "freeCompute" + ] + }, + "description": "Start a free compute job (no payment) on a free-tier environment." + } + }, + { + "name": "Get Compute Status", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/api/services/compute?consumerAddress={{consumerAddress}}&jobId={{jobId}}&agreementId=", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "compute" + ], + "query": [ + { + "key": "consumerAddress", + "value": "{{consumerAddress}}" + }, + { + "key": "jobId", + "value": "{{jobId}}" + }, + { + "key": "agreementId", + "value": "" + }, + { + "key": "node", + "value": "", + "disabled": true + } + ] + }, + "description": "Get the status of a compute job." + } + }, + { + "name": "Stop Compute", + "request": { + "method": "PUT", + "header": [ + { + "key": "Authorization", + "value": "{{token}}", + "disabled": true + } + ], + "url": { + "raw": "{{baseUrl}}/api/services/compute?consumerAddress={{consumerAddress}}&signature={{signature}}&nonce={{nonce}}&jobId={{jobId}}&agreementId=", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "compute" + ], + "query": [ + { + "key": "consumerAddress", + "value": "{{consumerAddress}}" + }, + { + "key": "signature", + "value": "{{signature}}" + }, + { + "key": "nonce", + "value": "{{nonce}}" + }, + { + "key": "jobId", + "value": "{{jobId}}" + }, + { + "key": "agreementId", + "value": "" + }, + { + "key": "node", + "value": "", + "disabled": true + } + ] + }, + "description": "Stop a running compute job (parameters are query strings)." + } + }, + { + "name": "Get Compute Result", + "request": { + "method": "GET", + "header": [ + { + "key": "Authorization", + "value": "{{token}}", + "disabled": true + } + ], + "url": { + "raw": "{{baseUrl}}/api/services/computeResult?consumerAddress={{consumerAddress}}&jobId={{jobId}}&index=0&signature={{signature}}&nonce={{nonce}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "computeResult" + ], + "query": [ + { + "key": "consumerAddress", + "value": "{{consumerAddress}}" + }, + { + "key": "jobId", + "value": "{{jobId}}" + }, + { + "key": "index", + "value": "0" + }, + { + "key": "signature", + "value": "{{signature}}" + }, + { + "key": "nonce", + "value": "{{nonce}}" + }, + { + "key": "node", + "value": "", + "disabled": true + } + ] + }, + "description": "Download a compute job result by index. Streams the result file." + } + }, + { + "name": "Get Compute Streamable Logs", + "request": { + "method": "GET", + "header": [ + { + "key": "Authorization", + "value": "{{token}}", + "disabled": true + } + ], + "url": { + "raw": "{{baseUrl}}/api/services/computeStreamableLogs?consumerAddress={{consumerAddress}}&jobId={{jobId}}&signature={{signature}}&nonce={{nonce}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "computeStreamableLogs" + ], + "query": [ + { + "key": "consumerAddress", + "value": "{{consumerAddress}}" + }, + { + "key": "jobId", + "value": "{{jobId}}" + }, + { + "key": "signature", + "value": "{{signature}}" + }, + { + "key": "nonce", + "value": "{{nonce}}" + }, + { + "key": "node", + "value": "", + "disabled": true + } + ] + }, + "description": "Stream real-time logs from a running compute job (404 if not running)." + } + } + ] + }, + { + "name": "Service on Demand", + "item": [ + { + "name": "Get Service Templates", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/api/services/serviceTemplates?chainId={{chainId}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "serviceTemplates" + ], + "query": [ + { + "key": "chainId", + "value": "{{chainId}}" + }, + { + "key": "node", + "value": "", + "disabled": true + } + ] + }, + "description": "List operator-published service templates (sanitized). Not authenticated." + } + }, + { + "name": "Start Service", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "{{token}}", + "disabled": true + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"consumerAddress\": \"{{consumerAddress}}\",\n \"nonce\": \"{{nonce}}\",\n \"signature\": \"{{signature}}\",\n \"environment\": \"\",\n \"image\": \"nginxinc/nginx-unprivileged\",\n \"tag\": \"alpine\",\n \"exposedPorts\": [8080],\n \"resources\": [ { \"id\": \"cpu\", \"amount\": 1 }, { \"id\": \"ram\", \"amount\": 1 } ],\n \"duration\": 3600,\n \"userData\": \"\",\n \"payment\": { \"chainId\": {{chainId}}, \"token\": \"0x...\" }\n}" + }, + "url": { + "raw": "{{baseUrl}}/api/services/serviceStart", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "serviceStart" + ] + }, + "description": "Launch a long-running service container, paid via escrow. Optional: checksum, dockerfile, additionalDockerFiles, dockerCmd, dockerEntrypoint. Services must listen on a high port (>1024)." + } + }, + { + "name": "Get Service Status", + "request": { + "method": "GET", + "header": [ + { + "key": "Authorization", + "value": "{{token}}", + "disabled": true + } + ], + "url": { + "raw": "{{baseUrl}}/api/services/serviceStatus?consumerAddress={{consumerAddress}}&nonce={{nonce}}&signature={{signature}}&serviceId={{serviceId}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "serviceStatus" + ], + "query": [ + { + "key": "consumerAddress", + "value": "{{consumerAddress}}" + }, + { + "key": "nonce", + "value": "{{nonce}}" + }, + { + "key": "signature", + "value": "{{signature}}" + }, + { + "key": "serviceId", + "value": "{{serviceId}}" + }, + { + "key": "node", + "value": "", + "disabled": true + } + ] + }, + "description": "Read service status/endpoints. Authenticated and owner-scoped. Omit serviceId to list all owned services." + } + }, + { + "name": "List Services", + "request": { + "method": "GET", + "header": [ + { + "key": "Authorization", + "value": "{{token}}", + "disabled": true + } + ], + "url": { + "raw": "{{baseUrl}}/api/services/serviceList?consumerAddress={{consumerAddress}}&nonce={{nonce}}&signature={{signature}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "serviceList" + ], + "query": [ + { + "key": "consumerAddress", + "value": "{{consumerAddress}}" + }, + { + "key": "nonce", + "value": "{{nonce}}" + }, + { + "key": "signature", + "value": "{{signature}}" + }, + { + "key": "status", + "value": "70", + "description": "one specific ServiceStatusNumber (e.g. 70 = Stopped, 75 = Expired); takes precedence over includeAllStatuses", + "disabled": true + }, + { + "key": "includeAllStatuses", + "value": "true", + "description": "every status instead of only the resource-holding set", + "disabled": true + }, + { + "key": "fromTimestamp", + "value": "2026-01-01T00:00:00Z", + "description": "only services created at/after this moment (ISO date or Unix timestamp)", + "disabled": true + }, + { + "key": "node", + "value": "", + "disabled": true + } + ] + }, + "description": "Node-wide service listing (authenticated, NOT owner-scoped). Default: only services currently holding a resource reservation (Running/Restarting/Stopping, mid-start states, paid Error, Stopped within the paid window). Filter with status / includeAllStatuses / fromTimestamp. Output is listing-sanitized: no userData, dockerCmd, dockerEntrypoint or Dockerfile." + } + }, + { + "name": "Extend Service", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "{{token}}", + "disabled": true + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"consumerAddress\": \"{{consumerAddress}}\",\n \"nonce\": \"{{nonce}}\",\n \"signature\": \"{{signature}}\",\n \"serviceId\": \"{{serviceId}}\",\n \"additionalDuration\": 1800,\n \"payment\": { \"chainId\": {{chainId}}, \"token\": \"0x...\" }\n}" + }, + "url": { + "raw": "{{baseUrl}}/api/services/serviceExtend", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "serviceExtend" + ] + }, + "description": "Pay to push the service expiry further out (additionalDuration in seconds, must be positive)." + } + }, + { + "name": "Restart Service", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "{{token}}", + "disabled": true + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"consumerAddress\": \"{{consumerAddress}}\",\n \"nonce\": \"{{nonce}}\",\n \"signature\": \"{{signature}}\",\n \"serviceId\": \"{{serviceId}}\",\n \"userData\": \"\"\n}" + }, + "url": { + "raw": "{{baseUrl}}/api/services/serviceRestart", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "serviceRestart" + ] + }, + "description": "Recreate the service container (no extra charge), keeping the same hostPort and expiresAt. Optional userData replaces stored env vars." + } + }, + { + "name": "Stop Service", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "{{token}}", + "disabled": true + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"consumerAddress\": \"{{consumerAddress}}\",\n \"nonce\": \"{{nonce}}\",\n \"signature\": \"{{signature}}\",\n \"serviceId\": \"{{serviceId}}\"\n}" + }, + "url": { + "raw": "{{baseUrl}}/api/services/serviceStop", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "serviceStop" + ] + }, + "description": "Tear down the service container and network and release resources. Owner-gated." + } + }, + { + "name": "Get Service Streamable Logs", + "request": { + "method": "GET", + "header": [ + { + "key": "Authorization", + "value": "{{token}}", + "disabled": true + } + ], + "url": { + "raw": "{{baseUrl}}/api/services/serviceStreamableLogs?consumerAddress={{consumerAddress}}&serviceId={{serviceId}}&signature={{signature}}&nonce={{nonce}}&since=1h", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "serviceStreamableLogs" + ], + "query": [ + { + "key": "consumerAddress", + "value": "{{consumerAddress}}" + }, + { + "key": "serviceId", + "value": "{{serviceId}}" + }, + { + "key": "signature", + "value": "{{signature}}" + }, + { + "key": "nonce", + "value": "{{nonce}}" + }, + { + "key": "since", + "value": "1h", + "description": "Optional: Unix timestamp (seconds) or relative duration like 30s/45m/2h/7d. Omit for full history + live follow." + }, + { + "key": "node", + "value": "", + "disabled": true + } + ] + }, + "description": "Stream real-time stdout/stderr logs from a service container. Authenticated and owner-scoped (401 if consumerAddress is not the owner). Available while Running or Error; 404 otherwise. Optional `since` skips history and starts from a recent point (Unix timestamp or relative duration e.g. 1h)." + } + } + ] + }, + { + "name": "Persistent Storage", + "item": [ + { + "name": "Create Bucket", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "{{token}}", + "disabled": true + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"consumerAddress\": \"{{consumerAddress}}\",\n \"signature\": \"{{signature}}\",\n \"nonce\": \"{{nonce}}\",\n \"label\": \"my-bucket\",\n \"accessLists\": []\n}" + }, + "url": { + "raw": "{{baseUrl}}/api/services/persistentStorage/buckets", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "persistentStorage", + "buckets" + ] + }, + "description": "Create a new persistent storage bucket owned by consumerAddress." + } + }, + { + "name": "Update Bucket", + "request": { + "method": "PATCH", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "{{token}}", + "disabled": true + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"label\": \"renamed-bucket\"\n}" + }, + "url": { + "raw": "{{baseUrl}}/api/services/persistentStorage/buckets/{{bucketId}}?consumerAddress={{consumerAddress}}&signature={{signature}}&nonce={{nonce}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "persistentStorage", + "buckets", + "{{bucketId}}" + ], + "query": [ + { + "key": "consumerAddress", + "value": "{{consumerAddress}}" + }, + { + "key": "signature", + "value": "{{signature}}" + }, + { + "key": "nonce", + "value": "{{nonce}}" + } + ] + }, + "description": "Update a bucket (e.g. set its label)." + } + }, + { + "name": "List Buckets", + "request": { + "method": "GET", + "header": [ + { + "key": "Authorization", + "value": "{{token}}", + "disabled": true + } + ], + "url": { + "raw": "{{baseUrl}}/api/services/persistentStorage/buckets?consumerAddress={{consumerAddress}}&signature={{signature}}&nonce={{nonce}}&owner={{owner}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "persistentStorage", + "buckets" + ], + "query": [ + { + "key": "consumerAddress", + "value": "{{consumerAddress}}" + }, + { + "key": "signature", + "value": "{{signature}}" + }, + { + "key": "nonce", + "value": "{{nonce}}" + }, + { + "key": "owner", + "value": "{{owner}}" + } + ] + }, + "description": "List buckets for an owner (filtered by access lists for the calling consumer)." + } + }, + { + "name": "List Files in Bucket", + "request": { + "method": "GET", + "header": [ + { + "key": "Authorization", + "value": "{{token}}", + "disabled": true + } + ], + "url": { + "raw": "{{baseUrl}}/api/services/persistentStorage/buckets/{{bucketId}}/files?consumerAddress={{consumerAddress}}&signature={{signature}}&nonce={{nonce}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "persistentStorage", + "buckets", + "{{bucketId}}", + "files" + ], + "query": [ + { + "key": "consumerAddress", + "value": "{{consumerAddress}}" + }, + { + "key": "signature", + "value": "{{signature}}" + }, + { + "key": "nonce", + "value": "{{nonce}}" + } + ] + }, + "description": "List all files in a bucket." + } + }, + { + "name": "Get File Object", + "request": { + "method": "GET", + "header": [ + { + "key": "Authorization", + "value": "{{token}}", + "disabled": true + } + ], + "url": { + "raw": "{{baseUrl}}/api/services/persistentStorage/buckets/{{bucketId}}/files/myfile.txt/object?consumerAddress={{consumerAddress}}&signature={{signature}}&nonce={{nonce}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "persistentStorage", + "buckets", + "{{bucketId}}", + "files", + "myfile.txt", + "object" + ], + "query": [ + { + "key": "consumerAddress", + "value": "{{consumerAddress}}" + }, + { + "key": "signature", + "value": "{{signature}}" + }, + { + "key": "nonce", + "value": "{{nonce}}" + } + ] + }, + "description": "Get the file object metadata for a file in a bucket." + } + }, + { + "name": "Upload File", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/octet-stream" + }, + { + "key": "Authorization", + "value": "{{token}}", + "disabled": true + } + ], + "body": { + "mode": "raw", + "raw": "" + }, + "url": { + "raw": "{{baseUrl}}/api/services/persistentStorage/buckets/{{bucketId}}/files/myfile.txt?consumerAddress={{consumerAddress}}&signature={{signature}}&nonce={{nonce}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "persistentStorage", + "buckets", + "{{bucketId}}", + "files", + "myfile.txt" + ], + "query": [ + { + "key": "consumerAddress", + "value": "{{consumerAddress}}" + }, + { + "key": "signature", + "value": "{{signature}}" + }, + { + "key": "nonce", + "value": "{{nonce}}" + } + ] + }, + "description": "Upload a file to a bucket. Body is the raw file bytes (supports chunked uploads)." + } + }, + { + "name": "Delete File", + "request": { + "method": "DELETE", + "header": [ + { + "key": "Authorization", + "value": "{{token}}", + "disabled": true + } + ], + "url": { + "raw": "{{baseUrl}}/api/services/persistentStorage/buckets/{{bucketId}}/files/myfile.txt?consumerAddress={{consumerAddress}}&signature={{signature}}&nonce={{nonce}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "persistentStorage", + "buckets", + "{{bucketId}}", + "files", + "myfile.txt" + ], + "query": [ + { + "key": "consumerAddress", + "value": "{{consumerAddress}}" + }, + { + "key": "signature", + "value": "{{signature}}" + }, + { + "key": "nonce", + "value": "{{nonce}}" + } + ] + }, + "description": "Delete a file from a bucket. Returns { success: true }." + } + } + ] + }, + { + "name": "Escrow", + "item": [ + { + "name": "Get Escrow Events", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/api/services/escrow/events?chainId={{chainId}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "escrow", + "events" + ], + "query": [ + { + "key": "chainId", + "value": "{{chainId}}" + }, + { + "key": "eventType", + "value": "", + "disabled": true + }, + { + "key": "payer", + "value": "", + "disabled": true + }, + { + "key": "payee", + "value": "", + "disabled": true + }, + { + "key": "token", + "value": "", + "disabled": true + }, + { + "key": "jobId", + "value": "", + "disabled": true + }, + { + "key": "txId", + "value": "", + "disabled": true + }, + { + "key": "offset", + "value": "", + "disabled": true + }, + { + "key": "size", + "value": "", + "disabled": true + } + ] + }, + "description": "Query escrow contract events with optional filters (eventType e.g. Lock/Claimed/Canceled/Deposit/Withdraw/Auth)." + } + } + ] + }, + { + "name": "Access Lists", + "item": [ + { + "name": "Search Access Lists by Wallet", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/api/services/accesslists?wallet={{wallet}}&chainId={{chainId}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "accesslists" + ], + "query": [ + { + "key": "wallet", + "value": "{{wallet}}" + }, + { + "key": "chainId", + "value": "{{chainId}}", + "disabled": true + } + ] + }, + "description": "Search access lists that include a given wallet address." + } + }, + { + "name": "Get Access List", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/api/services/accesslists/{{chainId}}/0xContractAddress", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "accesslists", + "{{chainId}}", + "0xContractAddress" + ] + }, + "description": "Get a specific access list by chainId and contract address." + } + } + ] + }, + { + "name": "Auth", + "item": [ + { + "name": "Create Auth Token", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"address\": \"{{consumerAddress}}\",\n \"signature\": \"{{signature}}\",\n \"nonce\": \"{{nonce}}\",\n \"validUntil\": null,\n \"chainId\": {{chainId}}\n}" + }, + "url": { + "raw": "{{baseUrl}}/api/services/auth/token", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "auth", + "token" + ] + }, + "description": "Create an auth token (returned token can be used as the Authorization header on authenticated routes)." + } + }, + { + "name": "Invalidate Auth Token", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"address\": \"{{consumerAddress}}\",\n \"signature\": \"{{signature}}\",\n \"token\": \"{{token}}\",\n \"nonce\": \"{{nonce}}\",\n \"chainId\": {{chainId}}\n}" + }, + "url": { + "raw": "{{baseUrl}}/api/services/auth/token/invalidate", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "auth", + "token", + "invalidate" + ] + }, + "description": "Invalidate an existing auth token." + } + } + ] + }, + { + "name": "Policy Server", + "item": [ + { + "name": "Policy Server Passthrough", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"policyServerPassthrough\": {}\n}" + }, + "url": { + "raw": "{{baseUrl}}/api/services/PolicyServerPassthrough", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "PolicyServerPassthrough" + ] + }, + "description": "Pass arbitrary data through to the configured policy server. Streams the response." + } + }, + { + "name": "Initialize PS Verification", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"documentId\": \"{{did}}\",\n \"serviceId\": \"\",\n \"consumerAddress\": \"{{consumerAddress}}\",\n \"policyServer\": {}\n}" + }, + "url": { + "raw": "{{baseUrl}}/api/services/initializePSVerification", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "initializePSVerification" + ] + }, + "description": "Initialize a policy server verification flow. Streams the response." + } + } + ] + }, + { + "name": "Admin", + "item": [ + { + "name": "Fetch Config", + "request": { + "method": "GET", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"address\": \"{{consumerAddress}}\",\n \"signature\": \"{{signature}}\",\n \"nonce\": \"{{nonce}}\"\n}" + }, + "url": { + "raw": "{{baseUrl}}/api/admin/config", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "admin", + "config" + ] + }, + "description": "Fetch the node configuration (admin-only). Note: this GET reads auth fields from the JSON body." + } + }, + { + "name": "Update Config", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"address\": \"{{consumerAddress}}\",\n \"signature\": \"{{signature}}\",\n \"nonce\": \"{{nonce}}\",\n \"config\": {}\n}" + }, + "url": { + "raw": "{{baseUrl}}/api/admin/config/update", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "admin", + "config", + "update" + ] + }, + "description": "Update the node configuration (admin-only)." + } + } + ] + }, + { + "name": "Queue & Jobs", + "item": [ + { + "name": "Get Index Queue", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/api/services/indexQueue", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "indexQueue" + ] + }, + "description": "Get the current indexing queue." + } + }, + { + "name": "Get Jobs for Identifier", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/api/services/jobs/{{jobId}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "jobs", + "{{jobId}}" + ] + }, + "description": "Get the indexer job pool for a given job identifier." + } + } + ] + }, + { + "name": "P2P Peers", + "item": [ + { + "name": "Get P2P Network Stats", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/getP2pNetworkStats", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "getP2pNetworkStats" + ] + }, + "description": "Get P2P network statistics." + } + }, + { + "name": "Get P2P Peers", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/getP2PPeers", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "getP2PPeers" + ] + }, + "description": "List all connected P2P peers." + } + }, + { + "name": "Get P2P Peer", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/getP2PPeer?peerId=", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "getP2PPeer" + ], + "query": [ + { + "key": "peerId", + "value": "" + } + ] + }, + "description": "Get details for a specific peer by peerId." + } + }, + { + "name": "Find Peer", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/findPeer?peerId=&timeout=10000", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "findPeer" + ], + "query": [ + { + "key": "peerId", + "value": "" + }, + { + "key": "timeout", + "value": "10000", + "disabled": true + } + ] + }, + "description": "Find a peer's multiaddress by peerId." + } + } + ] + }, + { + "name": "DIDs / Providers", + "item": [ + { + "name": "Get Providers for String", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/getProvidersForString?input={{did}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "getProvidersForString" + ], + "query": [ + { + "key": "input", + "value": "{{did}}" + } + ] + }, + "description": "Get the providers (peers) that serve a single string/CID." + } + }, + { + "name": "Get Providers for Strings", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "[\n \"did:op:...\",\n \"did:op:...\"\n]" + }, + "url": { + "raw": "{{baseUrl}}/getProvidersForStrings?timeout=10000", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "getProvidersForStrings" + ], + "query": [ + { + "key": "timeout", + "value": "10000", + "disabled": true + } + ] + }, + "description": "Batch lookup: get providers for an array of strings/CIDs. Body is a JSON array of strings." + } + } + ] + }, + { + "name": "Logs", + "item": [ + { + "name": "Get Logs", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"address\": \"{{consumerAddress}}\",\n \"signature\": \"{{signature}}\",\n \"nonce\": \"{{nonce}}\"\n}" + }, + "url": { + "raw": "{{baseUrl}}/logs?maxLogs=100&page=1", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "logs" + ], + "query": [ + { + "key": "maxLogs", + "value": "100" + }, + { + "key": "page", + "value": "1" + }, + { + "key": "moduleName", + "value": "", + "disabled": true + }, + { + "key": "level", + "value": "", + "disabled": true + }, + { + "key": "startTime", + "value": "", + "disabled": true + }, + { + "key": "endTime", + "value": "", + "disabled": true + } + ] + }, + "description": "Retrieve node logs (admin auth via signed body). Filter/paginate with query params." + } + }, + { + "name": "Get Log by ID", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"address\": \"{{consumerAddress}}\",\n \"signature\": \"{{signature}}\",\n \"nonce\": \"{{nonce}}\",\n \"logId\": \"123\"\n}" + }, + "url": { + "raw": "{{baseUrl}}/log/123", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "log", + "123" + ] + }, + "description": "Retrieve a single log entry by id (logId in body must match the :id path param)." + } + } + ] + }, + { + "name": "Direct Command (P2P)", + "description": "POST /directCommand is a single endpoint that dispatches to a protocol command handler chosen by the `command` field. It can run locally or be forwarded to another peer via `node`/`multiAddrs`. Below are representative command bodies; any PROTOCOL_COMMANDS value is accepted (download, encrypt, encryptFile, decryptDDO, getDDO, query, nonce, status, detailedStatus, findDDO, getFees, fileInfo, validateDDO, getComputeEnvironments, startCompute, freeStartCompute, stopCompute, getComputeStatus, getComputeStreamableLogs, getComputeResult, initializeCompute, reindexTx, reindexChain, collectFees, PolicyServerPassthrough, getP2PPeer(s), getP2PNetworkStats, findPeer, createAuthToken, invalidateAuthToken, fetchConfig, pushConfig, getLogs, jobs, persistentStorage*, getAccessList, searchAccessList, getEscrowEvents, serviceGetTemplates, serviceStart, serviceStop, serviceRestart, serviceGetStatus, serviceExtend, serviceGetStreamableLogs, stopNode, ...).", + "item": [ + { + "name": "getDDO", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"command\": \"getDDO\",\n \"id\": \"{{did}}\"\n}" + }, + "url": { + "raw": "{{baseUrl}}/directCommand", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "directCommand" + ] + }, + "description": "Fetch a DDO by id over the command interface." + } + }, + { + "name": "nonce", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"command\": \"nonce\",\n \"address\": \"{{consumerAddress}}\"\n}" + }, + "url": { + "raw": "{{baseUrl}}/directCommand", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "directCommand" + ] + }, + "description": "Get the nonce for an address." + } + }, + { + "name": "status", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"command\": \"status\"\n}" + }, + "url": { + "raw": "{{baseUrl}}/directCommand", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "directCommand" + ] + }, + "description": "Get node status. Use \"command\": \"detailedStatus\" for the extended report." + } + }, + { + "name": "query", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"command\": \"query\",\n \"query\": { \"query\": { \"match_all\": {} }, \"from\": 0, \"size\": 10 }\n}" + }, + "url": { + "raw": "{{baseUrl}}/directCommand", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "directCommand" + ] + }, + "description": "Query indexed assets." + } + }, + { + "name": "getFees", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"command\": \"getFees\",\n \"ddoId\": \"{{did}}\",\n \"serviceId\": \"\",\n \"consumerAddress\": \"{{consumerAddress}}\",\n \"validUntil\": 0\n}" + }, + "url": { + "raw": "{{baseUrl}}/directCommand", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "directCommand" + ] + }, + "description": "Get the provider fees for an asset service." + } + }, + { + "name": "fileInfo", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"command\": \"fileInfo\",\n \"did\": \"{{did}}\",\n \"serviceId\": \"\"\n}" + }, + "url": { + "raw": "{{baseUrl}}/directCommand", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "directCommand" + ] + }, + "description": "Get file info for an asset or a raw file descriptor (type/url/...)." + } + }, + { + "name": "download", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"command\": \"download\",\n \"fileIndex\": 0,\n \"documentId\": \"{{did}}\",\n \"serviceId\": \"\",\n \"transferTxId\": \"\",\n \"consumerAddress\": \"{{consumerAddress}}\",\n \"signature\": \"{{signature}}\",\n \"nonce\": \"{{nonce}}\"\n}" + }, + "url": { + "raw": "{{baseUrl}}/directCommand", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "directCommand" + ] + }, + "description": "Download an asset file (streams the response)." + } + }, + { + "name": "encrypt", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"command\": \"encrypt\",\n \"blob\": \"hello\",\n \"consumerAddress\": \"{{consumerAddress}}\",\n \"signature\": \"{{signature}}\",\n \"nonce\": \"{{nonce}}\",\n \"encoding\": \"string\",\n \"encryptionType\": \"ECIES\"\n}" + }, + "url": { + "raw": "{{baseUrl}}/directCommand", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "directCommand" + ] + }, + "description": "Encrypt a blob (AES or ECIES)." + } + }, + { + "name": "decryptDDO", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"command\": \"decryptDDO\",\n \"decrypterAddress\": \"{{consumerAddress}}\",\n \"chainId\": {{chainId}},\n \"nonce\": \"{{nonce}}\",\n \"signature\": \"{{signature}}\",\n \"transactionId\": \"\",\n \"dataNftAddress\": \"\"\n}" + }, + "url": { + "raw": "{{baseUrl}}/directCommand", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "directCommand" + ] + }, + "description": "Decrypt a DDO document." + } + }, + { + "name": "findDDO", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"command\": \"findDDO\",\n \"id\": \"{{did}}\"\n}" + }, + "url": { + "raw": "{{baseUrl}}/directCommand", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "directCommand" + ] + }, + "description": "Find which peers can serve a DDO." + } + }, + { + "name": "validateDDO", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"command\": \"validateDDO\",\n \"ddo\": {}\n}" + }, + "url": { + "raw": "{{baseUrl}}/directCommand", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "directCommand" + ] + }, + "description": "Validate a DDO." + } + }, + { + "name": "getComputeEnvironments", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"command\": \"getComputeEnvironments\",\n \"chainId\": {{chainId}}\n}" + }, + "url": { + "raw": "{{baseUrl}}/directCommand", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "directCommand" + ] + }, + "description": "List compute environments over the command interface." + } + }, + { + "name": "getEscrowEvents", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"command\": \"getEscrowEvents\",\n \"chainId\": {{chainId}}\n}" + }, + "url": { + "raw": "{{baseUrl}}/directCommand", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "directCommand" + ] + }, + "description": "Query escrow events over the command interface." + } + } + ] + } + ] +} \ No newline at end of file diff --git a/docs/compute-pricing.md b/docs/compute-pricing.md deleted file mode 100644 index c6598e760..000000000 --- a/docs/compute-pricing.md +++ /dev/null @@ -1,279 +0,0 @@ -# Compute Environment Configuration and Pricing - -This guide explains how to configure your node’s Docker compute environments and how to set prices for each resource. It covers the `DOCKER_COMPUTE_ENVIRONMENTS` variable (or equivalent config), the fee structure, pricing units, and examples for CPU, RAM, disk, and GPU. - -## Overview - -- **Configuration**: Define compute environments via the `DOCKER_COMPUTE_ENVIRONMENTS` environment variable (JSON) or via `config.json` under `dockerComputeEnvironments`. -- **Environment**: Is a group of resources, payment and accesslists. -- **Resources**: Each environment declares resources (e.g. `cpu`, `ram`, `disk`, and optionally GPUs). You must declare a `disk` resource. -- **Pricing**: For each chain and fee token, you set a `price` per resource. Cost is computed as **price × amount × duration (in minutes, rounded up)**. -- **Free**: Environments which does not require a payment for the resources, but most likley are very limited in terms of resources available and job duration. -- **Image building**: **Free jobs cannot build images** (Dockerfiles are not allowed). For **paid jobs**, **image build time counts toward billable duration** and also consumes the job’s `maxJobDuration`. - -## Pricing Units - -| Resource | Unit of `amount` | Price meaning | Cost formula | -| -------- | ---------------- | ------------------ | ------------------------------------ | -| **CPU** | Number of CPUs | Per CPU per minute | `price × cpus × ceil(duration/60)` | -| **RAM** | Gigabytes (GB) | Per GB per minute | `price × ramGB × ceil(duration/60)` | -| **Disk** | Gigabytes (GB) | Per GB per minute | `price × diskGB × ceil(duration/60)` | -| **GPU** | Number of GPUs | Per GPU per minute | `price × gpus × ceil(duration/60)` | - -So: - -- **CPU and GPU**: price is **per resource per minute** (e.g. 2 CPUs at price 1 for 90 minutes → 2 × 1 × 90 = 180). -- **Memory (RAM) and storage (disk)**: price is **per minute per gigabyte** (e.g. 4 GB RAM at price 0.5 for 60 minutes → 0.5 × 4 × 60 = 120). - -Duration is always in seconds; it is converted to minutes with **ceil(duration / 60)** (e.g. 61 seconds → 2 minutes). - ---- - -## Where to Configure - -1. **Environment variable** - Set `DOCKER_COMPUTE_ENVIRONMENTS` to a JSON string (array of compute environment objects). - Example: - `export DOCKER_COMPUTE_ENVIRONMENTS='[{"socketPath":"/var/run/docker.sock",...}]'` - -2. **Config file** - Put the same array in your JSON config under the key `dockerComputeEnvironments`, and point the node to that file (e.g. via `CONFIG_PATH`). - -If both are set, the environment variable typically overrides the config. See [Environmental Variables](env.md) and `ENVIRONMENT_VARIABLES` in `src/utils/constants.ts`. - ---- - -## Environment Structure (Summary) - -Each element of `DOCKER_COMPUTE_ENVIRONMENTS` is an object with at least: - -- **socketPath**: Docker socket (e.g. `"/var/run/docker.sock"`). -- **resources**: List of resources (see below). Must include `disk`. -- **storageExpiry**, **maxJobDuration**, **minJobDuration**: Required (seconds). -- **fees**: Per-chain, per-token pricing (see next section). -- **access** (optional): Who can run paid jobs (`addresses`, `accessLists`). -- **free** (optional): Limits and access for free jobs. - -### Resources - -- **cpu**, **ram**, **disk**: Standard resources. `disk` is mandatory. - **Disk** and **RAM** are in **GB** (e.g. `"total": 10` = 10 GB). -- **GPU**: Add a resource with `"type": "gpu"` and either `deviceRequests` (NVIDIA) or `advanced` (AMD/Intel). See [GPU.md](GPU.md) for full examples. - ---- - -## Fee Structure - -`fees` is an object keyed by **chain ID** (string). Each value is an array of fee options: - -```json -"fees": { - "1": [ - { - "feeToken": "0x...", - "prices": [ - { "id": "cpu", "price": 1 }, - { "id": "ram", "price": 0.5 }, - { "id": "disk", "price": 0.2 }, - { "id": "myGPU", "price": 3 } - ] - } - ] -} -``` - -- **feeToken**: Token contract address used for payment on that chain. -- **prices**: List of `{ "id": "", "price": }`. - Only resources listed here are billable; omit a resource to offer it without charge (e.g. for free tier only). - -**Important**: The `id` in `prices` must match the resource `id` in `resources` and in `free.resources` (e.g. if the GPU resource is `"id": "myGPU"`, use `"id": "myGPU"` in `prices`, not `"nyGPU"`). - ---- - -## Cost Examples - -Assume: - -- **CPU** price = 1 (per CPU per minute) -- **RAM** price = 0.5 (per GB per minute) -- **Disk** price = 0.2 (per GB per minute) -- **GPU** price = 3 (per GPU per minute) - -Job: 2 CPUs, 4 GB RAM, 10 GB disk, 1 GPU, duration **125 seconds** (ceil = 3 minutes). - -- CPU: 1 × 2 × 3 = **6** -- RAM: 0.5 × 4 × 3 = **6** -- Disk: 0.2 × 10 × 3 = **6** -- GPU: 3 × 1 × 3 = **9** - **Total cost = 27** (in the smallest unit of the fee token). - ---- - -## Example 1: CPU, RAM, and Disk with Prices - -```json -[ - { - "socketPath": "/var/run/docker.sock", - "resources": [{ "id": "disk", "total": 10 }], - "storageExpiry": 604800, - "maxJobDuration": 3600, - "minJobDuration": 60, - "fees": { - "1": [ - { - "feeToken": "0x967da4048cD07aB37855c090aAF366e4ce1b9F48", - "prices": [ - { "id": "cpu", "price": 1 }, - { "id": "ram", "price": 0.5 }, - { "id": "disk", "price": 0.2 } - ] - } - ] - }, - "free": { - "maxJobDuration": 60, - "minJobDuration": 10, - "maxJobs": 3, - "resources": [ - { "id": "cpu", "max": 1 }, - { "id": "ram", "max": 1 }, - { "id": "disk", "max": 1 } - ] - } - } -] -``` - -- **CPU**: 1 unit per CPU per minute. -- **RAM**: 0.5 units per GB per minute. -- **Disk**: 0.2 units per GB per minute. - ---- - -## Example 2: CPU + NVIDIA GPU - -Get the GPU UUID: - -```bash -nvidia-smi --query-gpu=name,uuid --format=csv -``` - -Then define one GPU and set a price per GPU per minute (e.g. 3): - -```json -[ - { - "socketPath": "/var/run/docker.sock", - "resources": [ - { - "id": "myGPU", - "description": "NVIDIA GeForce GTX 1060 3GB", - "type": "gpu", - "total": 1, - "init": { - "deviceRequests": { - "Driver": "nvidia", - "DeviceIDs": ["GPU-294c6802-bb2f-fedb-f9e0-a26b9142dd81"], - "Capabilities": [["gpu"]] - } - } - }, - { "id": "disk", "total": 1 } - ], - "storageExpiry": 604800, - "maxJobDuration": 3600, - "minJobDuration": 60, - "fees": { - "1": [ - { - "feeToken": "0x123", - "prices": [ - { "id": "cpu", "price": 1 }, - { "id": "myGPU", "price": 3 } - ] - } - ] - }, - "free": { - "maxJobDuration": 60, - "minJobDuration": 10, - "maxJobs": 3, - "resources": [ - { "id": "cpu", "max": 1 }, - { "id": "ram", "max": 1 }, - { "id": "disk", "max": 1 }, - { "id": "myGPU", "max": 1 } - ] - } - } -] -``` - -Ensure the fee `id` matches the resource `id` (`myGPU`). Price 3 = 3 units per GPU per minute. - ---- - -## Example 3: Multiple Chains and Tokens - -You can support several chains and multiple fee tokens per chain: - -```json -"fees": { - "1": [ - { - "feeToken": "0xTokenA", - "prices": [ - { "id": "cpu", "price": 1 }, - { "id": "ram", "price": 0.5 }, - { "id": "disk", "price": 0.2 } - ] - }, - { - "feeToken": "0xTokenB", - "prices": [ - { "id": "cpu", "price": 2 }, - { "id": "ram", "price": 1 }, - { "id": "disk", "price": 0.5 } - ] - } - ], - "137": [ - { - "feeToken": "0xPolygonToken", - "prices": [ - { "id": "cpu", "price": 1 }, - { "id": "ram", "price": 0.5 }, - { "id": "disk", "price": 0.2 } - ] - } - ] -} -``` - -Consumers choose chain and token when starting a job; the node uses the matching `prices` for that chain and token. - ---- - -## Example 4: AMD or Intel GPU - -For **AMD (e.g. ROCm on WSL2)** or **Intel Arc**, use a GPU resource with `init.advanced` instead of `deviceRequests`. Still set a **per-GPU per-minute** price in `fees.prices` (same formula: price × gpus × ceil(duration/60)). - -- **AMD Radeon (WSL2/ROCm)**: See [GPU.md – AMD Radeon 9070 XT](GPU.md#amd-radeon-9070-xt-on-wsl2) for `advanced` (Devices, Binds, CapAdd, etc.). Use a consistent `id` (e.g. `myGPU`) in `resources` and in `fees.prices`. -- **Intel Arc**: See [GPU.md – Intel Arc GPU](GPU.md#intel-arc-gpu-example) for `advanced` (Devices, GroupAdd, CapAdd). Again, use the same `id` in `fees.prices` (e.g. `intelGPU`). - -In all cases, the pricing rule is: **price × amount × ceil(duration/60)** with amount = number of GPUs. - ---- - -## Checklist - -- [ ] `DOCKER_COMPUTE_ENVIRONMENTS` (or `dockerComputeEnvironments` in config) is a JSON array. -- [ ] Every environment has a **disk** resource (and optionally cpu, ram, GPU). -- [ ] **Disk** and **RAM** amounts are in **GB**. -- [ ] **fees** has an entry per chain ID; each entry has `feeToken` and `prices` with `id` matching resource ids. -- [ ] **CPU / GPU**: price = per resource per minute. -- [ ] **RAM / Disk**: price = per GB per minute. -- [ ] For free tier, list the same resource ids in `free.resources`; omit from `prices` if they should be free only. - -For GPU setup details (NVIDIA, AMD, Intel), see [GPU.md](GPU.md). For other env vars and config options, see [env.md](env.md). diff --git a/docs/compute.md b/docs/compute.md new file mode 100644 index 000000000..41616f52c --- /dev/null +++ b/docs/compute.md @@ -0,0 +1,932 @@ +# Compute (Compute-to-Data) Configuration + +The single reference for configuring your node's Docker compute environments — from +node-wide resource declaration, through GPU setup, down to per-environment constraints and +pricing. It covers the `DOCKER_COMPUTE_ENVIRONMENTS` variable (or `dockerComputeEnvironments` +config key), the two-level resource model, GPU vendors (NVIDIA/AMD/Intel), resource +constraints, availability gating, and the fee structure. + +> On-demand **services** run on the same compute environments and draw from the same +> resource pool described here — see [services.md](services.md). + +## Contents + +1. [Overview](#overview) +2. [Where to configure](#where-to-configure) +3. [Configuration layout](#configuration-layout) +4. [Node-wide (connection-level) resources](#node-wide-connection-level-resources) +5. [Configuring GPUs](#configuring-gpus) +6. [Environment-level resource refs](#environment-level-resource-refs) +7. [Resource constraints](#resource-constraints) +8. [Availability & tracking](#availability--tracking) +9. [Pricing](#pricing) +10. [Free tier](#free-tier) +11. [Verify](#verify) +12. [Checklist](#checklist) +13. [Full example](#full-example) + +--- + +## Overview + +- **Configuration**: Define compute environments via the `DOCKER_COMPUTE_ENVIRONMENTS` + environment variable (JSON) or via `config.json` under `dockerComputeEnvironments`. +- **Two-level layout**: Resources are defined once at the **Docker-connection level** + (`socketPath`) and **referenced** by each environment. This lets multiple environments + share the same hardware (e.g. both a paid and a free environment can use the same GPU). +- **Auto-detection**: `cpu` and `ram` are automatically detected from the host at startup. + `disk` is measured via `statfs`. You only need to declare them in `resources` if you want + to cap/override the detected value. +- **Resources**: The connection-level `resources` array holds full hardware definitions. + Each environment's `resources` array holds lightweight refs + (`{ id, total?, min?, max?, constraints? }`) pointing to those pool entries. +- **Dual-gate tracking** (fungible resources like CPU/RAM/disk): Gate 1 enforces the + per-environment ceiling; Gate 2 enforces the engine-wide physical pool ceiling. Both must + pass for a job to be admitted. +- **Pricing**: For each chain and fee token, you set a `price` per resource. Cost is + **price × amount × duration (in minutes, rounded up)**. +- **Free tier**: Environments can have a `free` block that permits jobs with no payment, but + with tighter resource limits. +- **Image building**: Free jobs cannot build images (Dockerfiles are not allowed). For paid + jobs, image build time counts toward billable duration. + +--- + +## Where to configure + +1. **Environment variable** + Set `DOCKER_COMPUTE_ENVIRONMENTS` to a JSON string (array of Docker-connection objects): + `export DOCKER_COMPUTE_ENVIRONMENTS='[{"socketPath":"/var/run/docker.sock",...}]'` + +2. **Config file** + Put the same array in your JSON config under the key `dockerComputeEnvironments`, and + point the node to that file (e.g. via `CONFIG_PATH`). + +If both are set, the environment variable overrides the config. See [env.md](env.md) for all +available fields. + +--- + +## Configuration layout + +``` +DOCKER_COMPUTE_ENVIRONMENTS +└── [ Docker connection ] ← socketPath, resources[], environments[] + ├── resources[] ← full hardware definitions (CPU, RAM, disk, GPU, …) + │ ├── { id: "cpu", total: 6 } (optional: caps auto-detected value) + │ ├── { id: "ram", total: 28 } (optional: caps auto-detected value) + │ ├── { id: "disk", total: 80 } (optional: caps auto-detected value) + │ └── { id: "gpu0", kind: "discrete", … } (required for custom hardware) + └── environments[] ← one or more compute environments + └── { id, fees, resources[], free? } + └── resources[] ← lightweight refs to the pool above + ├── { id: "cpu", total: 4, min: 1, max: 4, constraints: [...] } + ├── { id: "ram", total: 16, min: 1, max: 8 } + ├── { id: "disk", max: 20 } + └── { id: "gpu0" } +``` + +`cpu`, `ram`, and `disk` are **auto-detected** at startup — you do not need to declare them +in the connection-level `resources` array. Include them only to cap the detected value (e.g. +limit an 8-core host to 6 cores for compute). Custom hardware (GPUs, NICs) must always be +declared. + +--- + +## Node-wide (connection-level) resources + +These fields go in the `resources` array at the Docker-connection level: + +| Field | Description | +|---|---| +| `id` | Unique identifier used in env refs and `fees.prices[].id` (e.g. `"cpu"`, `"gpu0"`) | +| `total` | Maximum units available in the pool. For `cpu`/`ram`/`disk`, caps the auto-detected value. | +| `kind` | `"fungible"` (CPU, RAM, disk — interchangeable units) or `"discrete"` (GPU, FPGA — named device). Auto-inferred if omitted: `"discrete"` when `init` is present, `"fungible"` otherwise. | +| `shareable` | `discrete` only. `true` → multiple jobs may use the device simultaneously (e.g. NIC, TPM). `false` (default) → exclusive per job (GPU, FPGA). | +| `min` | Minimum units per job request | +| `max` | Maximum units per job request (defaults to `total`) | +| `type` | Hint string used for grouping and display: `"cpu"`, `"ram"`, `"disk"`, `"gpu"`, `"fpga"`, … Group constraints (see below) match on this. | +| `description` | Human-readable label shown in `getComputeEnvironments` | +| `driverVersion` | GPU driver version string | +| `memoryTotal` | GPU VRAM string (e.g. `"40960 MiB"`) | +| `platform` | GPU vendor: `"nvidia"`, `"amd"`, `"intel"` | +| `init` | Docker container configuration (`deviceRequests` for NVIDIA, `advanced` for AMD/Intel). Makes `kind` default to `"discrete"`. | +| `constraints` | Companion resource requirements — see [Resource constraints](#resource-constraints). | + +### CPU pinning with `cpuList` + +For the `cpu` resource only, you may pin jobs to specific host cores with `cpuList` instead +of `total` (the two are mutually exclusive). It is a cpuset string of ascending, +non-overlapping core IDs and/or ranges: `"3"`, `"0-1,3"`, `"0-15,32-47"` (no spaces, floats, +or negatives). The effective `total` becomes the number of expanded cores, and every id is +validated against the host's CPU count at startup. + +--- + +## Configuring GPUs + +Supporting GPUs comes down to: + +- define each GPU as a **named resource at the connection level** (same level as `socketPath`), +- pass Docker device args inside each GPU's `init` block, +- reference the GPU by `id` in the environment's `resources` list, +- set a price for each GPU in the environment's `fees` (see [Pricing](#pricing)). + +### Key rules + +- Each physical GPU is its own resource with a unique `id`. For NVIDIA (`init.deviceRequests`), + give each resource exactly **one** `DeviceID`; AMD/Intel instead map the device through + `init.advanced.Devices` (e.g. `/dev/dri/card0`, `/dev/dxg`, `/dev/dri/renderD128`). +- `kind: "discrete"` (non-fungible): only one job at a time can use the device. This is the + default when `init` is present. +- `cpu`, `ram`, and `disk` are **auto-detected** — you do not need to declare them unless you + want to cap their totals. +- Environment `resources` are **lightweight refs** (`id` + optional + `total`/`min`/`max`/`constraints`). Hardware details (`init`, `driverVersion`, `platform`, + etc.) live only at connection level. + +> **Security note**: `init.advanced` entries (`Binds`, `CapAdd`, `Devices`, `SecurityOpt`) +> apply to every job in every environment that references the resource. Review them carefully +> before adding to production configs. + +### NVIDIA GPU + +Install the [NVIDIA CUDA drivers](https://docs.nvidia.com/cuda/cuda-installation-guide-linux/) +and [NVIDIA container toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html). + +Get the GPU UUID: + +```bash +nvidia-smi --query-gpu=name,uuid,driver_version,memory.total --format=csv +# NVIDIA GeForce GTX 1060 3GB, GPU-294c6802-bb2f-fedb-f9e0-a26b9142dd81, 570.195.03, 3072 MiB +``` + +Define the GPU at the **connection level** with `kind: "discrete"` and a single `DeviceID`. +The environment references it by `id`. + +```json +[ + { + "socketPath": "/var/run/docker.sock", + "resources": [ + { + "id": "gpu0", + "kind": "discrete", + "type": "gpu", + "total": 1, + "description": "NVIDIA GeForce GTX 1060 3GB", + "platform": "nvidia", + "driverVersion": "570.195.03", + "memoryTotal": "3072 MiB", + "init": { + "deviceRequests": { + "Driver": "nvidia", + "DeviceIDs": ["GPU-294c6802-bb2f-fedb-f9e0-a26b9142dd81"], + "Capabilities": [["gpu"]] + } + }, + "constraints": [{ "id": "ram", "min": 2 }, { "id": "cpu", "min": 1 }] + } + ], + "environments": [ + { + "id": "gpu-env", + "description": "NVIDIA GPU environment", + "storageExpiry": 604800, + "maxJobDuration": 3600, + "minJobDuration": 60, + "resources": [ + { "id": "cpu", "min": 1, "max": 4 }, + { "id": "ram", "min": 1, "max": 8 }, + { "id": "disk", "min": 1, "max": 50 }, + { "id": "gpu0" } + ], + "fees": { + "1": [ + { + "feeToken": "0x967da4048cD07aB37855c090aAF366e4ce1b9F48", + "prices": [{ "id": "cpu", "price": 1 }, { "id": "gpu0", "price": 3 }] + } + ] + } + } + ] + } +] +``` + +### AMD Radeon (ROCm) + +Install [ROCm](https://rocm.docs.amd.com/projects/radeon/en/latest/docs/install/wsl/install-radeon.html), +then define the GPU with `init.advanced` instead of `deviceRequests`: + +```json +{ + "id": "gpu0", + "kind": "discrete", + "type": "gpu", + "total": 1, + "description": "AMD Radeon RX 9070 XT", + "driverVersion": "26.2.2", + "memoryTotal": "16384 MiB", + "init": { + "advanced": { + "IpcMode": "host", + "ShmSize": 8589934592, + "CapAdd": ["SYS_PTRACE"], + "Devices": ["/dev/dxg", "/dev/dri/card0"], + "Binds": [ + "/usr/lib/wsl/lib/libdxcore.so:/usr/lib/libdxcore.so", + "/opt/rocm/lib/libhsa-runtime64.so.1:/opt/rocm/lib/libhsa-runtime64.so.1" + ], + "SecurityOpt": { "seccomp": "unconfined" } + } + }, + "constraints": [{ "id": "ram", "min": 4 }, { "id": "cpu", "min": 2 }] +} +``` + +### Intel Arc + +Install the [Intel GPU drivers](https://dgpu-docs.intel.com/driver/installation.html), then: + +```json +{ + "id": "gpu0", + "kind": "discrete", + "type": "gpu", + "total": 1, + "description": "Intel Arc A770M Graphics", + "driverVersion": "32.0.101.8531", + "memoryTotal": "16384 MiB", + "init": { + "advanced": { + "Devices": ["/dev/dri/renderD128", "/dev/dri/card0"], + "GroupAdd": ["video", "render"], + "CapAdd": ["SYS_ADMIN"] + } + }, + "constraints": [{ "id": "ram", "min": 2 }, { "id": "cpu", "min": 1 }] +} +``` + +### Multiple GPUs shared between environments + +Each physical GPU is its own resource. Multiple environments can reference the same GPUs; the +engine tracks discrete usage **globally**, so no GPU is ever double-allocated across +environments. + +```json +"resources": [ + { "id": "gpu0", "kind": "discrete", "type": "gpu", "total": 1, + "init": { "deviceRequests": { "Driver": "nvidia", "DeviceIDs": ["GPU-uuid-a"], "Capabilities": [["gpu"]] } } }, + { "id": "gpu1", "kind": "discrete", "type": "gpu", "total": 1, + "init": { "deviceRequests": { "Driver": "nvidia", "DeviceIDs": ["GPU-uuid-b"], "Capabilities": [["gpu"]] } } } +], +"environments": [ + { "id": "premium", "resources": [ { "id": "cpu" }, { "id": "ram" }, { "id": "gpu0" }, { "id": "gpu1" } ], "fees": { ... } }, + { "id": "standard", "resources": [ { "id": "cpu" }, { "id": "ram" } ], "fees": { ... } } +] +``` + +### Shareable devices (NIC, TPM, HSM) + +Use `kind: "discrete"` and `shareable: true` for devices that multiple jobs may use +simultaneously. The engine tracks `inUse` for visibility but never blocks allocation on them. + +```json +{ + "id": "nic0", + "kind": "discrete", + "shareable": true, + "type": "network", + "total": 1, + "description": "SR-IOV NIC", + "init": { "advanced": { "Devices": [{ "PathOnHost": "/dev/net/tun", "PathInContainer": "/dev/net/tun" }] } } +} +``` + +> `shareable: true` is **not** allowed on `type: "gpu"` or `type: "fpga"` — the node refuses +> to start. GPUs and FPGAs require exclusive per-job access. + +### Migration from the old format + +The old format placed hardware resources (`init`, `driverVersion`, etc.) inside +environments. This is now a startup error. + +```json +// Old (rejected): +"environments": [{ "resources": [{ "id": "myGPU", "total": 1, "init": {...} }] }] + +// New: +"resources": [{ "id": "myGPU", "kind": "discrete", "total": 1, "init": {...} }], +"environments": [{ "resources": [{ "id": "myGPU" }] }] +``` + +Move all `init`, `description`, `driverVersion`, `platform`, `memoryTotal`, `type`, `kind`, +and `constraints` fields to the connection-level `resources` array. Each environment's +`resources` keeps only `id` and optionally `total`/`min`/`max`/`constraints`. + +--- + +## Environment-level resource refs + +These fields go in each environment's `resources` array (lightweight refs to the pool): + +| Field | Description | +|---|---| +| `id` | Must match a connection-level resource `id`. `cpu`, `ram`, `disk` are always valid (auto-detected) and are injected as baseline refs even if omitted. | +| `total` | Environment aggregate ceiling: maximum units all jobs in this environment can use simultaneously. Omit to default to the pool total. (Fungible resources only.) | +| `min` | Per-job minimum override for this environment | +| `max` | Per-job maximum override (capped to `total` if both are set) | +| `constraints` | Per-env override — see below. Replaces the pool resource's constraints entirely for this environment. Omit to inherit pool constraints. Set `[]` to remove all constraints for this env. | + +--- + +## Resource constraints + +A **constraint** ties one resource's requested amount to another's. Constraints are declared +on a "parent" resource (via its `constraints` array) and are evaluated only when that parent +is actually requested (`amount > 0`). There are two forms. + +### Constraint fields + +| Field | Description | +|---|---| +| `id` | Target a **single** resource by exact id (e.g. `"ram"`, `"gpu0"`). | +| `type` | Target a **group** of resources by `type` (e.g. `"gpu"`), aggregated across all resources of that type the environment exposes. Mutually exclusive with `id` — set exactly one. | +| `min` | Minimum units of the target. | +| `max` | Maximum units of the target. | +| `perUnit` | `true`/omitted → **ratio**: the bound is `parentAmount × value`. `false` → **floor/ceiling**: the bound is the absolute `value`, regardless of how much of the parent was requested. | +| `aggregate` | Single-`id` targets only. When `true`, this constraint's per-parent contribution is **summed** with matching aggregate constraints on other requested resources into one shared target — so per-device GPUs can jointly scale a companion resource (see below). Rejected on a `type` group. | + +Behavior: +- **min** — if the target is below the required minimum it is **auto-bumped** up to it (for a + group, the deficit is distributed across the group's members, preferring the ones with the + most availability). If the required minimum exceeds the target's aggregate max, the request + is rejected with `Cannot satisfy constraint …`. +- **max** — if the target exceeds the allowed maximum the request is rejected with + `Too much …` (never auto-reduced). + +### Companion constraints (single `id`, ratio) + +The classic use: a GPU that needs a minimum amount of companion RAM/CPU per unit. This also +prevents the GPU from being scheduled when fungible resources are exhausted. + +```json +{ + "id": "gpu0", + "kind": "discrete", + "constraints": [ + { "id": "ram", "min": 8 }, + { "id": "cpu", "min": 2 }, + { "id": "disk", "min": 10 } + ] +} +``` + +With `perUnit` defaulting to `true`, requesting 1 `gpu0` requires ≥ 8 GB RAM; the amounts +scale with the parent (`requiredMin = gpuAmount × min`). + +### `min` + `max` ratios: counted vs per-device resources + +Both `min` and `max` are supported, and with the default (ratio) semantics **both** scale by +the parent amount: `requiredMin = parentAmount × min`, `requiredMax = parentAmount × max`. +`min` **auto-bumps** the target up; `max` is **never auto-reduced** — exceeding it throws. + +Whether the ratio *aggregates across multiple GPUs* depends on how the GPUs are modeled, +because a constraint's **parent is always a single resource entry** — the engine evaluates +each resource's constraints on its own. (The `type` grouping described below applies only to +the constraint's *target*, not its parent, so there is no "sum across all GPUs" as the +parent.) + +Take a GPU constraint of `{ "id": "cpu", "min": 1, "max": 4 }` — "1–4 CPUs per GPU": + +**Counted model** — one `gpu` resource with `total: 2`. The parent amount *is* the number of +GPUs requested, so the ratio aggregates: + +| Request | Result | +|---|---| +| `gpu: 2` | `cpu` auto-bumped to **2** (= 2 × 1) | +| `gpu: 2, cpu: 8` | allowed (max = 2 × 4 = **8**) | +| `gpu: 2, cpu: 10` | rejected — `Too much cpu for 2 gpu. Max allowed: 8, requested: 10` | + +**Per-device model** — separate `GPU1` / `GPU2` resources (each `max: 1`), each carrying the +same constraint. Each is evaluated independently; the parent amount is only ever 0 or 1, so +the bounds **do not sum**: + +| Request | Result | +|---|---| +| `GPU1: 1, GPU2: 1` | `cpu` = **1**, *not* 2 — each GPU only requires ≥ 1, they don't add up | +| `GPU1: 1, GPU2: 1, cpu: 5` | rejected — `Too much cpu for 1 GPU1. Max allowed: 4` (max is **4** per GPU, not 8) | + +So the "N GPUs ⇒ N×min / N×max" behavior only holds for a single **counted** GPU resource. +The trade-off: a counted resource with explicit `DeviceIDs` attaches *all* of them to the +container regardless of the requested amount, so it can't cleanly pin "1 of 2 physical GPUs" +— use it when you don't need per-device selection. The recommended one-resource-per-GPU model +gives clean per-device allocation and global tracking, but its per-unit constraints are +per-GPU, not aggregated. + +#### `aggregate`: summing across per-device GPUs + +To get per-device allocation **and** the summed behavior, mark the constraint `aggregate: true` +on each GPU. Matching aggregate constraints (same target `id`) accumulate their per-parent +contribution across every requested GPU into one shared requirement: + +```json +{ "id": "GPU1", "constraints": [{ "id": "cpu", "min": 1, "max": 4, "aggregate": true }] }, +{ "id": "GPU2", "constraints": [{ "id": "cpu", "min": 1, "max": 4, "aggregate": true }] } +``` + +| Request | Result | +|---|---| +| `GPU1: 1` | `cpu` in **[1, 4]** (only one GPU contributes) | +| `GPU1: 1, GPU2: 1` | `cpu` auto-bumped to **2** (= 1+1); max is **8** (= 4+4) | +| `GPU1: 1, GPU2: 1, cpu: 10` | rejected — `Too much cpu for the requested resources. Max allowed: 8, requested: 10` | + +Notes: +- `aggregate` targets a **single `id`** (it sums *into* one resource); it cannot target a + `type` group — the config is rejected at startup if you try. +- The summed **min** auto-bumps the target up (capped by the target's own `max`); the summed + **max** is enforced but never auto-reduces. +- Non-aggregate constraints on the same target keep their independent per-parent behavior; the + aggregate pass runs after them and only raises the floor further. Avoid mixing aggregate and + non-aggregate constraints on the same target to keep behavior obvious. + +### Group constraints (`type`) + floor (`perUnit: false`) + +To express **"if any CPU is selected, the job needs at least 1 GPU — no matter which id"**, +target the `gpu` **group** with an absolute **floor**: + +```json +"environments": [ + { + "resources": [ + { "id": "cpu", "min": 1, "max": 16, + "constraints": [ { "type": "gpu", "min": 1, "perUnit": false } ] }, + { "id": "ram" }, + { "id": "disk" }, + { "id": "GPU1" }, + { "id": "GPU2" } + ] + } +] +``` + +- `type: "gpu"` aggregates across every `type:"gpu"` resource **the environment exposes** + (here `GPU1` + `GPU2`). +- `perUnit: false` makes `min: 1` an **absolute floor** — one GPU total is enough whether the + job asks for 1 CPU or 16 (a ratio would instead demand one GPU *per* CPU). +- If the consumer selects a CPU but requests no GPU, the engine auto-assigns one — picking the + GPU with the most availability. Because each physical GPU is its own single-`DeviceID` + resource, exactly that GPU is attached to the container. + +> **Place group constraints on the environment's ref, not on the connection-level pool.** +> A pool-level constraint is inherited by *every* environment; in a GPU-less environment the +> `type:"gpu"` group would be empty (aggregate max 0), so the floor of 1 could never be met +> and every CPU job there would be rejected with `Cannot satisfy`. + +### Per-env override + +Environments can override pool-level constraints via the `EnvironmentResourceRef`: + +```json +{ "id": "gpu0", "constraints": [{ "id": "ram", "min": 16 }, { "id": "cpu", "min": 4 }] } +``` + +Omit `constraints` to inherit the pool's; set `"constraints": []` to remove all constraints +for that environment. + +### More constraint recipes + +Common patterns, with the behavior verified against the constraint engine. Each snippet shows +just the relevant resource entry; place group/floor constraints on the environment's ref (see +the note above). + +**1. RAM range per GPU** — companion `min` + `max` (ratio): + +```json +{ "id": "gpu0", "constraints": [{ "id": "ram", "min": 8, "max": 16 }] } +``` + +Renting `gpu0` forces RAM into **[8, 16]** GB per GPU. RAM below 8 → bumped to 8; `ram: 20` → +rejected (`Too much ram for 1 gpu0. Max allowed: 16, requested: 20`). + +**2. One GPU per CPU (strict 1:1)** — group + ratio (`perUnit` omitted), on the env `cpu` ref: + +```json +{ "id": "cpu", "constraints": [{ "type": "gpu", "min": 1 }] } +``` + +`cpu: 2` → the engine auto-assigns **2 GPUs**; `cpu: 3` when only 2 GPUs are exposed → rejected +(`Cannot satisfy constraint: 3 cpu requires at least 3 gpu resources, but max is 2`). Contrast +with `perUnit: false`, which needs just **one** GPU regardless of CPU count. + +**3. Cap accelerators per job** — group ceiling (`max` + `perUnit: false`), on the `cpu` ref: + +```json +{ "id": "cpu", "constraints": [{ "type": "gpu", "max": 2, "perUnit": false }] } +``` + +At most **2 GPUs** total per job regardless of CPUs; requesting 3 → rejected (`Too much gpu +resources for 1 cpu. Max allowed: 2, requested: 3`). + +**4. Total RAM scales with GPU count** — `aggregate` across per-device GPUs: + +```json +{ "id": "GPU1", "constraints": [{ "id": "ram", "min": 8, "aggregate": true }] }, +{ "id": "GPU2", "constraints": [{ "id": "ram", "min": 8, "aggregate": true }] } +``` + +2 GPUs → RAM auto-bumped to **16**; 1 GPU → RAM **8**. + +**5. Several companions at once** — multiple constraints on one GPU: + +```json +{ "id": "gpu0", "constraints": [ + { "id": "ram", "min": 8 }, + { "id": "cpu", "min": 2 }, + { "id": "disk", "min": 20 } +] } +``` + +Renting `gpu0` auto-bumps RAM → 8, CPU → 2, disk → 20 in a single pass. + +**6. Absolute RAM floor for a GPU job** — `perUnit: false` on a single `id`: + +```json +{ "id": "gpu0", "constraints": [{ "id": "ram", "min": 32, "perUnit": false }] } +``` + +Any job renting `gpu0` gets RAM ≥ **32** total, regardless of GPU count (a ratio would instead +scale RAM with the number of GPUs). + +--- + +## Availability & tracking + +Before a job is admitted, the engine checks availability per requested resource: + +- **Gate 1 (per-environment ceiling)** — *fungible only*: `env.total - env.inUse >= requested`. + Controlled by the ref's `total`. Prevents one environment from starving others. +- **Gate 2 (engine-wide pool ceiling)** — *fungible + exclusive discrete*: usage summed + across all environments must not exceed the pool's physical `total`. + +Tracking rules: +- **Fungible** resources (cpu/ram/disk) are tracked **per-environment**. +- **Discrete** resources (GPU/FPGA) are tracked **globally** — a GPU used in one environment + is unavailable in all others. +- **Shareable discrete** (`shareable:true`, e.g. NIC) are tracked for visibility but never + block allocation. + +Running on-demand **services** occupy the same pool and are counted in the same accounting. + +--- + +## Pricing + +`fees` is an object keyed by **chain ID** (string). Each value is an array of fee options: + +```json +"fees": { + "1": [ + { + "feeToken": "0x...", + "prices": [ + { "id": "cpu", "price": 1 }, + { "id": "ram", "price": 0.5 }, + { "id": "disk", "price": 0.2 }, + { "id": "gpu0", "price": 3 } + ] + } + ] +} +``` + +- **feeToken**: token contract address used for payment on that chain. +- **prices**: list of `{ "id": "", "price": }`. The `id` must match a + connection-level resource `id`. Only resources listed here are billable; omit a resource to + offer it at no charge. + +### Pricing units & cost formula + +| Resource | Unit of `amount` | Price meaning | Cost formula | +| -------- | ---------------- | ------------------ | ------------------------------------ | +| **CPU** | Number of CPUs | Per CPU per minute | `price × cpus × ceil(duration/60)` | +| **RAM** | Gigabytes (GB) | Per GB per minute | `price × ramGB × ceil(duration/60)` | +| **Disk** | Gigabytes (GB) | Per GB per minute | `price × diskGB × ceil(duration/60)` | +| **GPU** | Number of GPUs | Per GPU per minute | `price × gpus × ceil(duration/60)` | + +Duration is always in seconds, converted to minutes with **ceil(duration / 60)** (e.g. 61 +seconds → 2 minutes). + +**Example** — CPU=1, RAM=0.5, Disk=0.2, GPU=3; job of 2 CPUs, 4 GB RAM, 10 GB disk, 1 GPU for +125 s (ceil = 3 min): + +- CPU: 1 × 2 × 3 = **6** +- RAM: 0.5 × 4 × 3 = **6** +- Disk: 0.2 × 10 × 3 = **6** +- GPU: 3 × 1 × 3 = **9** → **Total = 27** (smallest unit of the fee token). + +### Multiple chains and tokens + +You can support several chains and multiple fee tokens per chain; consumers choose chain and +token when starting a job, and the node uses the matching `prices`. + +```json +"fees": { + "1": [ { "feeToken": "0xTokenA", "prices": [ { "id": "cpu", "price": 1 } ] }, + { "feeToken": "0xTokenB", "prices": [ { "id": "cpu", "price": 2 } ] } ], + "137": [ { "feeToken": "0xPolygonToken", "prices": [ { "id": "cpu", "price": 1 } ] } ] +} +``` + +--- + +## Free tier + +An environment may expose a `free` block permitting jobs with no payment but tighter limits. +List the same resource ids under `free.resources`; a resource not listed there is unavailable +to free jobs even if the paid environment offers it. Free jobs cannot build images. + +```json +"free": { + "maxJobDuration": 60, + "minJobDuration": 10, + "maxJobs": 3, + "resources": [ + { "id": "cpu", "max": 1 }, + { "id": "ram", "max": 2 }, + { "id": "disk", "max": 5 }, + { "id": "gpu0" } + ] +} +``` + +--- + +## Verify + +```bash +curl http://localhost:8000/api/services/computeEnvironments +``` + +The response includes each environment's `resources` fully resolved (including `init`, +`driverVersion`, etc.) and `inUse` counters. Start a free GPU job: + +```json +{ + "command": "freeStartCompute", + "algorithm": { + "meta": { + "container": { + "image": "tensorflow/tensorflow", + "tag": "2.17.0-gpu", + "entrypoint": "python $ALGO" + }, + "rawcode": "import tensorflow as tf\nprint('Num GPUs Available:', len(tf.config.list_physical_devices('GPU')))" + } + }, + "consumerAddress": "0x00", + "signature": "123", + "nonce": 1, + "environment": "", + "resources": [ + { "id": "cpu", "amount": 1 }, + { "id": "gpu0", "amount": 1 } + ] +} +``` + +--- + +## Checklist + +- [ ] `DOCKER_COMPUTE_ENVIRONMENTS` (or `dockerComputeEnvironments` in config) is a JSON array. +- [ ] GPUs and other custom hardware are defined in the **connection-level** `resources` + array with `kind: "discrete"` and a unique `id` each (NVIDIA: exactly one `DeviceID`; + AMD/Intel: the device mapped via `init.advanced.Devices`). +- [ ] Each environment's `resources` array contains lightweight refs + (`{ id, total?, min?, max?, constraints? }`). +- [ ] `cpu`, `ram`, `disk` are auto-detected — declare them only to cap/override the value. +- [ ] **Disk** and **RAM** amounts are in **GB**. +- [ ] Group constraints (`type`) live on the **environment ref**, not the connection-level pool. +- [ ] `fees.prices[].id` matches a connection-level resource `id`. +- [ ] **CPU / GPU** price = per resource per minute; **RAM / Disk** price = per GB per minute. +- [ ] For the free tier, list resource ids in `free.resources`; omit from `prices` to make + them free only. + +--- + +## Full example + +A single, valid `DOCKER_COMPUTE_ENVIRONMENTS` (or `dockerComputeEnvironments`) config that +exercises **everything** in this guide at once: + +- a **connection-level pool** that caps CPU/RAM/disk and declares **two NVIDIA GPUs**, each + with companion (`id`, ratio) constraints; +- a paid **`gpu-premium`** environment that guarantees a GPU for any CPU job (group + floor + constraint), prices resources on **two chains**, and offers a **free tier**; +- a CPU-only **`cpu-standard`** environment — cheaper, no GPU, with its own free tier. + +> This exact JSON is validated against the config schema in CI. Replace the GPU `DeviceIDs`, +> `feeToken` addresses, and totals with your own. + +```json +[ + { + "socketPath": "/var/run/docker.sock", + "resources": [ + { "id": "cpu", "type": "cpu", "total": 16 }, + { "id": "ram", "type": "ram", "total": 64 }, + { "id": "disk", "type": "disk", "total": 2000 }, + { + "id": "GPU1", + "kind": "discrete", + "type": "gpu", + "total": 1, + "description": "NVIDIA A100 40GB (slot 0)", + "platform": "nvidia", + "driverVersion": "570.195.03", + "memoryTotal": "40960 MiB", + "init": { + "deviceRequests": { + "Driver": "nvidia", + "DeviceIDs": ["GPU-uuid-1"], + "Capabilities": [["gpu"]] + } + }, + "constraints": [{ "id": "ram", "min": 8 }, { "id": "cpu", "min": 2 }] + }, + { + "id": "GPU2", + "kind": "discrete", + "type": "gpu", + "total": 1, + "description": "NVIDIA A100 40GB (slot 1)", + "platform": "nvidia", + "driverVersion": "570.195.03", + "memoryTotal": "40960 MiB", + "init": { + "deviceRequests": { + "Driver": "nvidia", + "DeviceIDs": ["GPU-uuid-2"], + "Capabilities": [["gpu"]] + } + }, + "constraints": [{ "id": "ram", "min": 8 }, { "id": "cpu", "min": 2 }] + } + ], + "environments": [ + { + "id": "gpu-premium", + "description": "Paid GPU environment — any CPU job is guaranteed a GPU", + "storageExpiry": 604800, + "maxJobDuration": 3600, + "minJobDuration": 60, + "maxJobs": 2, + "enableNetwork": true, + "resources": [ + { "id": "cpu", "min": 1, "max": 8, + "constraints": [{ "type": "gpu", "min": 1, "perUnit": false }] }, + { "id": "ram", "min": 1, "max": 32 }, + { "id": "disk", "min": 1, "max": 500 }, + { "id": "GPU1" }, + { "id": "GPU2" } + ], + "access": { "addresses": [], "accessLists": null }, + "fees": { + "1": [ + { + "feeToken": "0x967da4048cD07aB37855c090aAF366e4ce1b9F48", + "prices": [ + { "id": "cpu", "price": 1 }, + { "id": "ram", "price": 0.5 }, + { "id": "disk", "price": 0.1 }, + { "id": "GPU1", "price": 10 }, + { "id": "GPU2", "price": 10 } + ] + } + ], + "137": [ + { + "feeToken": "0x2791Bca1f2de4661ED88A30C99A7a9449Aa84174", + "prices": [ + { "id": "cpu", "price": 0.5 }, + { "id": "ram", "price": 0.2 }, + { "id": "GPU1", "price": 5 }, + { "id": "GPU2", "price": 5 } + ] + } + ] + }, + "free": { + "maxJobDuration": 300, + "minJobDuration": 10, + "maxJobs": 1, + "access": { "addresses": [], "accessLists": null }, + "resources": [ + { "id": "cpu", "max": 2, + "constraints": [{ "type": "gpu", "min": 1, "perUnit": false }] }, + { "id": "ram", "max": 8 }, + { "id": "disk", "max": 10 }, + { "id": "GPU1", "constraints": [{ "id": "ram", "min": 4 }, { "id": "cpu", "min": 1 }] } + ] + } + }, + { + "id": "cpu-standard", + "description": "CPU-only environment — cheaper, no GPU", + "storageExpiry": 604800, + "maxJobDuration": 1800, + "minJobDuration": 60, + "maxJobs": 10, + "enableNetwork": false, + "resources": [ + { "id": "cpu", "total": 8, "min": 1, "max": 4 }, + { "id": "ram", "total": 16, "min": 1, "max": 8 }, + { "id": "disk", "max": 100 } + ], + "access": { "addresses": [], "accessLists": null }, + "fees": { + "1": [ + { + "feeToken": "0x967da4048cD07aB37855c090aAF366e4ce1b9F48", + "prices": [ + { "id": "cpu", "price": 0.3 }, + { "id": "ram", "price": 0.1 }, + { "id": "disk", "price": 0.05 } + ] + } + ] + }, + "free": { + "maxJobDuration": 120, + "minJobDuration": 10, + "maxJobs": 3, + "access": { "addresses": [], "accessLists": null }, + "resources": [ + { "id": "cpu", "max": 1 }, + { "id": "ram", "max": 2 }, + { "id": "disk", "max": 5 } + ] + } + } + ] + } +] +``` + +### What this config does + +**Connection-level pool** — `cpu`/`ram`/`disk` are auto-detected; the declared `total`s only +*cap* them (16 cores, 64 GB RAM, 2 TB disk max for compute). `GPU1` and `GPU2` are two physical +NVIDIA cards, each its own `discrete` resource with a single `DeviceID` and a **companion +constraint**: renting a GPU requires ≥ 8 GB RAM and ≥ 2 CPUs (ratio — per GPU). + +**`gpu-premium` (paid + free)** +- **Guaranteed GPU**: the `cpu` ref carries `{ "type": "gpu", "min": 1, "perUnit": false }` — a + group + **floor** constraint. Any job that requests a CPU must end up with **at least one + GPU total** (`GPU1` *or* `GPU2`), regardless of how many CPUs. If the consumer didn't ask for + a GPU, the engine auto-assigns the freer one. +- **Constraint chaining & ordering**: because `cpu` is listed **before** the GPU refs, a single + resolution pass first satisfies the CPU→GPU floor, then applies the auto-assigned GPU's own + companion constraints (RAM/CPU). List the floor's parent (`cpu`) before the GPUs so this + chains in one pass. +- **Pricing on two chains**: Ethereum (`"1"`) and Polygon (`"137"`), each with its own + `feeToken` and per-resource prices. Consumers pick the chain/token at job start. +- **Free tier**: no payment, tighter caps (≤ 2 CPU, 8 GB RAM, 10 GB disk, 300 s), a single + concurrent free job, and access to `GPU1` only. Free-tier resource refs are **configured + independently** from the paid ones — note the floor constraint and `GPU1`'s companion + constraint are repeated here, because constraints for free jobs are read from + `free.resources`, not inherited from the paid refs. + +**`cpu-standard` (paid + free)** — CPU-only, cheaper prices, higher job concurrency +(`maxJobs: 10`), a modest free tier. It deliberately carries **no** GPU and **no** group +constraint — that's why the `type:"gpu"` floor lives on the environment ref and not the pool: a +pool-level floor would be inherited here and reject every CPU job (empty GPU group). + +### Worked scenarios (on `gpu-premium`, paid, chain `"1"`) + +The engine resolves the requested resources through the constraints before pricing. Resolved +allocations (verified against the constraint engine): + +| Consumer requests | Resolved allocation | Why | +|---|---|---| +| `cpu: 2` | `cpu=2, ram=8, disk=1, GPU1=1` | Floor auto-assigns `GPU1`; its companion bumps RAM to 8 (CPU already ≥ 2). | +| `cpu: 1, GPU2: 1` | `cpu=2, ram=8, disk=1, GPU2=1` | `GPU2` already satisfies the floor; its companion bumps RAM to 8 **and CPU to 2**. | +| `cpu: 1` while `GPU1` is busy | `cpu=2, ram=8, disk=1, GPU2=1` | Floor auto-assigns the freer GPU (`GPU2`); companion bumps RAM/CPU. | + +**Cost** for the first row run for 600 s (`ceil(600/60) = 10` min) at chain `"1"` prices +(cpu 1, ram 0.5, disk 0.1, GPU1 10): + +``` +cpu: 1 × 2 × 10 = 20 +ram: 0.5 × 8 × 10 = 40 +disk: 0.1 × 1 × 10 = 1 +GPU1: 10 × 1 × 10 = 100 + ---- + total = 161 (smallest unit of the fee token) +``` + +For all env vars and config options, see [env.md](env.md). diff --git a/docs/env.md b/docs/env.md index dd8761849..ab1995d92 100644 --- a/docs/env.md +++ b/docs/env.md @@ -129,114 +129,137 @@ Environmental variables are also tracked in `ENVIRONMENT_VARIABLES` within `src/ - `C2D_DOWNLOAD_TIMEOUT`: Timeout (in seconds) for pulling the algorithm docker image during a C2D job. If the pull exceeds this timeout, the job fails with `PullImageFailed` instead of getting stuck. Defaults to `900` (15 minutes). Example: `900` -The `DOCKER_COMPUTE_ENVIRONMENTS` environment variable is used to configure Docker-based compute environments in Ocean Node. This guide will walk you through the options available for defining `DOCKER_COMPUTE_ENVIRONMENTS` and how to set it up correctly. For configuring compute environments and setting prices for each resource (including pricing units and examples), see [Compute pricing](compute-pricing.md). +- `SERVICE_TEMPLATES_PATH`: Path to a folder of operator-published Service-on-Demand template files (`*.json`, validated against the template schema). The folder is re-read on every `serviceTemplates` request, so templates can be added, edited, or removed without restarting the node. Maps to the `serviceTemplatesPath` config field. Defaults to `databases/serviceTemplates/`. See the [Services guide](services.md). Example: `docs/serviceTemplates/` -Example Configuration -The `DOCKER_COMPUTE_ENVIRONMENTS` environment variable should be a JSON array of objects, where each object represents a Docker compute environment configuration. Below is an example configuration: +The `DOCKER_COMPUTE_ENVIRONMENTS` environment variable is used to configure Docker-based compute environments in Ocean Node. For the full guide — resources, GPU setup, constraints and pricing — see [Compute Configuration](compute.md). -`Disk` and `Ram` resources are always expressed in GB. +`cpu`, `ram`, and `disk` resources are **auto-detected** from the host at startup. All resource values are expressed in natural units: CPU in cores, RAM and disk in GB. + +The config has a two-level structure: +- **Connection level** (`C2DDockerConfig`): Docker connection details + optional hardware resource pool (GPUs, NICs, or overrides for auto-detected cpu/ram/disk) +- **Environment level** (`C2DEnvironmentConfig`): per-environment business rules (fees, access, durations) + lightweight resource refs ```json [ { "socketPath": "/var/run/docker.sock", - "scanImages": true, - "enableNetwork": false, + "scanImages": false, "imageRetentionDays": 7, "imageCleanupInterval": 86400, + "paymentClaimInterval": 3600, + "resources": [ - { - "id": "disk", - "total": 10 - } + { "id": "cpu", "total": 6 }, + { "id": "disk", "total": 50 } ], - "storageExpiry": 604800, - "maxJobDuration": 3600, - "minJobDuration": 60, - "access": { - "addresses": ["0x123", "0x456"], - "accessLists": [] - }, - "fees": { - "1": [ - { - "feeToken": "0x123", - "prices": [ + + "environments": [ + { + "id": "default", + "description": "CPU compute environment", + "storageExpiry": 604800, + "maxJobDuration": 3600, + "minJobDuration": 60, + "enableNetwork": false, + "access": { + "addresses": ["0x123", "0x456"], + "accessLists": [] + }, + "fees": { + "1": [ { - "id": "cpu", - "price": 1 + "feeToken": "0x967da4048cD07aB37855c090aAF366e4ce1b9F48", + "prices": [{ "id": "cpu", "price": 1 }] } ] - } - ] - }, - "free": { - "maxJobDuration": 60, - "minJobDuration": 10, - "maxJobs": 3, - "access": { - "addresses": [], - "accessLists": ["0x789"] - }, - "resources": [ - { - "id": "cpu", - "max": 1 - }, - { - "id": "ram", - "max": 1 }, - { - "id": "disk", - "max": 1 + "resources": [ + { "id": "cpu", "min": 1, "max": 4 }, + { "id": "ram", "min": 1, "max": 8 }, + { "id": "disk", "min": 1, "max": 50 } + ], + "free": { + "maxJobDuration": 60, + "minJobDuration": 10, + "maxJobs": 3, + "access": { "addresses": [], "accessLists": [] }, + "resources": [ + { "id": "cpu", "max": 1 }, + { "id": "ram", "max": 1 }, + { "id": "disk", "max": 1 } + ] } - ] - } + } + ] } ] ``` -#### Configuration Options - -- **socketPath**: Path to the Docker socket (e.g., docker.sock). -- **scanImages**: Whether Docker images should be scanned for vulnerabilities using Trivy. If enabled and critical vulnerabilities are found, the C2D job is rejected. -- **scanImageDBUpdateInterval**: How often to update the vulnerability database, in seconds. Default: 43200 (12 hours) -- **enableNetwork**: Whether networking is enabled for algorithm containers. Default: false -- **imageRetentionDays** - how long docker images are kept, in days. Default: 7 -- **imageCleanupInterval** - how often to run cleanup for docker images, in seconds. Min: 3600 (1hour), Default: 86400 (24 hours) -- **paymentClaimInterval** - how often to run payment claiming, in seconds. Default: 3600 (1 hour) -- **enableBenchmark** - when set to `true`, the node will auto-create a benchmark compute environment at startup using the system's available resources (CPU, RAM, disk, GPUs). Default: `false` -- **storageExpiry**: Amount of seconds for storage expiry.(Mandatory) -- **maxJobDuration**: Maximum duration in seconds for a job.(Mandatory) -- **minJobDuration**: Minimum duration in seconds for a job.(Mandatory) -- **access**: Access control configuration for paid compute jobs. If both `addresses` and `accessLists` are empty, all addresses are allowed. - - **addresses**: Array of Ethereum addresses allowed to run compute jobs. If empty and no access lists are configured, all addresses are allowed. - - **accessLists**: Array of AccessList contract addresses. Users holding NFTs from these contracts can run compute jobs. Checked across all supported networks. -- **fees**: Fee structure for the compute environment. - - **feeToken**: Token address for the fee. - - **prices**: Array of resource pricing information. - - **id**: Resource type (e.g., `cpu`, `ram`, `disk`). - - **price**: Price per unit of the resource. -- **resources**: Array of resources available in the compute environment. - - **id**: Resource type (e.g., `cpu`, `ram`, `disk`). - - **total**: Total number of the resource available. - - **min**: Minimum number of the resource needed for a job. - - **max**: Maximum number of the resource for a job. -- **free**: Optional configuration for free jobs. - - **storageExpiry**: Amount of seconds for storage expiry for free jobs. - - **maxJobDuration**: Maximum duration in seconds for a free job. - - **minJobDuration**: Minimum duration in seconds for a free job. - - **maxJobs**: Maximum number of simultaneous free jobs. - - **allowImageBuild**: If building images is allowed on free envs. Default: false - - **access**: Access control configuration for free compute jobs. Works the same as the main `access` field. - - **addresses**: Array of Ethereum addresses allowed to run free compute jobs. - - **accessLists**: Array of AccessList contract addresses for free compute access control. - - **resources**: Array of resources available for free jobs. - - **id**: Resource type (e.g., `cpu`, `ram`, `disk`). - - **total**: Total number of the resource available. - - **min**: Minimum number of the resource needed for a job. - - **max**: Maximum number of the resource for a job. +#### Connection-level fields + +- **socketPath** / **host** / **port** / **protocol** / **caPath** / **certPath** / **keyPath**: Docker connection settings. +- **scanImages**: Scan algorithm images for vulnerabilities with Trivy. Default: `false` +- **scanImageDBUpdateInterval**: Vulnerability DB update interval in seconds. Default: `43200` (12 hours) +- **imageRetentionDays**: How long to keep Docker images, in days. Default: `7` +- **imageCleanupInterval**: Image cleanup interval in seconds. Min: `3600`, Default: `86400` +- **paymentClaimInterval**: Payment claim interval in seconds. Min: `60`, Default: `3600` +- **resources** *(optional)*: Hardware resource pool for this connection. `cpu`, `ram`, and `disk` are auto-detected from the host — include them only to cap their totals or to add custom resources (GPUs, NICs, etc.). + - **id**: Resource identifier. `cpu`, `ram`, `disk` are built-in; any other string defines a custom resource. + - **kind**: `"discrete"` (non-fungible device, e.g. GPU) or `"fungible"` (interchangeable units, e.g. CPU). Auto-inferred: `"discrete"` if `init` is present, `"fungible"` otherwise. + - **shareable** *(discrete only)*: `true` allows multiple jobs to use the device simultaneously (NIC, TPM). Default: `false`. **Not allowed** on `type: "gpu"` or `type: "fpga"`. + - **total**: Total units available. Capped at the physical host limit. + - **cpuList** *(cpu resource only)*: Restricts which host core IDs compute containers may be pinned to. Comma-separated core IDs and/or integer ranges, e.g. `"3"`, `"0-1,3"` or `"32-63"`. Ranges `a-b` must have `b` strictly greater than `a`, all parts must be ascending and non-overlapping, core IDs may not exceed `8192`, and every core ID must exist on the host — otherwise the node fails to start with an error. Mutually exclusive with **total**: the cpu resource must specify exactly one of the two, and with `cpuList` the effective total is the number of listed cores. + - **min** / **max**: Per-job minimum/maximum. + - **description**, **platform**, **driverVersion**, **memoryTotal**: Informational metadata. + - **init**: Docker device configuration (`deviceRequests` for NVIDIA, `advanced` for AMD/Intel). See [Compute Configuration](compute.md#configuring-gpus). + - **constraints**: Cross-resource requirements. `{ "id": "ram", "min": 4 }` means renting this resource also requires 4 GB RAM. + +#### Environment-level fields + +- **id** *(optional)*: Stable identifier for the environment. Used to compute the environment hash. +- **description**: Human-readable description. +- **storageExpiry**: Seconds before compute results expire. +- **maxJobDuration** / **minJobDuration**: Maximum/minimum job duration in seconds. +- **maxJobs**: Maximum simultaneous paid jobs. +- **enableNetwork**: Whether algorithm containers can make outbound network connections. Default: `false` +- **access**: Access control for paid jobs. + - **addresses**: Ethereum addresses allowed to submit jobs. Empty + no accessLists = open access. + - **accessLists**: AccessList NFT contract addresses. NFT holders can submit jobs. +- **fees**: Fee structure per chain. + - **feeToken**: ERC-20 token address for payment. + - **prices**: `[{ "id": "", "price": }]` +- **resources**: Lightweight refs to the connection pool. `cpu`, `ram`, `disk` are always available. + - **id**: Must match a connection-level resource id, or `cpu` / `ram` / `disk`. + - **total**: Env aggregate ceiling — max units all running jobs in this env can use simultaneously. Omit for no per-env cap. + - **min** / **max**: Per-job limits for this environment (further restricted from pool values). + - **constraints**: Per-env override for pool-level constraints. Replaces (not merges) the pool constraints. Set `[]` to remove constraints for this env. +- **free** *(optional)*: Free tier configuration. + - **maxJobDuration** / **minJobDuration** / **maxJobs**: Free job limits. + - **allowImageBuild**: Allow image builds on free jobs. Default: `false` + - **access**: Same structure as the paid access field. + - **resources**: Same structure as environment resources — lightweight refs limiting what free jobs can request. + +> **Strict isolation**: If you need strict physical CPU isolation between environments (e.g., for regulated data), run each environment on a separate Docker connection. All environments on the same connection share the same CPU core pool dynamically. + +#### Migration from old format + +The old format placed hardware details (`init`, `driverVersion`, etc.) inside environments. This is now **a startup error**. + +**Old (rejected):** +```json +[{ "socketPath": "...", "environments": [{ "resources": [{ "id": "myGPU", "init": {...} }] }] }] +``` + +**New:** +```json +[{ + "socketPath": "...", + "resources": [{ "id": "myGPU", "kind": "discrete", "total": 1, "init": {...} }], + "environments": [{ "resources": [{ "id": "myGPU" }] }] +}] +``` + +Move all `init`, `driverVersion`, `platform`, `memoryTotal`, `type`, `kind`, and `constraints` fields to the connection-level `resources` array. Environment resources keep only `id` and optionally `total`/`min`/`max`/`constraints`. ### Docker Registry Authentication diff --git a/docs/serviceTemplates/README.md b/docs/serviceTemplates/README.md new file mode 100644 index 000000000..a2e79f073 --- /dev/null +++ b/docs/serviceTemplates/README.md @@ -0,0 +1,155 @@ +# Service templates + +Example service templates for the ocean-node *services on demand* feature. A template +describes a long-running containerized service (image, ports, launch command, resource +requirements) that consumers can start on a node via `SERVICE_START`. + +## How templates are loaded + +- The node reads templates from the directory set by `serviceTemplatesPath` in the node + config (default: `databases/serviceTemplates/`). This folder (`docs/serviceTemplates/`) + is a set of examples — copy the ones you want to offer into your configured path. +- Only `*.json` files are read; anything else (including this `README.md`) is ignored. +- A file may contain a single template object or an array of templates. +- Files are re-read on every request, so you can add/edit/remove templates without + restarting the node. +- Invalid JSON or schema-invalid templates are skipped with a warning; on duplicate + `id`s the first occurrence (filename-sorted) wins. + +## Template format + +Templates are validated against `ServiceTemplateSchema` +(`src/utils/config/schemas.ts`); the TypeScript shape is `ServiceTemplate` +(`src/@types/C2D/ServiceOnDemand.ts`). Key fields: + +| Field | Meaning | +| --- | --- | +| `id` | Unique id, `[a-z0-9][a-z0-9_-]{0,63}` | +| `image` + exactly one of `tag` / `checksum` / `dockerfile` | Image spec (`dockerfile` triggers a build and is gated per daemon by `allowImageBuild`) | +| `exposedPorts` | Container ports forwarded to host ports and returned as service endpoints | +| `command` / `entrypoint` | Docker CMD / ENTRYPOINT overrides | +| `envVars` | Fixed operator-set env vars (values never returned to callers) | +| `userConfigurableEnvVars` | Env vars the consumer supplies via ECIES-encrypted `userData` (optional regex `validation`, `sensitive` UI hint) | +| `requiredResources` / `recommendedResources` | Gate/score environment selection (`min` is enforced at `SERVICE_START`) | + +## Templates in this folder + +### `vllm-hf-model.json` — vLLM, any Hugging Face model (GPU) + +OpenAI-compatible inference server (`vllm/vllm-openai`) where the consumer picks the +model: the `MODEL_ID` user env var is substituted into the launch command (`${MODEL_ID}`) +and downloaded from the Hugging Face Hub at startup. Serves port 8000. Provide `HF_TOKEN` +for gated/private models. Requires a CUDA GPU sized to the chosen model. + +### `vllm-qwen-0_5b.json` — vLLM, Qwen2.5 0.5B Instruct (GPU) + +Same server, but the model is fixed by the operator to the small +`Qwen/Qwen2.5-0.5B-Instruct` (≥ 6 GB VRAM is plenty). Serves port 8000. `HF_TOKEN` is +only needed if you later switch to a gated model. + +### `vllm-nomic-embed.json` — vLLM, embeddings API (GPU) + +Runs vLLM in embedding/pooling mode (`--task embed`) serving +`nomic-ai/nomic-embed-text-v1.5` on port 8000 (`POST /v1/embeddings`). Needs +`--trust-remote-code` because the model ships custom modeling code executed in the +container. A few GB of VRAM is enough for this 137M model. + +### `llamacpp-phi4-cpu.json` — llama.cpp, Phi-4 (CPU) + +CPU-only OpenAI-compatible chat server. The `vllm/vllm-openai` image is CUDA-only, so +this template uses llama.cpp (`ghcr.io/ggml-org/llama.cpp:server`) instead, downloading +the full Phi-4 (14B) as a Q4_K_M GGUF quantization — quantization is what makes 14B +feasible on CPU. Serves port 8080. For CPU inference with vLLM proper, see +`vllm-dual-lite-cpu.json` below. + +### `vllm-dual-lite-gpu.json` — vLLM, two lite models on one 3 GB GPU + +Two OpenAI-compatible vLLM servers in a single container, sized to fit **together** on a +3 GB VRAM GPU: + +- `Qwen/Qwen2.5-0.5B-Instruct` (~1.0 GB fp16) on port 8000, `--gpu-memory-utilization 0.42` +- `HuggingFaceTB/SmolLM2-360M-Instruct` (~0.7 GB fp16) on port 8001, `--gpu-memory-utilization 0.30` + +Budget math for 3 GB: the utilization split caps vLLM at ~2.2 GB combined, leaving +~0.8 GB for the two CUDA contexts. `--enforce-eager` (no CUDA-graph memory), +`--max-model-len 2048`, `--max-num-seqs 4`, `--dtype half` (works on GPUs without bf16, +e.g. the T4) and `--swap-space 1` keep everything inside the envelope. + +Tuning caveat: 3 GB for two vLLM instances is inherently tight — if a particular GPU's +driver/context overhead runs high, loosen `--max-model-len` first or drop SmolLM2's +utilization to 0.25. + +### `vllm-dual-lite-cpu.json` — vLLM, two lite models on CPU only + +Two OpenAI-compatible vLLM servers, CPU backend, no GPU resource required. Uses the +**official vLLM CPU image** (`public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo`, x86-64, +AVX-512 recommended) — genuinely vLLM on CPU, unlike the llama.cpp workaround above. + +- `Qwen/Qwen2.5-0.5B-Instruct` on port 8000 +- `microsoft/Phi-4-mini-instruct` (3.8B) on port 8001 — the lite Phi-4 variant; full 14B + Phi-4 needs ~28 GB unquantized and is impractical on CPU + +Each process gets a 2 GiB CPU KV cache (`VLLM_CPU_KVCACHE_SPACE=2`, exported in the +launch script) and `--max-model-len 4096`; both share the CPU cores, so expect modest +throughput. + +## The dual-model pattern + +One vLLM process serves exactly one model, so the two dual templates override +`entrypoint` to `["/bin/bash", "-c"]` and run two `vllm serve` processes from a single +`command` script: + +1. Start model A in the background and remember its PID. +2. Poll until port 8000 accepts connections (bash `/dev/tcp` probe — no curl needed), + bailing out if the process died. On GPU this also guarantees the two instances never + profile GPU memory at the same time. +3. Start model B on port 8001. +4. `wait -n` — the script (container PID 1) exits as soon as either server exits, so a + crashed model stops the whole service instead of leaving it half-alive. + +Both dual templates expose an optional `HF_TOKEN` user env var; all models referenced +here are ungated, so it is only needed if an operator swaps in a gated model. + +## GPU memory with multiple vLLM instances: who manages what + +Within one instance vLLM manages memory for you; across instances it does nothing — +the GPU must be partitioned manually. That is why the dual GPU template hard-codes the +split. + +**What vLLM manages** — inside a single `vllm serve` process, memory management is +excellent: it loads the weights, profiles peak activation usage, then pre-allocates +everything left in its budget as paged KV cache (PagedAttention), and handles +scheduling/preemption within that. Per-request memory is never the operator's problem. + +**What it doesn't** — that budget comes from `--gpu-memory-utilization`, which is a +*per-process* fraction of the GPU (default **0.9**). Each instance assumes it owns that +slice and knows nothing about other processes. Two instances with defaults means +0.9 + 0.9 of the same GPU → the second one OOMs during KV-cache allocation. There is no +cross-instance coordination, negotiation, or dynamic rebalancing — the split is static +for the life of the process. + +When running multiple models on one GPU, the template (or operator) is responsible for +three things: + +1. **Fractions that sum below 1.0** — and not just barely: the CUDA context + (~250–500 MB per process) and allocator fragmentation live *outside* vLLM's + accounting. The dual GPU template uses 0.42 + 0.30 (~2.2 GB of 3 GB), leaving + ~0.8 GB for the two contexts. +2. **Startup sequencing** — during startup each instance profiles memory and has a + usage peak; overlapping profiling on a tight GPU can OOM even if steady-state fits. + The template's `/dev/tcp` readiness gate serializes this. +3. **Caps that shrink the budget it will try to claim** — `--max-model-len`, + `--max-num-seqs`, and `--enforce-eager` (CUDA graphs cost extra memory outside the + KV budget on some versions) keep both the profiling peak and steady state + predictable. + +Two caveats worth knowing: + +- **One vLLM server = one model.** vLLM's OpenAI server cannot serve two different base + models from a single process, which is why the dual templates run two processes. The + exception is **LoRA adapters**: if the "multiple models" are fine-tunes of the same + base, one instance with `--enable-lora` serves them all under one memory budget with + shared base weights — vLLM manages everything, and it is far more memory-efficient. +- For harder isolation than "fractions that behave", the GPU-side options are MIG + partitions (A100/H100 class) or CUDA MPS — but for the 3 GB template scenario, the + static fraction split is the right tool. diff --git a/docs/serviceTemplates/llamacpp-phi4-cpu.json b/docs/serviceTemplates/llamacpp-phi4-cpu.json new file mode 100644 index 000000000..c5f171b7a --- /dev/null +++ b/docs/serviceTemplates/llamacpp-phi4-cpu.json @@ -0,0 +1,23 @@ +{ + "id": "llamacpp-phi4-cpu", + "name": "llama.cpp — Phi-4 (CPU)", + "description": "CPU-only OpenAI-compatible chat server (llama.cpp) running Microsoft Phi-4 as a Q4_K_M GGUF quantization. On startup llama.cpp downloads the GGUF from the Hugging Face Hub (-hf), then serves /v1/chat/completions on port 8080. No GPU required. Note: vLLM's published image is CUDA-only, so this CPU template uses llama.cpp instead; the model is downloaded then run locally just like the vLLM templates.", + "image": "ghcr.io/ggml-org/llama.cpp", + "tag": "server", + "exposedPorts": [8080], + "command": [ + "-hf", + "bartowski/phi-4-GGUF:Q4_K_M", + "--host", + "0.0.0.0", + "--port", + "8080", + "-c", + "8192" + ], + "requiredResources": [ + { "id": "cpu", "min": 4, "recommended": 8, "unit": "cores" }, + { "id": "ram", "min": 12, "recommended": 16, "unit": "GB" }, + { "id": "disk", "min": 15, "recommended": 20, "unit": "GB" } + ] +} diff --git a/docs/serviceTemplates/vllm-dual-lite-cpu.json b/docs/serviceTemplates/vllm-dual-lite-cpu.json new file mode 100644 index 000000000..b9d8333ef --- /dev/null +++ b/docs/serviceTemplates/vllm-dual-lite-cpu.json @@ -0,0 +1,24 @@ +{ + "id": "vllm-dual-lite-cpu", + "name": "vLLM — two lite models on CPU only (Qwen2.5 0.5B + Phi-4-mini)", + "description": "Two local OpenAI-compatible LLM inference servers (vLLM, CPU backend — no GPU required) in a single container, using the official vLLM CPU image (public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo, x86-64 with AVX-512 recommended). On startup each vLLM process downloads its model from the Hugging Face Hub: Qwen/Qwen2.5-0.5B-Instruct is served on port 8000 and microsoft/Phi-4-mini-instruct (3.8B — the lite Phi-4 variant; full 14B Phi-4 is impractical on CPU) on port 8001 (e.g. POST /v1/chat/completions on either port). The second model starts only after the first is reachable; each process gets a 2 GiB CPU KV cache (VLLM_CPU_KVCACHE_SPACE) and both share the CPU cores, so expect modest throughput. If either server exits the container stops. Both models are ungated; set HF_TOKEN only if you later switch to a gated model.", + "image": "public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo", + "tag": "latest", + "exposedPorts": [8000, 8001], + "entrypoint": ["/bin/bash", "-c"], + "command": [ + "export VLLM_CPU_KVCACHE_SPACE=2\nvllm serve Qwen/Qwen2.5-0.5B-Instruct --host 0.0.0.0 --port 8000 --max-model-len 4096 --max-num-seqs 4 & P1=$!\nuntil (exec 3<>/dev/tcp/127.0.0.1/8000) 2>/dev/null; do kill -0 $P1 2>/dev/null || exit 1; sleep 5; done\nvllm serve microsoft/Phi-4-mini-instruct --host 0.0.0.0 --port 8001 --max-model-len 4096 --max-num-seqs 4 & P2=$!\nwait -n $P1 $P2" + ], + "userConfigurableEnvVars": [ + { + "key": "HF_TOKEN", + "validation": "^hf_[A-Za-z0-9]{20,}$", + "sensitive": true + } + ], + "requiredResources": [ + { "id": "cpu", "min": 4, "recommended": 8, "unit": "cores" }, + { "id": "ram", "min": 16, "recommended": 32, "unit": "GB" }, + { "id": "disk", "min": 20, "recommended": 40, "unit": "GB" } + ] +} diff --git a/docs/serviceTemplates/vllm-dual-lite-gpu.json b/docs/serviceTemplates/vllm-dual-lite-gpu.json new file mode 100644 index 000000000..587e02912 --- /dev/null +++ b/docs/serviceTemplates/vllm-dual-lite-gpu.json @@ -0,0 +1,32 @@ +{ + "id": "vllm-dual-lite-gpu", + "name": "vLLM — two lite models on one small GPU (Qwen2.5 0.5B + SmolLM2 360M)", + "description": "Two local OpenAI-compatible LLM inference servers (vLLM) in a single container, sized to fit together on a 3 GB VRAM GPU. On startup each vLLM process downloads its model from the Hugging Face Hub: Qwen/Qwen2.5-0.5B-Instruct is served on port 8000 and HuggingFaceTB/SmolLM2-360M-Instruct on port 8001 (e.g. POST /v1/chat/completions on either port). The second model starts only after the first is reachable, so GPU memory profiling never overlaps; the GPU budget is split via --gpu-memory-utilization (0.42 + 0.30), and --enforce-eager, --max-model-len 2048, --max-num-seqs 4, fp16 keep both inside 3 GB. If either server exits the container stops. Both models are ungated; set HF_TOKEN only if you later switch to a gated model.", + "image": "vllm/vllm-openai", + "tag": "latest", + "exposedPorts": [8000, 8001], + "entrypoint": ["/bin/bash", "-c"], + "command": [ + "vllm serve Qwen/Qwen2.5-0.5B-Instruct --host 0.0.0.0 --port 8000 --dtype half --max-model-len 2048 --max-num-seqs 4 --gpu-memory-utilization 0.42 --swap-space 1 --enforce-eager & P1=$!\nuntil (exec 3<>/dev/tcp/127.0.0.1/8000) 2>/dev/null; do kill -0 $P1 2>/dev/null || exit 1; sleep 5; done\nvllm serve HuggingFaceTB/SmolLM2-360M-Instruct --host 0.0.0.0 --port 8001 --dtype half --max-model-len 2048 --max-num-seqs 4 --gpu-memory-utilization 0.30 --swap-space 1 --enforce-eager & P2=$!\nwait -n $P1 $P2" + ], + "userConfigurableEnvVars": [ + { + "key": "HF_TOKEN", + "validation": "^hf_[A-Za-z0-9]{20,}$", + "sensitive": true + } + ], + "requiredResources": [ + { "id": "cpu", "min": 2, "recommended": 4, "unit": "cores" }, + { "id": "ram", "min": 8, "recommended": 16, "unit": "GB" }, + { "id": "disk", "min": 10, "recommended": 20, "unit": "GB" }, + { + "kind": "discrete", + "type": "gpu", + "min": 1, + "recommended": 1, + "unit": "count", + "description": "CUDA-capable GPU with >= 3 GB VRAM (compute capability >= 7.0); both models share the one GPU" + } + ] +} diff --git a/docs/serviceTemplates/vllm-hf-model.json b/docs/serviceTemplates/vllm-hf-model.json new file mode 100644 index 000000000..373de1b74 --- /dev/null +++ b/docs/serviceTemplates/vllm-hf-model.json @@ -0,0 +1,44 @@ +{ + "id": "vllm-hf-model", + "name": "vLLM — any Hugging Face model (MODEL_ID)", + "description": "Local OpenAI-compatible LLM inference server (vLLM) that serves a Hugging Face model of the consumer's choice. The model id is supplied via MODEL_ID and substituted into the launch command (${MODEL_ID}); on startup vLLM downloads that model from the Hugging Face Hub, then serves it on port 8000 (e.g. POST /v1/chat/completions). Provide HF_TOKEN for gated/private models. Requires a CUDA GPU sized to the chosen model.", + "image": "vllm/vllm-openai", + "tag": "latest", + "exposedPorts": [8000], + "command": [ + "--model", + "${MODEL_ID}", + "--host", + "0.0.0.0", + "--port", + "8000", + "--max-model-len", + "8192", + "--gpu-memory-utilization", + "0.9" + ], + "userConfigurableEnvVars": [ + { + "key": "MODEL_ID", + "validation": "^[A-Za-z0-9][\\w.-]*(/[A-Za-z0-9][\\w.-]*)?$" + }, + { + "key": "HF_TOKEN", + "validation": "^hf_[A-Za-z0-9]{20,}$", + "sensitive": true + } + ], + "requiredResources": [ + { "id": "cpu", "min": 2, "recommended": 4, "unit": "cores" }, + { "id": "ram", "min": 8, "recommended": 16, "unit": "GB" }, + { "id": "disk", "min": 20, "recommended": 50, "unit": "GB" }, + { + "kind": "discrete", + "type": "gpu", + "min": 1, + "recommended": 1, + "unit": "count", + "description": "CUDA-capable GPU; size VRAM to the chosen MODEL_ID" + } + ] +} diff --git a/docs/serviceTemplates/vllm-nomic-embed.json b/docs/serviceTemplates/vllm-nomic-embed.json new file mode 100644 index 000000000..ad2baa744 --- /dev/null +++ b/docs/serviceTemplates/vllm-nomic-embed.json @@ -0,0 +1,32 @@ +{ + "id": "vllm-nomic-embed", + "name": "vLLM — Nomic Embed (embeddings API)", + "description": "Local embedding server (vLLM) serving nomic-ai/nomic-embed-text-v1.5. On startup vLLM downloads the model from the Hugging Face Hub, then exposes the OpenAI-compatible embeddings API on port 8000 (POST /v1/embeddings) — handy for testing the embeddings endpoint. Runs vLLM in embedding/pooling mode (--task embed) and requires --trust-remote-code because the model ships custom modeling code that is executed in the container. The model is small, so a few GB of VRAM is enough.", + "image": "vllm/vllm-openai", + "tag": "latest", + "exposedPorts": [8000], + "command": [ + "--model", + "nomic-ai/nomic-embed-text-v1.5", + "--task", + "embed", + "--trust-remote-code", + "--host", + "0.0.0.0", + "--port", + "8000" + ], + "requiredResources": [ + { "id": "cpu", "min": 2, "recommended": 4, "unit": "cores" }, + { "id": "ram", "min": 4, "recommended": 8, "unit": "GB" }, + { "id": "disk", "min": 5, "recommended": 10, "unit": "GB" }, + { + "kind": "discrete", + "type": "gpu", + "min": 1, + "recommended": 1, + "unit": "count", + "description": "CUDA-capable GPU; a few GB VRAM is enough for this 137M embedding model" + } + ] +} diff --git a/docs/serviceTemplates/vllm-qwen-0_5b.json b/docs/serviceTemplates/vllm-qwen-0_5b.json new file mode 100644 index 000000000..3aefff604 --- /dev/null +++ b/docs/serviceTemplates/vllm-qwen-0_5b.json @@ -0,0 +1,40 @@ +{ + "id": "vllm-qwen-0-5b", + "name": "vLLM — Qwen2.5 0.5B Instruct (local)", + "description": "Local OpenAI-compatible LLM inference server (vLLM) serving the small Qwen/Qwen2.5-0.5B-Instruct model. On startup vLLM downloads the model from the Hugging Face Hub, then serves it locally on port 8000 (e.g. POST /v1/chat/completions). The model is fixed by the operator; the consumer only needs a GPU-enabled environment. Set HF_TOKEN only if you later switch to a gated model.", + "image": "vllm/vllm-openai", + "tag": "latest", + "exposedPorts": [8000], + "command": [ + "--model", + "Qwen/Qwen2.5-0.5B-Instruct", + "--host", + "0.0.0.0", + "--port", + "8000", + "--max-model-len", + "8192", + "--gpu-memory-utilization", + "0.9" + ], + "userConfigurableEnvVars": [ + { + "key": "HF_TOKEN", + "validation": "^hf_[A-Za-z0-9]{20,}$", + "sensitive": true + } + ], + "requiredResources": [ + { "id": "cpu", "min": 2, "recommended": 4, "unit": "cores" }, + { "id": "ram", "min": 8, "recommended": 16, "unit": "GB" }, + { "id": "disk", "min": 10, "recommended": 20, "unit": "GB" }, + { + "kind": "discrete", + "type": "gpu", + "min": 1, + "recommended": 1, + "unit": "count", + "description": "CUDA-capable GPU (>= 6 GB VRAM is plenty for this 0.5B model)" + } + ] +} diff --git a/docs/services.md b/docs/services.md new file mode 100644 index 000000000..97540bf38 --- /dev/null +++ b/docs/services.md @@ -0,0 +1,209 @@ +# Services (Service-on-Demand) + +A high-level overview of the service-on-demand feature: how it works, how it is +configured, and the security properties you should be aware of. + +## What is a service? + +A **service** is a long-running Docker container that a consumer launches on a compute +environment and pays for up front via on-chain escrow. Unlike a compute job — which runs +an algorithm to completion and exits — a service stays up for a requested **duration** and +exposes one or more network **endpoints** (`http://:`) that the +consumer can connect to while it runs. + +The consumer supplies the container spec directly in the request: an `image` +(referenced by `tag` or `checksum`, or an inline `dockerfile` when the operator allows +building), optional `dockerCmd` / `dockerEntrypoint`, the container ports to expose, the +requested resources ([cpu/ram/disk/gpu](compute.md)), the duration, and encrypted `userData` +that is injected as container environment variables. + +## Lifecycle + +All endpoints live under `/api/services`. Every request except `serviceTemplates` is +authenticated by a signature (or auth token) over the caller's `consumerAddress` + +`nonce` + command. `serviceStatus` is a `GET`, so it carries `consumerAddress`, `nonce`, +and `signature` as query parameters (or an auth-token `Authorization` header). + +| Command | Route | Method | Purpose | +| --- | --- | --- | --- | +| `SERVICE_START` | `/api/services/serviceStart` | POST | Validate, persist a `Starting` record, and return the `serviceId` immediately (escrow + image + container happen in the background) | +| `SERVICE_GET_STATUS` | `/api/services/serviceStatus` | GET | Read job status / endpoints — authenticated, owner-scoped (see notice below); poll this to follow a starting service | +| `SERVICE_LIST` | `/api/services/serviceList` | GET | Node-wide service listing — authenticated, **not** owner-scoped. Default: only services currently holding a resource reservation; `status=` filters to one specific status, `includeAllStatuses=true` returns everything, `fromTimestamp` keeps services created at/after that moment. Output is listing-sanitized (no `userData`, no `dockerCmd`/`dockerEntrypoint`, no Dockerfile) | +| `SERVICE_EXTEND` | `/api/services/serviceExtend` | POST | Pay to push the expiry further out | +| `SERVICE_RESTART` | `/api/services/serviceRestart` | POST | Recreate the container (no extra charge); asynchronous like start — returns once the job is `Restarting`, poll `serviceStatus` | +| `SERVICE_STOP` | `/api/services/serviceStop` | POST | Tear down the container; the paid resource reservation (cpu/ram/gpu + host ports) is kept until `expiresAt`, so the service can be restarted anytime on the same endpoints | +| `SERVICE_GET_TEMPLATES` | `/api/services/serviceTemplates` | GET | List operator-published service templates | +| `SERVICE_GET_STREAMABLE_LOGS` | `/api/services/serviceStreamableLogs` | GET | Stream the container's live stdout/stderr logs — authenticated, owner-scoped; available while `Running` or `Error`; optional `since` to skip history | + +**Start is asynchronous.** `serviceStart` does only the fast, synchronous validation and then +returns the `serviceId` right away — it does **not** wait for escrow or the (potentially +multi-minute) image pull/build. A background loop on the node then advances the service through +a sequence of statuses; clients **poll `serviceStatus`** to follow it to `Running` (or a +terminal `*Failed` / `Error`). + +**Handler (synchronous, before responding):** signature check → environment + access-list + +`features.services` check → `userData` decrypt (validity check) → duration cap → resource +resolution & availability → cost computed from **server-side** environment pricing → escrow +funds pre-check (fail fast with `400 Insufficient escrow funds` when the consumer's available +escrow visibly can't cover the cost; best-effort — an RPC hiccup skips it and the background +Locking step remains the authoritative check) → persist the job as `Starting` (which also +reserves its resources) → respond `200` with the `serviceId`. + +**Background pipeline (per the start statuses below):** +`Starting (10)` → **locking** `Locking (20)`: escrow `createLock` (+ wait for it to mine) → +**image** `PullImage (11)` / `BuildImage (13)`: pull or build the image and run the vulnerability +scan → **payment** `Claiming (30)`: `claimLock` on success, or `cancelLock` (refund) if the image +step failed → allocate host ports, create the network, create + start the container → +`Running (40)`. + +Escrow is **claimed only after the image succeeds**; if the image pull/build/scan fails, or +container creation fails before the claim, the lock is **cancelled (refunded)** and the job ends +in a `*Failed` / `Error` status. This is a change from the previous synchronous flow, which +locked-then-claimed up front. + +**Restart is asynchronous too.** `serviceRestart` performs only the fast validations +(ownership, environment/access, not expired, payment not refunded), persists the job as +`Restarting (45)` and responds immediately — the teardown, image re-pull/rebuild and new +container happen in the background under the same per-service lock. Poll `serviceStatus` +and watch `Restarting` → `PullImage`/`BuildImage` → `Running` (or `Error` with the failure +reason in `statusText`). A service whose start payment was **never claimed** — the escrow +lock failed outright (e.g. insufficient funds) or was refunded before being claimed — +cannot be restarted: it was never paid for, so restarting it would run the service for +free. Start a new service instead. + +**The reservation lasts the whole paid window — only `Expired` releases it.** The consumer +paid for the resources for a time interval and may use them as they please within it: +running the service, stopping it, restarting it. An explicit `SERVICE_STOP` therefore tears +down the container/network but **keeps** the resource amounts (cpu/ram/gpu) counted and the +host ports reserved — another consumer cannot take them, and a restart resumes on the same +endpoints. The reservation is tied to **payment**: an `Error`/`Stopped` job whose payment +was never claimed (lock failed or refunded) does not reserve anything — otherwise anyone +could squat a node's GPU for free by starting services against an empty escrow account. +Once `expiresAt` passes, the expiry sweep tears down whatever is left, marks the +job `Expired`, and only then releases everything. The sweep refuses to mark `Expired` while +teardown fails (e.g. Docker unreachable) — the job stays `Error` and is retried every tick, +so a resource release is never silently skipped. + +**`Running` is monitored too.** The same background loop that advances a starting service also +checks every `Running` service's container on each tick (~every few seconds). If the container +exits on its own — crash, OOM, or the Docker daemon itself becoming unreachable — the job is +moved to `Error` immediately instead of waiting for `expiresAt`. This health check does **not** +release the service's reserved host ports/network/container record, since the consumer already +paid for them; use `SERVICE_RESTART` to bring the service back on the same endpoints. `Error` +counts as an active/resource-reserving status just like `Running` and `Stopped` do — it still +occupies its cpu/ram/gpu allocation and keeps its host ports held — until it is restarted or +swept by the expiry check once `expiresAt` passes (which then fully releases everything). + +**Restart is self-healing with respect to leftover Docker state.** Each service gets a Docker +network with the deterministic name `ocean-svc-`. Teardown (restart, stop, expiry +sweep) removes that network by name — not just by the stored network id — force-removing any +stale attached container first, so state leaked by a node crash mid-start cannot wedge the +service. If network creation still hits a name conflict, the stale network is removed and +creation is retried once. + +A leftover network is deliberately **removed and recreated rather than reused**. Reusing it +would save nothing: a leaked network can still have a stale container attached (crashed after +`container.start()` but before the job record was persisted), still bound to the service's +host ports — so the old container must be inspected and force-removed either way, and at that +point recreating the now-empty network is a single cheap API call. Recreating also guarantees +the network always reflects the current code's configuration instead of silently inheriting +whatever options a previous node version created it with, and it matches restart's overall +tear-down-and-rebuild semantics (the container is never reused either). + +**Lifecycle operations are exclusive per service.** At most one lifecycle operation — the +background start pipeline, `SERVICE_RESTART`, `SERVICE_STOP`, or the expiry sweep — runs per +service at a time. A restart or stop issued while another operation is in flight (e.g. a +restart still pulling the image) is rejected with +`Service has a start/stop/restart operation in progress — retry shortly`; simply retry +once the in-flight operation settles. Without this exclusivity, the background loop's +crash-orphan recovery could tear down the `ocean-svc-` network in the middle of a +restart that had just created it, failing the restart with +`network ocean-svc- not found`. If a service expires while such an operation is in +flight, the expiry sweep simply retries on a later tick. + +Exclusivity holds **across node processes** too, not just within one: each operation also +takes a lease row in the SQLite `service_locks` table, so two processes sharing the same +`databases/` directory and Docker daemon (e.g. an old container still running during a +redeploy) cannot run conflicting operations on the same service. Leases are heartbeated +every 30 s while the operation runs; a lease not refreshed for 2 minutes belongs to a +crashed process and is stolen automatically, so no manual cleanup is ever needed. + +## Configuration + +Service-on-demand is configured per Docker connection under `serviceOnDemand`: + +| Field | Meaning | +| --- | --- | +| `enabled` | Master switch for the feature on this connection. | +| `nodeHost` | Externally reachable host used to build endpoint URLs. | +| `hostPortRange` | `[start, end]` range the node allocates published host ports from. | +| `maxDurationSeconds` | Upper bound on a service's lifetime (default 86400). | +| `allowImageBuild` | If true, consumers may submit an inline `dockerfile` to build. | + +Whether a given environment accepts services is gated by its `features.services` flag, +and access can be restricted with the environment's `access` allow-list +(`addresses` + on-chain `accessLists`). Operator-published **templates** are loaded from +`serviceTemplatesPath` (default `databases/serviceTemplates/`); template secret values +are never returned by the API (only the env-var keys are exposed). + +The compute environments a service can run on — and the resources (cpu/ram/disk/gpu) it may +request — are the same ones configured at the node's Docker-connection level for compute jobs, +and services draw from and are counted against the same shared resource pool. For how to +declare resources, configure GPUs, set per-environment constraints, and price them, see the +[Compute Configuration guide](compute.md). + +## Security model & important notices + +- **Container hardening.** Service containers are created with + `SecurityOpt: ['no-new-privileges']`, `CapDrop: ['ALL']`, and `PidsLimit: 512`. + Unlike the compute path, the service path does **not** force a non-root `User` — + arbitrary service images often expect to start as root, so the image's declared user + is kept. Dropping all capabilities + `no-new-privileges` keeps that root process + unprivileged. + +- **⚠️ Low ports won't bind inside the container.** Because `CapDrop: ['ALL']` removes + `NET_BIND_SERVICE`, a process **inside** the container cannot bind to a container port + below 1024. Have your service listen on a **high port** (the externally published + *host* port is allocated by the node from `hostPortRange` regardless). If a specific + image genuinely needs a low in-container port, that requires explicitly adding + `CapAdd: ['NET_BIND_SERVICE']` in the engine — it is intentionally not enabled by + default. + +- **Access lists apply to the whole lifecycle.** `start`, `extend`, and `restart` all + re-check the environment's `access` allow-list (access lists are mutable, so a + revoked consumer cannot keep a service alive). `stop` is owner-gated only, so a + revoked owner can still shut their own service down. + +- **No privileged/advanced Docker config.** The service path deliberately omits the + user-injectable advanced Docker config (host bind mounts, extra capabilities, + `seccomp:unconfined`, devices beyond the priced GPU pool) that the compute path + supports. Do not thread it in. + +- **Payment is server-priced.** Cost is computed only from the environment's configured + pricing for the requested token/chain; the consumer cannot influence the charged + amount, and the escrow payer is always the signature-authenticated `consumerAddress` + (you cannot charge someone else). + +- **`serviceStatus` is authenticated and owner-scoped.** The caller must supply + `consumerAddress` plus a valid `nonce`/`signature` (or auth token) proving control of + that address; results are restricted to services owned by it, so one consumer cannot + read another's job records or endpoint URLs. That said, a published service endpoint is + still reachable by anyone who learns or guesses its URL — the node only port-forwards + and does not authenticate traffic to the container, so put your own authentication in + front of any sensitive service and do not rely on endpoint-URL secrecy as access + control. + +- **`serviceStreamableLogs` is authenticated and owner-scoped, like `serviceStatus`.** + Container stdout/stderr can leak secrets or sensitive request data, so the same + proof-of-`consumerAddress` + ownership check gates log access — a non-owner gets `401`. + Logs are only served while the service is `Running` or `Error` (a crashed container's + logs stay available for diagnosis until `stop`/`restart` tears it down); otherwise the + route returns `404`. By default the full history since container start is returned before + the stream switches to following live output — for a service that has been running for + days or weeks that can be a lot of data, so pass `since` (a Unix timestamp, or a relative + duration like `1h`) to skip straight to recent output. + +- **`allowImageBuild` runs arbitrary build instructions.** When enabled, a consumer's + inline `dockerfile` is built by the Docker daemon, so its `RUN` steps execute arbitrary + commands in the daemon's build sandbox. Leave it disabled unless you intend to offer + build-from-source and trust the consumer set. diff --git a/package-lock.json b/package-lock.json index 9b473de12..0066e6c4d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -35,9 +35,8 @@ "@libp2p/upnp-nat": "^4.0.9", "@libp2p/websockets": "^10.1.2", "@multiformats/multiaddr": "^12.2.3", - "@oceanprotocol/contracts": "^2.7.0", + "@oceanprotocol/contracts": "^2.9.0", "@oceanprotocol/ddo-js": "^0.4.0", - "axios": "^1.15.0", "base58-js": "^2.0.0", "basic-ftp": "^5.3.1", "cors": "^2.8.5", @@ -58,7 +57,6 @@ "lodash": "^4.18.1", "lzma-purejs-requirejs": "^1.0.0", "node-cron": "^3.0.3", - "sqlite3": "^6.0.1", "stream-concat": "^1.0.0", "tar": "^7.5.16", "uint8arrays": "^4.0.6", @@ -99,6 +97,9 @@ "sinon": "^19.0.2", "tsx": "^4.22.4", "typescript": "^5.9.3" + }, + "engines": { + "node": ">=22.13.0" } }, "node_modules/@achingbrain/http-parser-js": { @@ -2765,16 +2766,6 @@ "node": ">=14" } }, - "node_modules/@gar/promise-retry": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@gar/promise-retry/-/promise-retry-1.0.3.tgz", - "integrity": "sha512-GmzA9ckNokPypTg10pgpeHNQe7ph+iIKKmhKu3Ob9ANkswreCx7R3cKmY781K8QK3AqVL3xVh9A42JvIAbkkSA==", - "license": "MIT", - "optional": true, - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, "node_modules/@grpc/grpc-js": { "version": "1.14.4", "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.14.4.tgz", @@ -5186,60 +5177,10 @@ "lodash": "^4.15.0" } }, - "node_modules/@npmcli/agent": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@npmcli/agent/-/agent-4.0.0.tgz", - "integrity": "sha512-kAQTcEN9E8ERLVg5AsGwLNoFb+oEG6engbqAU2P43gD4JEIkNGMHdVQ096FsOAAYpZPB0RSt0zgInKIAS1l5QA==", - "license": "ISC", - "optional": true, - "dependencies": { - "agent-base": "^7.1.0", - "http-proxy-agent": "^7.0.0", - "https-proxy-agent": "^7.0.1", - "lru-cache": "^11.2.1", - "socks-proxy-agent": "^8.0.3" - }, - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, - "node_modules/@npmcli/agent/node_modules/lru-cache": { - "version": "11.3.3", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.3.3.tgz", - "integrity": "sha512-JvNw9Y81y33E+BEYPr0U7omo+U9AySnsMsEiXgwT6yqd31VQWTLNQqmT4ou5eqPFUrTfIDFta2wKhB1hyohtAQ==", - "license": "BlueOak-1.0.0", - "optional": true, - "engines": { - "node": "20 || >=22" - } - }, - "node_modules/@npmcli/fs": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/@npmcli/fs/-/fs-5.0.0.tgz", - "integrity": "sha512-7OsC1gNORBEawOa5+j2pXN9vsicaIOH5cPXxoR6fJOmH6/EXpJB2CajXOu1fPRFun2m1lktEFX11+P89hqO/og==", - "license": "ISC", - "optional": true, - "dependencies": { - "semver": "^7.3.5" - }, - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, - "node_modules/@npmcli/redact": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@npmcli/redact/-/redact-4.0.0.tgz", - "integrity": "sha512-gOBg5YHMfZy+TfHArfVogwgfBeQnKbbGo3pSUyK/gSI0AVu+pEiDVcKlQb0D8Mg1LNRZILZ6XG8I5dJ4KuAd9Q==", - "license": "ISC", - "optional": true, - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, "node_modules/@oceanprotocol/contracts": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/@oceanprotocol/contracts/-/contracts-2.7.0.tgz", - "integrity": "sha512-6rXT/agjty4VyT0j/Do13Rf8dH0eweL57e7rFSv4KmANFx3wdTcQoInHZsoRpwXZf1MmMgHVWL4/ZvqHtRqMRA==", + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/@oceanprotocol/contracts/-/contracts-2.9.0.tgz", + "integrity": "sha512-B3dQNxIYD7bASNE066vfZu6Ik5uHZ/1c+QEcUvAsfoNvUUJ5+uQfIvhwrpdMCza1GtE11EW2glDPZvbr2LwFsg==", "license": "Apache-2.0" }, "node_modules/@oceanprotocol/ddo-js": { @@ -7204,16 +7145,6 @@ "integrity": "sha512-fbOadP7twxt0ZYT9mgIC+xQMk6f3pYYLI5a/2UJ/mc/ygqb/NoVv2ryK3lTtoi74xwkdpUeDwIuFQSosowzUgg==", "license": "MIT" }, - "node_modules/abbrev": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-4.0.0.tgz", - "integrity": "sha512-a1wflyaL0tHtJSmLSOVybYhy22vRih4eduhhrkcjgrWGnRfrZtovJ2FRjxuTtkkj47O/baf0R86QU5OuYpz8fA==", - "license": "ISC", - "optional": true, - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, "node_modules/abort-controller": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", @@ -7331,16 +7262,6 @@ "integrity": "sha512-G965FqalsNyrPqgEGON7nIx1e/OVENSgiEIzyC63haUMuvNnwIgIjMs52hlTCKhkBny7A2ORNlfY9Zu+jmGk1Q==", "license": "MIT" }, - "node_modules/agent-base": { - "version": "7.1.4", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", - "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", - "license": "MIT", - "optional": true, - "engines": { - "node": ">= 14" - } - }, "node_modules/aggregate-error": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz", @@ -7885,15 +7806,6 @@ "dev": true, "license": "Apache-2.0" }, - "node_modules/bindings": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", - "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", - "license": "MIT", - "dependencies": { - "file-uri-to-path": "1.0.0" - } - }, "node_modules/bl": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", @@ -8227,125 +8139,6 @@ "url": "https://paulmillr.com/funding/" } }, - "node_modules/cacache": { - "version": "20.0.4", - "resolved": "https://registry.npmjs.org/cacache/-/cacache-20.0.4.tgz", - "integrity": "sha512-M3Lab8NPYlZU2exsL3bMVvMrMqgwCnMWfdZbK28bn3pK6APT/Te/I8hjRPNu1uwORY9a1eEQoifXbKPQMfMTOA==", - "license": "ISC", - "optional": true, - "dependencies": { - "@npmcli/fs": "^5.0.0", - "fs-minipass": "^3.0.0", - "glob": "^13.0.0", - "lru-cache": "^11.1.0", - "minipass": "^7.0.3", - "minipass-collect": "^2.0.1", - "minipass-flush": "^1.0.5", - "minipass-pipeline": "^1.2.4", - "p-map": "^7.0.2", - "ssri": "^13.0.0" - }, - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, - "node_modules/cacache/node_modules/balanced-match": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", - "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", - "license": "MIT", - "optional": true, - "engines": { - "node": "18 || 20 || >=22" - } - }, - "node_modules/cacache/node_modules/brace-expansion": { - "version": "5.0.5", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.5.tgz", - "integrity": "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==", - "license": "MIT", - "optional": true, - "dependencies": { - "balanced-match": "^4.0.2" - }, - "engines": { - "node": "18 || 20 || >=22" - } - }, - "node_modules/cacache/node_modules/glob": { - "version": "13.0.6", - "resolved": "https://registry.npmjs.org/glob/-/glob-13.0.6.tgz", - "integrity": "sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw==", - "license": "BlueOak-1.0.0", - "optional": true, - "dependencies": { - "minimatch": "^10.2.2", - "minipass": "^7.1.3", - "path-scurry": "^2.0.2" - }, - "engines": { - "node": "18 || 20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/cacache/node_modules/lru-cache": { - "version": "11.3.3", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.3.3.tgz", - "integrity": "sha512-JvNw9Y81y33E+BEYPr0U7omo+U9AySnsMsEiXgwT6yqd31VQWTLNQqmT4ou5eqPFUrTfIDFta2wKhB1hyohtAQ==", - "license": "BlueOak-1.0.0", - "optional": true, - "engines": { - "node": "20 || >=22" - } - }, - "node_modules/cacache/node_modules/minimatch": { - "version": "10.2.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", - "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", - "license": "BlueOak-1.0.0", - "optional": true, - "dependencies": { - "brace-expansion": "^5.0.5" - }, - "engines": { - "node": "18 || 20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/cacache/node_modules/p-map": { - "version": "7.0.4", - "resolved": "https://registry.npmjs.org/p-map/-/p-map-7.0.4.tgz", - "integrity": "sha512-tkAQEw8ysMzmkhgw8k+1U/iPhWNhykKnSk4Rd5zLoPJCuJaGRPo6YposrZgaxHKzDHdDWWZvE/Sk7hsL2X/CpQ==", - "license": "MIT", - "optional": true, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/cacache/node_modules/path-scurry": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.2.tgz", - "integrity": "sha512-3O/iVVsJAPsOnpwWIeD+d6z/7PmqApyQePUtCndjatj/9I5LylHvt5qluFaBT3I5h3r1ejfR056c+FCv+NnNXg==", - "license": "BlueOak-1.0.0", - "optional": true, - "dependencies": { - "lru-cache": "^11.0.0", - "minipass": "^7.1.2" - }, - "engines": { - "node": "18 || 20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/caching-transform": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/caching-transform/-/caching-transform-4.0.0.tgz", @@ -9255,21 +9048,6 @@ "node": ">=0.10.0" } }, - "node_modules/decompress-response": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", - "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", - "license": "MIT", - "dependencies": { - "mimic-response": "^3.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/deep-eql": { "version": "4.1.4", "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-4.1.4.tgz", @@ -9283,15 +9061,6 @@ "node": ">=6" } }, - "node_modules/deep-extend": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", - "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", - "license": "MIT", - "engines": { - "node": ">=4.0.0" - } - }, "node_modules/deep-is": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", @@ -9465,15 +9234,6 @@ "npm": "1.2.8000 || >= 1.4.16" } }, - "node_modules/detect-libc": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", - "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", - "license": "Apache-2.0", - "engines": { - "node": ">=8" - } - }, "node_modules/diff": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/diff/-/diff-7.0.0.tgz", @@ -9768,16 +9528,6 @@ "once": "^1.4.0" } }, - "node_modules/env-paths": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.1.tgz", - "integrity": "sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==", - "license": "MIT", - "optional": true, - "engines": { - "node": ">=6" - } - }, "node_modules/err-code": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/err-code/-/err-code-3.0.1.tgz", @@ -11146,22 +10896,6 @@ "safe-buffer": "^5.1.1" } }, - "node_modules/expand-template": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", - "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", - "license": "(MIT OR WTFPL)", - "engines": { - "node": ">=6" - } - }, - "node_modules/exponential-backoff": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/exponential-backoff/-/exponential-backoff-3.1.3.tgz", - "integrity": "sha512-ZgEeZXj30q+I0EN+CbSSpIyPaJ5HVQD18Z1m+u1FXbAeT94mr1zw50q4q6jiiC447Nl/YTcIYSAftiGqetwXCA==", - "license": "Apache-2.0", - "optional": true - }, "node_modules/express": { "version": "4.22.2", "resolved": "https://registry.npmjs.org/express/-/express-4.22.2.tgz", @@ -11439,12 +11173,6 @@ "moment": "^2.29.1" } }, - "node_modules/file-uri-to-path": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", - "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", - "license": "MIT" - }, "node_modules/fill-range": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", @@ -11734,19 +11462,6 @@ "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", "license": "MIT" }, - "node_modules/fs-minipass": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-3.0.3.tgz", - "integrity": "sha512-XUBA9XClHbnJWSfBzjkm6RvPsyg3sryZt06BEQoXcF7EK/xpGaQYJgQKDJSUH5SGZ76Y7pFx1QBnXz09rU5Fbw==", - "license": "ISC", - "optional": true, - "dependencies": { - "minipass": "^7.0.3" - }, - "engines": { - "node": "^14.17.0 || ^16.13.0 || >=18.0.0" - } - }, "node_modules/fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -11998,12 +11713,6 @@ "git-up": "^8.1.0" } }, - "node_modules/github-from-package": { - "version": "0.0.0", - "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", - "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==", - "license": "MIT" - }, "node_modules/glob": { "version": "10.5.0", "resolved": "https://registry.npmjs.org/glob/-/glob-10.5.0.tgz", @@ -12125,7 +11834,7 @@ "version": "4.2.11", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", - "devOptional": true, + "dev": true, "license": "ISC" }, "node_modules/graphemer": { @@ -12396,13 +12105,6 @@ "dev": true, "license": "MIT" }, - "node_modules/http-cache-semantics": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.2.0.tgz", - "integrity": "sha512-dTxcvPXqPvXBQpq5dUr6mEMJX4oIEFv6bwom3FDwKRDsuIjjJGANqhBuoAn9c1RQJIdAKav33ED65E2ys+87QQ==", - "license": "BSD-2-Clause", - "optional": true - }, "node_modules/http-errors": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", @@ -12423,34 +12125,6 @@ "url": "https://opencollective.com/express" } }, - "node_modules/http-proxy-agent": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", - "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", - "license": "MIT", - "optional": true, - "dependencies": { - "agent-base": "^7.1.0", - "debug": "^4.3.4" - }, - "engines": { - "node": ">= 14" - } - }, - "node_modules/https-proxy-agent": { - "version": "7.0.6", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", - "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", - "license": "MIT", - "optional": true, - "dependencies": { - "agent-base": "^7.1.2", - "debug": "4" - }, - "engines": { - "node": ">= 14" - } - }, "node_modules/humanhash": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/humanhash/-/humanhash-1.0.4.tgz", @@ -12622,12 +12296,6 @@ "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", "license": "ISC" }, - "node_modules/ini": { - "version": "1.3.8", - "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", - "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", - "license": "ISC" - }, "node_modules/interface-datastore": { "version": "9.0.2", "resolved": "https://registry.npmjs.org/interface-datastore/-/interface-datastore-9.0.2.tgz", @@ -12672,7 +12340,7 @@ "version": "10.2.0", "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.2.0.tgz", "integrity": "sha512-/+S6j4E9AHvW9SWMSEY9Xfy66O5PWvVEJ08O0y5JGyEKQpojb0K0GKpz/v5HJ/G0vi3D2sjGK78119oXZeE0qA==", - "devOptional": true, + "dev": true, "license": "MIT", "engines": { "node": ">= 12" @@ -14346,40 +14014,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/make-fetch-happen": { - "version": "15.0.5", - "resolved": "https://registry.npmjs.org/make-fetch-happen/-/make-fetch-happen-15.0.5.tgz", - "integrity": "sha512-uCbIa8jWWmQZt4dSnEStkVC6gdakiinAm4PiGsywIkguF0eWMdcjDz0ECYhUolFU3pFLOev9VNPCEygydXnddg==", - "license": "ISC", - "optional": true, - "dependencies": { - "@gar/promise-retry": "^1.0.0", - "@npmcli/agent": "^4.0.0", - "@npmcli/redact": "^4.0.0", - "cacache": "^20.0.1", - "http-cache-semantics": "^4.1.1", - "minipass": "^7.0.2", - "minipass-fetch": "^5.0.0", - "minipass-flush": "^1.0.5", - "minipass-pipeline": "^1.2.4", - "negotiator": "^1.0.0", - "proc-log": "^6.0.0", - "ssri": "^13.0.0" - }, - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, - "node_modules/make-fetch-happen/node_modules/negotiator": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.0.0.tgz", - "integrity": "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==", - "license": "MIT", - "optional": true, - "engines": { - "node": ">= 0.6" - } - }, "node_modules/math-intrinsics": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", @@ -14518,18 +14152,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/mimic-response": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", - "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/minimalistic-assert": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz", @@ -14562,6 +14184,7 @@ "version": "1.2.8", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "dev": true, "license": "MIT", "funding": { "url": "https://github.com/sponsors/ljharb" @@ -14576,119 +14199,6 @@ "node": ">=16 || 14 >=14.17" } }, - "node_modules/minipass-collect": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/minipass-collect/-/minipass-collect-2.0.1.tgz", - "integrity": "sha512-D7V8PO9oaz7PWGLbCACuI1qEOsq7UKfLotx/C0Aet43fCUB/wfQ7DYeq2oR/svFJGYDHPr38SHATeaj/ZoKHKw==", - "license": "ISC", - "optional": true, - "dependencies": { - "minipass": "^7.0.3" - }, - "engines": { - "node": ">=16 || 14 >=14.17" - } - }, - "node_modules/minipass-fetch": { - "version": "5.0.2", - "resolved": "https://registry.npmjs.org/minipass-fetch/-/minipass-fetch-5.0.2.tgz", - "integrity": "sha512-2d0q2a8eCi2IRg/IGubCNRJoYbA1+YPXAzQVRFmB45gdGZafyivnZ5YSEfo3JikbjGxOdntGFvBQGqaSMXlAFQ==", - "license": "MIT", - "optional": true, - "dependencies": { - "minipass": "^7.0.3", - "minipass-sized": "^2.0.0", - "minizlib": "^3.0.1" - }, - "engines": { - "node": "^20.17.0 || >=22.9.0" - }, - "optionalDependencies": { - "iconv-lite": "^0.7.2" - } - }, - "node_modules/minipass-fetch/node_modules/iconv-lite": { - "version": "0.7.2", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.2.tgz", - "integrity": "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==", - "license": "MIT", - "optional": true, - "dependencies": { - "safer-buffer": ">= 2.1.2 < 3.0.0" - }, - "engines": { - "node": ">=0.10.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/minipass-flush": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/minipass-flush/-/minipass-flush-1.0.7.tgz", - "integrity": "sha512-TbqTz9cUwWyHS2Dy89P3ocAGUGxKjjLuR9z8w4WUTGAVgEj17/4nhgo2Du56i0Fm3Pm30g4iA8Lcqctc76jCzA==", - "license": "BlueOak-1.0.0", - "optional": true, - "dependencies": { - "minipass": "^3.0.0" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/minipass-flush/node_modules/minipass": { - "version": "3.3.6", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", - "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", - "license": "ISC", - "optional": true, - "dependencies": { - "yallist": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/minipass-pipeline": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/minipass-pipeline/-/minipass-pipeline-1.2.4.tgz", - "integrity": "sha512-xuIq7cIOt09RPRJ19gdi4b+RiNvDFYe5JH+ggNvBqGqpQXcru3PcRmOZuHBKWK1Txf9+cQ+HMVN4d6z46LZP7A==", - "license": "ISC", - "optional": true, - "dependencies": { - "minipass": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/minipass-pipeline/node_modules/minipass": { - "version": "3.3.6", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", - "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", - "license": "ISC", - "optional": true, - "dependencies": { - "yallist": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/minipass-sized": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/minipass-sized/-/minipass-sized-2.0.0.tgz", - "integrity": "sha512-zSsHhto5BcUVM2m1LurnXY6M//cGhVaegT71OfOXoprxT6o780GZd792ea6FfrQkuU4usHZIUczAQMRUE2plzA==", - "license": "ISC", - "optional": true, - "dependencies": { - "minipass": "^7.1.2" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/minizlib": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.1.0.tgz", @@ -14901,12 +14411,6 @@ "node": "^18 || >=20" } }, - "node_modules/napi-build-utils": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-2.0.0.tgz", - "integrity": "sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==", - "license": "MIT" - }, "node_modules/napi-macros": { "version": "2.2.2", "resolved": "https://registry.npmjs.org/napi-macros/-/napi-macros-2.2.2.tgz", @@ -15008,18 +14512,6 @@ "url": "https://opencollective.com/express" } }, - "node_modules/node-abi": { - "version": "3.89.0", - "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.89.0.tgz", - "integrity": "sha512-6u9UwL0HlAl21+agMN3YAMXcKByMqwGx+pq+P76vii5f7hTPtKDp08/H9py6DY+cfDw7kQNTGEj/rly3IgbNQA==", - "license": "MIT", - "dependencies": { - "semver": "^7.3.5" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/node-addon-api": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-2.0.2.tgz", @@ -15123,31 +14615,6 @@ "node": ">= 6.13.0" } }, - "node_modules/node-gyp": { - "version": "12.2.0", - "resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-12.2.0.tgz", - "integrity": "sha512-q23WdzrQv48KozXlr0U1v9dwO/k59NHeSzn6loGcasyf0UnSrtzs8kRxM+mfwJSf0DkX0s43hcqgnSO4/VNthQ==", - "license": "MIT", - "optional": true, - "dependencies": { - "env-paths": "^2.2.0", - "exponential-backoff": "^3.1.1", - "graceful-fs": "^4.2.6", - "make-fetch-happen": "^15.0.0", - "nopt": "^9.0.0", - "proc-log": "^6.0.0", - "semver": "^7.3.5", - "tar": "^7.5.4", - "tinyglobby": "^0.2.12", - "which": "^6.0.0" - }, - "bin": { - "node-gyp": "bin/node-gyp.js" - }, - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, "node_modules/node-gyp-build": { "version": "4.8.4", "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.8.4.tgz", @@ -15159,32 +14626,6 @@ "node-gyp-build-test": "build-test.js" } }, - "node_modules/node-gyp/node_modules/isexe": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/isexe/-/isexe-4.0.0.tgz", - "integrity": "sha512-FFUtZMpoZ8RqHS3XeXEmHWLA4thH+ZxCv2lOiPIn1Xc7CxrqhWzNSDzD+/chS/zbYezmiwWLdQC09JdQKmthOw==", - "license": "BlueOak-1.0.0", - "optional": true, - "engines": { - "node": ">=20" - } - }, - "node_modules/node-gyp/node_modules/which": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/which/-/which-6.0.1.tgz", - "integrity": "sha512-oGLe46MIrCRqX7ytPUf66EAYvdeMIZYn3WaocqqKZAxrBpkqHfL/qvTyJ/bTk5+AqHCjXmrv3CEWgy368zhRUg==", - "license": "ISC", - "optional": true, - "dependencies": { - "isexe": "^4.0.0" - }, - "bin": { - "node-which": "bin/which.js" - }, - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, "node_modules/node-preload": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/node-preload/-/node-preload-0.2.1.tgz", @@ -15205,22 +14646,6 @@ "dev": true, "license": "MIT" }, - "node_modules/nopt": { - "version": "9.0.0", - "resolved": "https://registry.npmjs.org/nopt/-/nopt-9.0.0.tgz", - "integrity": "sha512-Zhq3a+yFKrYwSBluL4H9XP3m3y5uvQkB/09CwDruCiRmR/UJYnn9W4R48ry0uGC70aeTPKLynBtscP9efFFcPw==", - "license": "ISC", - "optional": true, - "dependencies": { - "abbrev": "^4.0.0" - }, - "bin": { - "nopt": "bin/nopt.js" - }, - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, "node_modules/null-prototype-object": { "version": "1.2.6", "resolved": "https://registry.npmjs.org/null-prototype-object/-/null-prototype-object-1.2.6.tgz", @@ -16452,33 +15877,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/prebuild-install": { - "version": "7.1.3", - "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz", - "integrity": "sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==", - "deprecated": "No longer maintained. Please contact the author of the relevant native addon; alternatives are available.", - "license": "MIT", - "dependencies": { - "detect-libc": "^2.0.0", - "expand-template": "^2.0.3", - "github-from-package": "0.0.0", - "minimist": "^1.2.3", - "mkdirp-classic": "^0.5.3", - "napi-build-utils": "^2.0.0", - "node-abi": "^3.3.0", - "pump": "^3.0.0", - "rc": "^1.2.7", - "simple-get": "^4.0.0", - "tar-fs": "^2.0.0", - "tunnel-agent": "^0.6.0" - }, - "bin": { - "prebuild-install": "bin.js" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", @@ -16533,16 +15931,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/proc-log": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/proc-log/-/proc-log-6.1.0.tgz", - "integrity": "sha512-iG+GYldRf2BQ0UDUAd6JQ/RwzaQy6mXmsk/IzlYyal4A4SNFw54MeH4/tLkF4I5WoWG9SQwuqWzS99jaFQHBuQ==", - "license": "ISC", - "optional": true, - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, "node_modules/process": { "version": "0.11.10", "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", @@ -16894,30 +16282,6 @@ "node": ">= 0.8" } }, - "node_modules/rc": { - "version": "1.2.8", - "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", - "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", - "license": "(BSD-2-Clause OR MIT OR Apache-2.0)", - "dependencies": { - "deep-extend": "^0.6.0", - "ini": "~1.3.0", - "minimist": "^1.2.0", - "strip-json-comments": "~2.0.1" - }, - "bin": { - "rc": "cli.js" - } - }, - "node_modules/rc/node_modules/strip-json-comments": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", - "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/rc9": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/rc9/-/rc9-2.1.2.tgz", @@ -17949,51 +17313,6 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/simple-concat": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", - "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT" - }, - "node_modules/simple-get": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz", - "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT", - "dependencies": { - "decompress-response": "^6.0.0", - "once": "^1.3.1", - "simple-concat": "^1.0.0" - } - }, "node_modules/sinon": { "version": "19.0.5", "resolved": "https://registry.npmjs.org/sinon/-/sinon-19.0.5.tgz", @@ -18027,7 +17346,7 @@ "version": "4.2.0", "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", - "devOptional": true, + "dev": true, "license": "MIT", "engines": { "node": ">= 6.0.0", @@ -18038,7 +17357,7 @@ "version": "2.8.7", "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.7.tgz", "integrity": "sha512-HLpt+uLy/pxB+bum/9DzAgiKS8CX1EvbWxI4zlmgGCExImLdiad2iCwXT5Z4c9c3Eq8rP2318mPW2c+QbtjK8A==", - "devOptional": true, + "dev": true, "license": "MIT", "dependencies": { "ip-address": "^10.0.1", @@ -18049,21 +17368,6 @@ "npm": ">= 3.0.0" } }, - "node_modules/socks-proxy-agent": { - "version": "8.0.5", - "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.5.tgz", - "integrity": "sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw==", - "license": "MIT", - "optional": true, - "dependencies": { - "agent-base": "^7.1.2", - "debug": "^4.3.4", - "socks": "^2.8.3" - }, - "engines": { - "node": ">= 14" - } - }, "node_modules/source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", @@ -18132,42 +17436,6 @@ "dev": true, "license": "BSD-3-Clause" }, - "node_modules/sqlite3": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/sqlite3/-/sqlite3-6.0.1.tgz", - "integrity": "sha512-X0czUUMG2tmSqJpEQa3tCuZSHKIx8PwM53vLZzKp/o6Rpy25fiVfjdbnZ988M8+O3ZWR1ih0K255VumCb3MAnQ==", - "hasInstallScript": true, - "license": "BSD-3-Clause", - "dependencies": { - "bindings": "^1.5.0", - "node-addon-api": "^8.0.0", - "prebuild-install": "^7.1.3", - "tar": "^7.5.10" - }, - "engines": { - "node": ">=20.17.0" - }, - "optionalDependencies": { - "node-gyp": "12.x" - }, - "peerDependencies": { - "node-gyp": "12.x" - }, - "peerDependenciesMeta": { - "node-gyp": { - "optional": true - } - } - }, - "node_modules/sqlite3/node_modules/node-addon-api": { - "version": "8.7.0", - "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.7.0.tgz", - "integrity": "sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA==", - "license": "MIT", - "engines": { - "node": "^18 || ^20 || >= 21" - } - }, "node_modules/ssh2": { "version": "1.17.0", "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.17.0.tgz", @@ -18192,19 +17460,6 @@ "license": "MIT", "optional": true }, - "node_modules/ssri": { - "version": "13.0.1", - "resolved": "https://registry.npmjs.org/ssri/-/ssri-13.0.1.tgz", - "integrity": "sha512-QUiRf1+u9wPTL/76GTYlKttDEBWV1ga9ZXW8BG6kfdeyyM8LGPix9gROyg9V2+P0xNyF3X2Go526xKFdMZrHSQ==", - "license": "ISC", - "optional": true, - "dependencies": { - "minipass": "^7.0.3" - }, - "engines": { - "node": "^20.17.0 || >=22.9.0" - } - }, "node_modules/stack-trace": { "version": "0.0.10", "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", @@ -18783,7 +18038,7 @@ "version": "0.2.15", "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.15.tgz", "integrity": "sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==", - "devOptional": true, + "dev": true, "license": "MIT", "dependencies": { "fdir": "^6.5.0", @@ -18800,7 +18055,7 @@ "version": "6.5.0", "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", - "devOptional": true, + "dev": true, "license": "MIT", "engines": { "node": ">=12.0.0" @@ -18818,7 +18073,7 @@ "version": "4.0.4", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", - "devOptional": true, + "dev": true, "license": "MIT", "engines": { "node": ">=12" @@ -18990,18 +18245,6 @@ "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", "license": "0BSD" }, - "node_modules/tunnel-agent": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", - "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", - "license": "Apache-2.0", - "dependencies": { - "safe-buffer": "^5.0.1" - }, - "engines": { - "node": "*" - } - }, "node_modules/tweetnacl": { "version": "0.14.5", "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", diff --git a/package.json b/package.json index e191bf0d0..0d268e2ca 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,9 @@ "license": "Apache-2.0", "type": "module", "main": "index.js", + "engines": { + "node": ">=22.13.0" + }, "bugs": { "url": "https://github.com/oceanprotocol/ocean-node/issues" }, @@ -32,6 +35,7 @@ "test:integration": "npm run build-tests && npm run mocha \"./dist/test/integration/**/*.test.js\"", "test:computeunit": "npm run build-tests && npm run mocha \"./dist/test/unit/compute.test.js\"", "test:computeintegration": "npm run build-tests && npm run mocha \"./dist/test/integration/compute.test.js\"", + "test:servicesintegration": "npm run build-tests && npm run mocha \"./dist/test/integration/services.test.js\"", "test:indexer": "npm run build-tests && npm run mocha \"./dist/test/integration/indexer.test.js\"", "test:integration:light": "npm run build-tests && npm run mocha-light \"./dist/test/integration/**/*.test.js\"", "test:unit:cover": "nyc --report-dir coverage/unit npm run test:unit", @@ -73,9 +77,8 @@ "@libp2p/upnp-nat": "^4.0.9", "@libp2p/websockets": "^10.1.2", "@multiformats/multiaddr": "^12.2.3", - "@oceanprotocol/contracts": "^2.7.0", + "@oceanprotocol/contracts": "^2.9.0", "@oceanprotocol/ddo-js": "^0.4.0", - "axios": "^1.15.0", "base58-js": "^2.0.0", "basic-ftp": "^5.3.1", "cors": "^2.8.5", @@ -96,7 +99,6 @@ "lodash": "^4.18.1", "lzma-purejs-requirejs": "^1.0.0", "node-cron": "^3.0.3", - "sqlite3": "^6.0.1", "stream-concat": "^1.0.0", "tar": "^7.5.16", "uint8arrays": "^4.0.6", diff --git a/scripts/ocean-node-quickstart.sh b/scripts/ocean-node-quickstart.sh index 5280c45f6..a67d08c4f 100755 --- a/scripts/ocean-node-quickstart.sh +++ b/scripts/ocean-node-quickstart.sh @@ -239,15 +239,27 @@ if [ -z "$DOCKER_COMPUTE_ENVIRONMENTS" ]; then export DOCKER_COMPUTE_ENVIRONMENTS='[ { "socketPath": "/var/run/docker.sock", + "resources": [ + { + "id": "disk", + "total": 10 + } + ], "environments": [ { "storageExpiry": 604800, "maxJobDuration": 36000, "minJobDuration": 60, "resources": [ + { + "id": "cpu" + }, + { + "id": "ram" + }, { "id": "disk", - "total": 10 + "max": 10 } ], "fees": { @@ -580,43 +592,48 @@ process_pci_line() { } | del(.. | select(. == null)) | del(.. | select(. == []))' } -# Function to get all GPUs in JSON array format +# Function to get all GPUs in JSON array format. +# Each physical GPU becomes its own resource entry (kind: "discrete", total: 1) +# with a single DeviceID — never multiple DeviceIDs in one resource. get_all_gpus_json() { ( get_nvidia_gpus get_generic_gpus ) | jq -s ' - group_by(.description) | map( + to_entries | map( { - id: (.[0].description | ascii_downcase | gsub("[^a-z0-9]"; "-") | gsub("-+"; "-") | sub("^-"; "") | sub("-$"; "")), - description: .[0].description, + id: ("gpu" + (.key | tostring)), + kind: "discrete", type: "gpu", - total: length, - driverVersion: (.[0].driverVersion // null), - memoryTotal: (.[0].memoryTotal // null), - platform: (if .[0].init.deviceRequests.Driver == "amdgpu" then "amd" else .[0].init.deviceRequests.Driver end), + total: 1, + description: .value.description, + driverVersion: .value.driverVersion, + memoryTotal: .value.memoryTotal, + platform: (if .value.init.deviceRequests.Driver == "amdgpu" then "amd" else .value.init.deviceRequests.Driver end), init: ( - if .[0].init.deviceRequests.Driver == "nvidia" then + if .value.init.deviceRequests.Driver == "nvidia" then { deviceRequests: { - Driver: .[0].init.deviceRequests.Driver, - DeviceIDs: (map(.init.deviceRequests.Devices[]?) | unique), + Driver: "nvidia", + DeviceIDs: .value.init.deviceRequests.Devices, Capabilities: [["gpu"]] } } else { - advanced: { - Driver: .[0].init.deviceRequests.Driver, - Devices: (map(.init.deviceRequests.Devices[]?) | unique), - Capabilities: [["gpu"]], - Binds: (map(.init.Binds[]?) | unique), - CapAdd: (map(.init.CapAdd[]?) | unique), - GroupAdd: (map(.init.GroupAdd[]?) | unique), - SecurityOpt: .[0].init.SecurityOpt, - ShmSize: .[0].init.ShmSize, - IpcMode: .[0].init.IpcMode - } | del(.. | select(. == null)) | del(.. | select(. == [])) + advanced: ( + { + Driver: .value.init.deviceRequests.Driver, + Devices: .value.init.deviceRequests.Devices, + Capabilities: [["gpu"]], + Binds: .value.init.Binds, + CapAdd: .value.init.CapAdd, + GroupAdd: .value.init.GroupAdd, + SecurityOpt: .value.init.SecurityOpt, + ShmSize: .value.init.ShmSize, + IpcMode: .value.init.IpcMode + } | del(.. | select(. == null)) | del(.. | select(. == [])) + ) } end ) @@ -632,8 +649,11 @@ if command -v jq &> /dev/null; then if [ "$GPU_COUNT" -gt 0 ]; then echo "Detected $GPU_COUNT GPU type(s). Updating configuration..." - DOCKER_COMPUTE_ENVIRONMENTS=$(echo "$DOCKER_COMPUTE_ENVIRONMENTS" | jq --argjson gpus "$DETECTED_GPUS" '.[0].environments[0].resources += $gpus') - echo "GPUs added to Compute Environment resources." + DOCKER_COMPUTE_ENVIRONMENTS=$(echo "$DOCKER_COMPUTE_ENVIRONMENTS" | jq --argjson gpus "$DETECTED_GPUS" ' + .[0].resources += $gpus | + .[0].environments[0].resources += ($gpus | map({ id: .id })) + ') + echo "GPUs added to connection-level resources; environment refs updated." else echo "No GPUs detected." fi diff --git a/src/@types/C2D/C2D.ts b/src/@types/C2D/C2D.ts index 9fe929356..74bd3e267 100644 --- a/src/@types/C2D/C2D.ts +++ b/src/@types/C2D/C2D.ts @@ -1,6 +1,14 @@ import { MetadataAlgorithm, ConsumerParameter } from '@oceanprotocol/ddo-js' import type { BaseFileObject, StorageObject, EncryptMethod } from '../fileObject.js' import type { AccessList } from '../AccessList.js' +import type { ServiceOnDemandConfig } from './ServiceOnDemand.js' + +// Per-environment capability flags. Both default to true at config-parse and +// at runtime construction; only an explicit false disables a capability. +export interface ComputeEnvFeatures { + computeJobs: boolean // false → COMPUTE_START + FREE_COMPUTE_START rejected + services: boolean // false → SERVICE_START rejected; env hidden from service matching +} export enum C2DClusterType { // eslint-disable-next-line no-unused-vars OPF_K8 = 0, @@ -22,11 +30,18 @@ export interface C2DClusterInfo { } export type ComputeResourceType = 'cpu' | 'ram' | 'disk' | any +export type ComputeResourceKind = 'discrete' | 'fungible' export interface ResourceConstraint { - id: ComputeResourceType // the resource being constrained - min?: number // min units of this resource per unit of parent resource - max?: number // max units of this resource per unit of parent resource + // Exactly one of `id` | `type` must be set. + id?: ComputeResourceType // exact single-resource target (e.g. 'ram', 'gpu0') + type?: string // group target: aggregate across ALL env resources whose `type` matches (e.g. 'gpu') + min?: number // min units of the constrained resource; per unit of parent when `perUnit` (default), absolute aggregate when `perUnit:false` + max?: number // max units of the constrained resource; per unit of parent when `perUnit` (default), absolute aggregate when `perUnit:false` + perUnit?: boolean // undefined/true = RATIO (parentAmount * min); false = FLOOR (absolute min/max, only enforced when parent > 0) + aggregate?: boolean // when true, this constraint's contribution SUMS with matching aggregate + // constraints on other requested parents into one shared target (single `id` only). e.g. two + // per-device GPUs each with {id:'cpu',min:1,max:4,aggregate:true} → cpu in [2,8] when both rented. } export interface ComputeResourcesPricingInfo { @@ -58,8 +73,15 @@ export interface ComputeResource { id: ComputeResourceType description?: string type?: string - kind?: string // discreet, named, etc + kind?: ComputeResourceKind // 'discrete' | 'fungible'. Auto-inferred if omitted. + shareable?: boolean // Only meaningful for kind:'discrete'. Default false. + // true → multiple jobs may share the device simultaneously (NIC, TPM, HSM) + // false → exclusive: only one job at a time (GPU, FPGA) total: number // total number of specific resource + cpuList?: string // connection-level cpu resource only: host core IDs jobs may be pinned to, + // as comma-separated core IDs and/or ranges, ascending and non-overlapping + // ("3", "0-1,3", "0-15,32-47"). Mutually exclusive with total; the effective + // total is the expanded list length. min: number // min number of resource needed for a job max: number // max number of resource for a job inUse?: number // for display purposes @@ -72,6 +94,15 @@ export interface ComputeResource { init?: dockerHwInit constraints?: ResourceConstraint[] // optional cross-resource constraints } +export interface EnvironmentResourceRef { + id: ComputeResourceType // must match a resource id in C2DDockerConfig.resources or auto-detected (cpu/ram/disk) + total?: number // env aggregate ceiling; if omitted → defaults to pool total (no per-env restriction) + min?: number // per-job minimum + max?: number // per-job maximum (capped to total if both present) + constraints?: ResourceConstraint[] // per-env override: replaces pool constraints entirely + // Omit to inherit pool constraints. Set [] to remove all constraints for this env. +} + export interface ComputeResourceRequest { id: string amount: number @@ -109,6 +140,19 @@ export interface ComputeEnvironmentFreeOptions { access: ComputeAccessList allowImageBuild?: boolean } + +// Config-time only — used in C2DEnvironmentConfig.free. +// resources are EnvironmentResourceRef[] (refs to pool) and resolved to ComputeResource[] at startup. +// Runtime free options live in ComputeEnvironmentFreeOptions (unchanged). +export interface C2DEnvironmentFreeConfig { + storageExpiry?: number + maxJobDuration?: number + minJobDuration?: number + maxJobs?: number + resources?: EnvironmentResourceRef[] + access?: ComputeAccessList + allowImageBuild?: boolean +} export interface ComputeEnvironmentBaseConfig { description?: string // v1 storageExpiry?: number // amount of seconds for storage @@ -121,6 +165,7 @@ export interface ComputeEnvironmentBaseConfig { free?: ComputeEnvironmentFreeOptions platform: RunningPlatform enableNetwork?: boolean // whether network is enabled for algorithm containers + features?: ComputeEnvFeatures // always populated at runtime construction; gates compute/service starts } export interface ComputeRuntimes { @@ -151,9 +196,10 @@ export interface C2DEnvironmentConfig { maxJobs?: number fees?: ComputeEnvFeesStructure access?: ComputeAccessList - free?: ComputeEnvironmentFreeOptions - resources?: ComputeResource[] + free?: C2DEnvironmentFreeConfig // config-time only; resolved to ComputeEnvironmentFreeOptions at startup + resources?: EnvironmentResourceRef[] // lightweight refs to connection pool enableNetwork?: boolean // whether network is enabled for algorithm containers + features?: ComputeEnvFeatures // config-time, optional } export interface C2DDockerConfig { @@ -169,7 +215,9 @@ export interface C2DDockerConfig { paymentClaimInterval?: number // Default: 3600 seconds (1 hours) scanImages?: boolean scanImageDBUpdateInterval?: number // Default: 12 hours + resources?: ComputeResource[] // optional: cpu/ram/disk auto-detected; include for GPUs/NICs or to cap auto-detected totals environments: C2DEnvironmentConfig[] + serviceOnDemand?: ServiceOnDemandConfig // per-daemon Service-on-Demand operational config } export type ComputeResultType = diff --git a/src/@types/C2D/ServiceOnDemand.ts b/src/@types/C2D/ServiceOnDemand.ts new file mode 100644 index 000000000..f859530b5 --- /dev/null +++ b/src/@types/C2D/ServiceOnDemand.ts @@ -0,0 +1,147 @@ +import type { DBComputeJobPayment, ComputeResourceRequestWithPrice } from './C2D.js' + +// ── Resource requirements ───────────────────────────────────────────── + +export interface TemplateResourceRequirement { + // Exactly one of `id` or `kind` must be set. + id?: string // exact resource id: 'cpu' | 'ram' | 'disk' | named GPU ('gpu-0') + kind?: 'discrete' | 'fungible' // match ANY resource of this kind across the env pool + type?: string // optional: further filter within kind ('gpu', 'fpga', 'tpu') + + min: number // MUST have at least this much — service is rejected otherwise + recommended?: number // ideal amount; below this the env gets a lower score + unit?: string // display hint: 'cores' | 'GB' | 'count' + description?: string // shown in UI: "CUDA GPU — 2 recommended for large models" +} + +// ── Template definition ─────────────────────────────────────────────── + +export interface UserConfigurableEnvVar { + key: string // env var name, passed in userData + validation?: string // optional regex; validated at SERVICE_START time + sensitive?: boolean // advisory hint for clients/UI (e.g. mask on input). The node receives ALL userData ECIES-encrypted, so this does not change node-side storage. +} + +export interface ServiceTemplate { + id: string // [a-z0-9][a-z0-9_-]{0,63} + name?: string + description?: string + // Image specification — exactly one of (tag | checksum | dockerfile) must be set: + image: string // base image name + tag?: string // e.g. "latest" — mutually exclusive with checksum/dockerfile + checksum?: string // digest: "sha256:<64 hex>" — mutually exclusive with tag/dockerfile + dockerfile?: string // inline Dockerfile content — triggers build; mutually exclusive with tag/checksum + additionalDockerFiles?: Record // filename → content; only valid with dockerfile + exposedPorts: number[] + envVars?: Record // fixed env vars — operator-set, never returned to callers + userConfigurableEnvVars?: UserConfigurableEnvVar[] + command?: string[] // Docker CMD override; ${KEY} expanded from userData + entrypoint?: string[] // Docker ENTRYPOINT override + requiredResources?: TemplateResourceRequirement[] // MUST satisfy — gates SERVICE_START + recommendedResources?: TemplateResourceRequirement[] // SHOULD satisfy — used for scoring + UI +} + +// ── Public / sanitized types ────────────────────────────────────────── + +// Safe to return in API responses: envVars values are stripped (keys only). Choosing a +// matching compute environment is the client's responsibility (see GET_COMPUTE_ENVIRONMENTS). +export interface ServiceTemplatePublic extends Omit { + envVarKeys?: string[] // keys of envVars only, never values +} + +// ── Operational config (per Docker daemon, not global) ──────────────── + +export interface ServiceOnDemandConfig { + enabled: boolean + nodeHost: string // host (or IP) clients use to reach forwarded service ports; e.g. 'localhost' + hostPortRange?: [number, number] // e.g. [30000, 32767]; specific to this daemon's host + maxDurationSeconds?: number // default: 86400 (24 h) + allowImageBuild?: boolean // default: false — gates Dockerfile-based services per daemon +} + +// ── Runtime service job ─────────────────────────────────────────────── + +export interface ServiceEndpoint { + containerPort: number + hostPort: number + url: string // e.g. "http://:31042" +} + +/* eslint-disable no-unused-vars */ +export enum ServiceStatusNumber { + Starting = 10, // DB record created by the start handler; awaits background processing + PullImage = 11, // pulling pre-built image from registry + PullImageFailed = 12, + BuildImage = 13, // building from Dockerfile + BuildImageFailed = 14, + VulnerableImage = 15, // Trivy scan found critical vulnerabilities + Locking = 20, // escrow createLock in progress (funds locked, not yet claimed) + Claiming = 30, // payment phase: claimLock on success, or cancelLock if the image step failed + Running = 40, + Restarting = 45, // SERVICE_RESTART accepted; teardown + re-pull/build + new container in progress + Stopping = 50, + Stopped = 70, + Expired = 75, + Error = 99 +} +/* eslint-enable no-unused-vars */ + +export const ServiceStatusText: Record = { + [ServiceStatusNumber.Starting]: 'Starting', + [ServiceStatusNumber.PullImage]: 'PullImage', + [ServiceStatusNumber.PullImageFailed]: 'PullImageFailed', + [ServiceStatusNumber.BuildImage]: 'BuildImage', + [ServiceStatusNumber.BuildImageFailed]: 'BuildImageFailed', + [ServiceStatusNumber.VulnerableImage]: 'VulnerableImage', + [ServiceStatusNumber.Locking]: 'Locking', + [ServiceStatusNumber.Claiming]: 'Claiming', + [ServiceStatusNumber.Running]: 'Running', + [ServiceStatusNumber.Restarting]: 'Restarting', + [ServiceStatusNumber.Stopping]: 'Stopping', + [ServiceStatusNumber.Stopped]: 'Stopped', + [ServiceStatusNumber.Expired]: 'Expired', + [ServiceStatusNumber.Error]: 'Error' +} + +// Statuses of a service job that is mid-start/restart and owned by an exclusive +// lifecycle operation. Single source of truth for getPendingServiceStarts (DB query) and +// the pipeline's staleness guard — the two MUST agree or a job can be picked up and then +// ignored (or vice versa). Restarting is included so a job orphaned by a crash +// mid-restart is recovered at boot exactly like a crash mid-start. +export const SERVICE_START_PENDING_STATUSES: readonly ServiceStatusNumber[] = [ + ServiceStatusNumber.Starting, + ServiceStatusNumber.Locking, + ServiceStatusNumber.PullImage, + ServiceStatusNumber.BuildImage, + ServiceStatusNumber.Claiming, + ServiceStatusNumber.Restarting +] + +export interface ServiceJob { + serviceId: string // unique id for a running service — distinct from a compute jobId + clusterHash: string + environment: string // envId the service runs on — used for shared resource accounting + pricing + owner: string // consumerAddress + image: string + tag?: string + checksum?: string + dockerfile?: string // inline Dockerfile (when built); kept so restart can rebuild + additionalDockerFiles?: Record // extra build-context files (only with dockerfile) + dockerCmd?: string[] // container CMD override + dockerEntrypoint?: string[] // container ENTRYPOINT override + containerImage: string // resolved final reference used by Docker (image:tag, image@digest, or built name) + containerId: string + networkId: string // per-service Docker network id + status: ServiceStatusNumber + statusText: string + dateCreated: string // ISO timestamp + updatedAt?: number + expiresAt: number // Unix ms timestamp + duration: number // requested seconds + exposedPorts: number[] + endpoints: ServiceEndpoint[] + userData?: string // ECIES(node key) string sent by the client; stored as-is, decrypted only at start/restart; never returned + resources: ComputeResourceRequestWithPrice[] + payment: DBComputeJobPayment // initial start payment + extendPayments?: DBComputeJobPayment[] // one entry per successful SERVICE_EXTEND +} diff --git a/src/@types/Escrow.ts b/src/@types/Escrow.ts index 20690f042..3b248a9ae 100644 --- a/src/@types/Escrow.ts +++ b/src/@types/Escrow.ts @@ -32,4 +32,7 @@ export interface EscrowEvent { maxLockedAmount?: string maxLockSeconds?: string maxLockCounts?: string + oldAmount?: string + newAmount?: string + newExpiry?: string } diff --git a/src/@types/OceanNode.ts b/src/@types/OceanNode.ts index 6717de5c6..1cb2a5f31 100644 --- a/src/@types/OceanNode.ts +++ b/src/@types/OceanNode.ts @@ -106,6 +106,7 @@ export interface dockerRegistrysAuth { export interface OceanNodeConfig { dockerComputeEnvironments: C2DDockerConfig[] + serviceTemplatesPath?: string // folder of *.json service templates; defaults to 'databases/serviceTemplates/' dockerRegistrysAuth: dockerRegistrysAuth authorizedDecrypters: string[] authorizedDecryptersList: AccessListContract | null diff --git a/src/@types/commands.ts b/src/@types/commands.ts index 335a0b827..21ae04436 100644 --- a/src/@types/commands.ts +++ b/src/@types/commands.ts @@ -398,3 +398,92 @@ export interface PersistentStorageDeleteFileCommand extends Command { bucketId: string fileName: string } + +// ── Service On Demand ───────────────────────────────────────────────── + +export interface ServiceGetTemplatesCommand extends Command { + chainId?: number +} + +export interface ServiceStartCommand extends Command { + consumerAddress: string + nonce: string + signature: string + environment: string // required: the envId to run the service on (from GET_COMPUTE_ENVIRONMENTS) + // Image spec — exactly one of tag/checksum/dockerfile. `image` is always required. + image: string // base image name (or build label when dockerfile is set) + tag?: string // pull by name:tag + checksum?: string // pull by digest: "sha256:<64 hex>" + dockerfile?: string // build from inline Dockerfile; requires allowImageBuild on the env + additionalDockerFiles?: Record // extra files in the build context + dockerCmd?: string[] // exact container command (Docker exec-form CMD override; no shell) + dockerEntrypoint?: string[] // container ENTRYPOINT override + exposedPorts?: number[] // container ports to forward + resources?: ComputeResourceRequest[] + duration: number // seconds; capped by serviceOnDemand.maxDurationSeconds + userData?: string // ECIES-encrypted (to the node's public key) JSON object → the container's env-var map + payment: { chainId: number; token: string } +} + +export interface ServiceStopCommand extends Command { + consumerAddress: string + nonce: string + signature: string + serviceId: string +} + +export interface ServiceGetStreamableLogsCommand extends Command { + consumerAddress: string + nonce: string + signature: string + serviceId: string + // Optional lower time bound for the returned logs: either a Unix timestamp in seconds + // (e.g. "1735689600"), or a relative duration counted back from now (e.g. "30s", "45m", + // "2h", "7d"). Omit to get the full history since container start, then follow live. + since?: string +} + +export interface ServiceGetStatusCommand extends Command { + consumerAddress: string + nonce: string + signature: string + serviceId?: string +} + +// Node-wide service listing (SERVICE_LIST), shaped like GetJobsCommand. Authenticated +// like every other service command (consumerAddress + signature/token), but NOT +// owner-scoped: it returns every consumer's services. Default (no filters): only the +// services currently holding a resource reservation. +export interface GetServicesCommand extends Command { + consumerAddress: string + nonce: string + signature: string + // filter to ONE specific status (any ServiceStatusNumber, incl. Expired); takes + // precedence over includeAllStatuses + status?: number + // return services in EVERY status instead of only the resource-holding set + includeAllStatuses?: boolean + // only services created at/after this moment: ISO date string, or a Unix timestamp + // (seconds or milliseconds) as a string + fromTimestamp?: string + updatedSince?: string +} + +export interface ServiceRestartCommand extends Command { + consumerAddress: string + nonce: string + signature: string + serviceId: string + userData?: string // optional ECIES-encrypted userData. If provided it REPLACES the stored userData (send the complete set). If omitted, the stored userData is reused — no re-supply needed. + dockerCmd?: string[] // optional. If provided (even []) it REPLACES the stored CMD override. If omitted, the stored dockerCmd is reused. + dockerEntrypoint?: string[] // optional. If provided (even []) it REPLACES the stored ENTRYPOINT override. If omitted, the stored dockerEntrypoint is reused. +} + +export interface ServiceExtendCommand extends Command { + consumerAddress: string + nonce: string + signature: string + serviceId: string + additionalDuration: number + payment: { chainId: number; token: string } +} diff --git a/src/components/Auth/index.ts b/src/components/Auth/index.ts index 6b671d6fd..2e4862254 100644 --- a/src/components/Auth/index.ts +++ b/src/components/Auth/index.ts @@ -1,9 +1,34 @@ import { AuthToken, AuthTokenDatabase } from '../database/AuthTokenDatabase.js' import jwt from 'jsonwebtoken' -import { checkNonce, NonceResponse } from '../core/utils/nonceHandler.js' +import { + checkNonce, + NonceResponse, + verifyConsumerSignature +} from '../core/utils/nonceHandler.js' import { OceanNode } from '../../OceanNode.js' +import { OceanP2P } from '../P2P/index.js' import { CommonValidation } from '../../utils/validators.js' import { OceanNodeConfig } from '../../@types/OceanNode.js' +import { PROTOCOL_COMMANDS } from '../../utils/constants.js' +import { peerIdFromString } from '@libp2p/peer-id' +import { Readable } from 'node:stream' + +const MAX_VERDICT_BYTES = 4096 + +async function readBounded(stream: Readable): Promise { + const chunks: Buffer[] = [] + let size = 0 + for await (const chunk of stream) { + size += chunk.length + if (size > MAX_VERDICT_BYTES) { + stream.destroy() + throw new Error('validation response too large') + } + chunks.push(Buffer.from(chunk)) + } + return Buffer.concat(chunks).toString() +} + export interface AuthValidation { token?: string address?: string @@ -16,10 +41,18 @@ export interface AuthValidation { export class Auth { private authTokenDatabase: AuthTokenDatabase private jwtSecret: string + private config: OceanNodeConfig + private getP2PNode: () => OceanP2P | undefined - public constructor(authTokenDatabase: AuthTokenDatabase, config: OceanNodeConfig) { + public constructor( + authTokenDatabase: AuthTokenDatabase, + config: OceanNodeConfig, + getP2PNode: () => OceanP2P | undefined = () => OceanNode.getInstance().getP2PNode() + ) { this.authTokenDatabase = authTokenDatabase this.jwtSecret = config.jwtSecret + this.config = config + this.getP2PNode = getP2PNode } public getJwtSecret(): string { @@ -27,12 +60,22 @@ export class Auth { } // eslint-disable-next-line require-await - async getJWTToken(address: string, nonce: string, createdAt: number): Promise { + async getJWTToken( + address: string, + nonce: string, + createdAt: number, + signature?: string, + issuerPeerId?: string, + chainId?: string | null + ): Promise { const jwtToken = jwt.sign( { address, nonce, - createdAt + createdAt, + signature, + issuerPeerId, + chainId }, this.getJwtSecret() ) @@ -62,10 +105,94 @@ export class Auth { async validateToken(token: string): Promise { const tokenEntry = await this.authTokenDatabase.validateToken(token) - if (!tokenEntry) { + if (tokenEntry) { + return tokenEntry + } + return await this.validateRemoteToken(token) + } + + async getLocalToken(token: string): Promise { + return await this.authTokenDatabase.validateToken(token) + } + + private async validateRemoteToken(token: string): Promise { + const decoded = jwt.decode(token) + if (!decoded || typeof decoded !== 'object') { + return null + } + const { address, nonce, signature, createdAt, issuerPeerId, chainId } = decoded as { + address?: string + nonce?: string + signature?: string + createdAt?: number + issuerPeerId?: string + chainId?: string | null + } + if ( + typeof address !== 'string' || + typeof nonce !== 'string' || + typeof signature !== 'string' || + typeof issuerPeerId !== 'string' + ) { + return null + } + const signatureValid = await verifyConsumerSignature( + address, + nonce, + signature, + issuerPeerId, + PROTOCOL_COMMANDS.CREATE_AUTH_TOKEN, + this.config, + chainId + ) + if (!signatureValid) { return null } - return tokenEntry + try { + peerIdFromString(issuerPeerId) + } catch { + return null + } + const p2p = this.getP2PNode() + if (!p2p || p2p.isTargetPeerSelf(issuerPeerId)) { + return null + } + let response + try { + response = await p2p.sendTo( + issuerPeerId, + JSON.stringify({ command: PROTOCOL_COMMANDS.VALIDATE_AUTH_TOKEN, token }) + ) + } catch { + return null + } + if (response?.status?.httpStatus !== 200 || !response.stream) { + return null + } + let verdict: { valid?: boolean; validUntil?: number | string | null } + try { + verdict = JSON.parse(await readBounded(response.stream as Readable)) + } catch { + return null + } + if (!verdict.valid) { + return null + } + if (verdict.validUntil == null) { + return null + } + const validUntilMs = Number(verdict.validUntil) + if (!Number.isFinite(validUntilMs) || validUntilMs <= Date.now()) { + return null + } + const createdNum = Number(createdAt) + return { + token, + address, + created: Number.isFinite(createdNum) ? new Date(createdNum) : new Date(), + validUntil: new Date(validUntilMs), + isValid: true + } } /** @@ -79,7 +206,7 @@ export class Auth { */ async validateAuthenticationOrToken( authValidation: AuthValidation - ): Promise { + ): Promise { const { token, address, nonce, signature, command, chainId } = authValidation try { if (signature && address && nonce) { @@ -99,14 +226,14 @@ export class Auth { } if (nonceCheckResult.valid) { - return { valid: true, error: '' } + return { valid: true, error: '', address } } } if (token) { const authToken = await this.validateToken(token) if (authToken) { - return { valid: true, error: '' } + return { valid: true, error: '', address: authToken.address } } return { valid: false, error: 'Invalid token' } diff --git a/src/components/Indexer/processor.ts b/src/components/Indexer/processor.ts index 171d6750a..051008351 100644 --- a/src/components/Indexer/processor.ts +++ b/src/components/Indexer/processor.ts @@ -46,6 +46,7 @@ const EVENT_PROCESSOR_MAP: Record = { [EVENTS.ADDRESS_REMOVED]: AddressRemovedEventProcessor, [EVENTS.ESCROW_AUTH]: EscrowEventProcessor, [EVENTS.ESCROW_LOCK]: EscrowEventProcessor, + [EVENTS.ESCROW_RELOCK]: EscrowEventProcessor, [EVENTS.ESCROW_CLAIMED]: EscrowEventProcessor, [EVENTS.ESCROW_CANCELED]: EscrowEventProcessor, [EVENTS.ESCROW_DEPOSIT]: EscrowEventProcessor, diff --git a/src/components/Indexer/processors/BaseProcessor.ts b/src/components/Indexer/processors/BaseProcessor.ts index ad6956256..cb8476121 100644 --- a/src/components/Indexer/processors/BaseProcessor.ts +++ b/src/components/Indexer/processors/BaseProcessor.ts @@ -1,5 +1,4 @@ import { VersionedDDO, DeprecatedDDO } from '@oceanprotocol/ddo-js' -import axios from 'axios' import { ZeroAddress, Signer, @@ -222,12 +221,13 @@ export abstract class BaseEventProcessor { INDEXER_LOGGER.logMessage( `decryptDDO: Making HTTP request for nonce. DecryptorURL: ${decryptorURL}` ) - const nonceResponse = await axios.get( + const nonceResponse = await fetch( `${decryptorURL}/api/services/nonce?userAddress=${address}`, - { timeout: 20000 } + { signal: AbortSignal.timeout(20000) } ) - return nonceResponse.status === 200 && nonceResponse.data - ? String(parseInt(nonceResponse.data.nonce) + 1) + const nonceData = nonceResponse.status === 200 ? await nonceResponse.json() : null + return nonceData?.nonce !== undefined + ? String(parseInt(nonceData.nonce) + 1) : Date.now().toString() } else { return Date.now().toString() @@ -287,17 +287,32 @@ export abstract class BaseEventProcessor { nonce } try { - const res = await axios({ - method: 'post', - url: `${decryptorURL}/api/services/decrypt`, - data: payload, - timeout: 30000, - validateStatus: (status) => { - return ( - (status >= 200 && status < 300) || status === 400 || status === 403 - ) - } + // fetch never throws on HTTP status; the manual status checks below + // reproduce the previous axios validateStatus contract (2xx/400/403 + // handled here, everything else throws and is retried by withRetrial). + const fetchRes = await fetch(`${decryptorURL}/api/services/decrypt`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(payload), + signal: AbortSignal.timeout(30000) }) + // axios's default transformResponse parsed JSON regardless of the + // content-type, falling back to the raw string. The decrypt endpoint + // returns the DDO as JSON but with a text/plain content-type, so a + // content-type gate would wrongly keep it a string and downstream + // create256Hash would receive an object. Replicate axios here. + const rawBody = await fetchRes.text() + let parsedBody: any = rawBody + try { + parsedBody = JSON.parse(rawBody) + } catch { + // not JSON — keep the raw string, same as axios did + } + const res = { + status: fetchRes.status, + statusText: fetchRes.statusText, + data: parsedBody + } INDEXER_LOGGER.log( LOG_LEVELS_STR.LEVEL_INFO, @@ -320,9 +335,11 @@ export abstract class BaseEventProcessor { } return res } catch (err: any) { - // Retry ONLY on ECONNREFUSED + // Retry ONLY on ECONNREFUSED. fetch wraps the OS error in + // `err.cause`, so check both there and on the error itself. if ( err.code === 'ECONNREFUSED' || + err.cause?.code === 'ECONNREFUSED' || (err.message && err.message.includes('ECONNREFUSED')) ) { INDEXER_LOGGER.log( diff --git a/src/components/Indexer/processors/EscrowEventProcessor.ts b/src/components/Indexer/processors/EscrowEventProcessor.ts index 45108583b..3c5392232 100644 --- a/src/components/Indexer/processors/EscrowEventProcessor.ts +++ b/src/components/Indexer/processors/EscrowEventProcessor.ts @@ -57,6 +57,7 @@ export class EscrowEventProcessor extends BaseEventProcessor { case EVENTS.ESCROW_AUTH: record.payer = addr(args.payer) record.payee = addr(args.payee) + record.token = addr(args.token) record.maxLockedAmount = num(args.maxLockedAmount) record.maxLockSeconds = num(args.maxLockSeconds) record.maxLockCounts = num(args.maxLockCounts) @@ -69,6 +70,15 @@ export class EscrowEventProcessor extends BaseEventProcessor { record.expiry = num(args.expiry) record.token = addr(args.token) break + case EVENTS.ESCROW_RELOCK: + record.payer = addr(args.payer) + record.payee = addr(args.payee) + record.jobId = num(args.jobId) + record.oldAmount = num(args.oldAmount) + record.newAmount = num(args.newAmount) + record.newExpiry = num(args.newExpiry) + record.token = addr(args.token) + break case EVENTS.ESCROW_CLAIMED: record.payee = addr(args.payee) record.jobId = num(args.jobId) diff --git a/src/components/Indexer/purgatory.ts b/src/components/Indexer/purgatory.ts index 9ee8c6384..9c0af1e9a 100644 --- a/src/components/Indexer/purgatory.ts +++ b/src/components/Indexer/purgatory.ts @@ -1,4 +1,3 @@ -import axios from 'axios' import { PurgatoryAccounts, PurgatoryAssets } from '../../@types/Purgatory.js' import { INDEXER_LOGGER } from '../../utils/logging/common.js' import { LOG_LEVELS_STR } from '../../utils/logging/Logger.js' @@ -43,16 +42,15 @@ export class Purgatory { const purgatoryAssets: Array = [] if (this.isAssetsPurgatoryEnabled()) { try { - const response = await axios({ - method: 'get', - url: this.assetPurgatoryUrl, - timeout: 2000 + const response = await fetch(this.assetPurgatoryUrl, { + method: 'GET', + signal: AbortSignal.timeout(2000) }) if (response.status !== 200) { INDEXER_LOGGER.log( LOG_LEVELS_STR.LEVEL_ERROR, `PURGATORY: Failure when retrieving new purgatory list from ASSET_PURGATORY_URL env var. - Response: ${response.data}, status: ${ + Response: ${await response.text()}, status: ${ response.status + response.statusText }`, true @@ -63,7 +61,8 @@ export class Purgatory { `PURGATORY: Successfully retrieved new purgatory list from ASSET_PURGATORY_URL env var.` ) - for (const asset of response.data) { + const assets = await response.json() + for (const asset of assets) { if (asset && 'did' in asset) { purgatoryAssets.push({ did: asset.did, reason: asset.reason }) } @@ -86,16 +85,15 @@ export class Purgatory { const purgatoryAccounts: Array = [] if (this.isAccountsPurgatoryEnabled()) { try { - const response = await axios({ - method: 'get', - url: this.accountPurgatoryUrl, - timeout: 2000 // small increase + const response = await fetch(this.accountPurgatoryUrl, { + method: 'GET', + signal: AbortSignal.timeout(2000) // small increase }) if (response.status !== 200) { INDEXER_LOGGER.log( LOG_LEVELS_STR.LEVEL_ERROR, `PURGATORY: Failure when retrieving new purgatory list from ACCOUNT_PURGATORY_URL env var. - Response: ${response.data}, status: ${ + Response: ${await response.text()}, status: ${ response.status + response.statusText }`, true @@ -105,7 +103,8 @@ export class Purgatory { INDEXER_LOGGER.logMessage( `PURGATORY: Successfully retrieved new purgatory list from ACCOUNT_PURGATORY_URL env var.` ) - for (const account of response.data) { + const accounts = await response.json() + for (const account of accounts) { if (account && 'address' in account) { purgatoryAccounts.push({ address: account.address, reason: account.reason }) } diff --git a/src/components/c2d/compute_engine_base.ts b/src/components/c2d/compute_engine_base.ts index 361599798..0c09dc84d 100644 --- a/src/components/c2d/compute_engine_base.ts +++ b/src/components/c2d/compute_engine_base.ts @@ -16,6 +16,7 @@ import type { DBComputeJobMetadata, ComputeEnvFees } from '../../@types/C2D/C2D.js' +import type { ServiceJob } from '../../@types/C2D/ServiceOnDemand.js' import { C2DClusterType } from '../../@types/C2D/C2D.js' import { C2DDatabase } from '../database/C2DDatabase.js' import { Escrow } from '../core/utils/escrow.js' @@ -79,6 +80,83 @@ export abstract class C2DEngine { return null } + // ── Service on Demand (Docker-only for Stage 1; concrete no-ops here) ── + // Persists the initial Starting record and returns immediately. The heavy lifting + // (escrow lock/claim, image pull/build, container start) is done asynchronously by + // processServiceStart(), driven by the engine's background loop. + // eslint-disable-next-line @typescript-eslint/no-unused-vars, require-await + public async createServiceJob( + environment: string, + image: string, + tag: string | undefined, + checksum: string | undefined, + dockerfile: string | undefined, + additionalDockerFiles: Record | undefined, + dockerCmd: string[] | undefined, + dockerEntrypoint: string[] | undefined, + exposedPorts: number[], + resources: ComputeResourceRequest[], + duration: number, + owner: string, + payment: DBComputeJobPayment, + serviceId: string, + userData?: string // ECIES-encrypted; the engine decrypts it transiently into the container env + ): Promise { + return null + } + + // Background pipeline that advances a Starting service job through locking → image → + // payment → container → Running. Never throws (terminal failures are persisted as status). + // eslint-disable-next-line @typescript-eslint/no-unused-vars, require-await + public async processServiceStart(job: ServiceJob): Promise {} + + // onlyIfExpired: expiry-sweep mode — re-validate expiresAt on the fresh row under the + // lifecycle lock and skip the teardown when the service was extended in the meantime. + // eslint-disable-next-line @typescript-eslint/no-unused-vars, require-await + public async stopService( + serviceId: string, + owner: string, + onlyIfExpired?: boolean + ): Promise { + return null + } + + // Runs fn serialized with the engine's per-service lifecycle operations (start + // pipeline, restart, stop, expiry sweep). Engines without a lock implementation run + // fn directly; C2DEngineDocker overrides this with its lifecycle lock + DB lease. + // eslint-disable-next-line @typescript-eslint/no-unused-vars + public async runExclusive(serviceId: string, fn: () => Promise): Promise { + return await fn() + } + + // eslint-disable-next-line @typescript-eslint/no-unused-vars, require-await + public async restartService( + serviceId: string, + owner: string, + newUserData?: string, + newDockerCmd?: string[], + newDockerEntrypoint?: string[] + ): Promise { + return null + } + + // eslint-disable-next-line @typescript-eslint/no-unused-vars, require-await + public async getServiceStatus( + consumerAddress?: string, + serviceId?: string + ): Promise { + return [] + } + + // eslint-disable-next-line @typescript-eslint/no-unused-vars, require-await + public async getServiceStreamableLogs( + serviceId: string, + owner: string, + since?: number + ): Promise { + return null + } + // eslint-disable-next-line require-await public abstract checkDockerImage( image: string, @@ -263,6 +341,24 @@ export abstract class C2DEngine { resources: ComputeResourceRequest[], env: ComputeEnvironment, isFree: boolean + ): void { + // Two-phase on purpose: a later parent's min bump can raise a target past an earlier + // parent's already-checked max (e.g. gpu1 {cpu min:8} bumping past gpu0 {cpu max:6}), + // so all floors settle first (direct, then aggregate) and every ceiling is validated + // afterwards against the final amounts — the outcome no longer depends on the order + // resources appear in the env definition. + this.enforceDirectConstraints(resources, env, isFree, 'min') + this.applyAggregateConstraints(resources, env, isFree) + this.enforceDirectConstraints(resources, env, isFree, 'max') + } + + // Non-aggregate constraints, one phase at a time: 'min' raises targets to their floors + // (throwing when a floor exceeds the target's own max), 'max' rejects ceilings violations. + protected enforceDirectConstraints( + resources: ComputeResourceRequest[], + env: ComputeEnvironment, + isFree: boolean, + phase: 'min' | 'max' ): void { const envResources = isFree ? (env.free?.resources ?? []) : (env.resources ?? []) for (const envResource of envResources) { @@ -271,29 +367,53 @@ export abstract class C2DEngine { if (!parentAmount || parentAmount <= 0) continue for (const constraint of envResource.constraints) { - let constrainedAmount = this.getResourceRequest(resources, constraint.id) ?? 0 - - if (constraint.min !== undefined) { - const requiredMin = parentAmount * constraint.min + // Aggregate constraints sum across parents — handled in applyAggregateConstraints. + if (constraint.aggregate) continue + // perUnit (default true) => RATIO (parentAmount * value); false => absolute FLOOR/ceiling. + const perUnit = constraint.perUnit !== false + // A constraint targets either a single resource (`id`) or a group (`type`, aggregated). + const isGroup = constraint.type !== undefined + const targetIds = isGroup + ? this.getGroupResourceIds(env, constraint.type!, isFree) + : [constraint.id as string] + const targetLabel = isGroup + ? `${constraint.type} resources` + : String(constraint.id) + const constrainedAmount = isGroup + ? this.getGroupRequestedAmount(resources, targetIds) + : (this.getResourceRequest(resources, constraint.id) ?? 0) + + if (phase === 'min' && constraint.min !== undefined) { + const requiredMin = perUnit ? parentAmount * constraint.min : constraint.min if (constrainedAmount < requiredMin) { - const constrainedMaxMin = this.getMaxMinResource(constraint.id, env, isFree) - if (requiredMin > constrainedMaxMin.max) { + // Aggregate per-job ceiling of the target (single max, or sum of group maxes). + const targetMax = isGroup + ? this.getGroupMax(env, targetIds, isFree) + : this.getMaxMinResource(constraint.id, env, isFree).max + if (requiredMin > targetMax) { throw new Error( - `Cannot satisfy constraint: ${parentAmount} ${envResource.id} requires at least ${requiredMin} ${constraint.id}, but max is ${constrainedMaxMin.max}` + `Cannot satisfy constraint: ${parentAmount} ${envResource.id} requires at least ${requiredMin} ${targetLabel}, but max is ${targetMax}` + ) + } + if (isGroup) { + this.bumpGroupToFloor( + resources, + env, + targetIds, + requiredMin - constrainedAmount, + isFree ) + } else { + this.setResourceAmount(resources, constraint.id, requiredMin) } - this.setResourceAmount(resources, constraint.id, requiredMin) - constrainedAmount = requiredMin } } - if (constraint.max !== undefined) { - const requiredMax = parentAmount * constraint.max - // re-read in case it was bumped above - constrainedAmount = this.getResourceRequest(resources, constraint.id) ?? 0 + if (phase === 'max' && constraint.max !== undefined) { + const requiredMax = perUnit ? parentAmount * constraint.max : constraint.max if (constrainedAmount > requiredMax) { throw new Error( - `Too much ${constraint.id} for ${parentAmount} ${envResource.id}. Max allowed: ${requiredMax}, requested: ${constrainedAmount}` + `Too much ${targetLabel} for ${parentAmount} ${envResource.id}. Max allowed: ${requiredMax}, requested: ${constrainedAmount}` ) } } @@ -301,6 +421,69 @@ export abstract class C2DEngine { } } + // Aggregate constraints (`aggregate: true`) accumulate their per-parent contribution + // ADDITIVELY into a shared single-`id` target, summed across every requested parent that + // carries a matching aggregate constraint. This is how per-device GPUs (GPU1, GPU2, each + // `max:1`) can jointly require e.g. cpu min = Σ(gpuAmount × min) and cpu max = Σ(gpuAmount × + // max) — 2 GPUs with a {min:1,max:4} per-GPU rule → cpu in [2, 8]. Non-aggregate constraints + // keep their independent per-parent behavior (handled in enforceDirectConstraints); this runs + // between its min and max phases and only ever raises the target's floor. + protected applyAggregateConstraints( + resources: ComputeResourceRequest[], + env: ComputeEnvironment, + isFree: boolean + ): void { + const envResources = isFree ? (env.free?.resources ?? []) : (env.resources ?? []) + const summedMin = new Map() + const summedMax = new Map() + const hasMax = new Set() + + for (const parent of envResources) { + if (!parent.constraints || parent.constraints.length === 0) continue + const parentAmount = this.getResourceRequest(resources, parent.id) ?? 0 + if (parentAmount <= 0) continue + for (const c of parent.constraints) { + // aggregate targets a single resource id (validated by schema) + if (!c.aggregate || c.id === undefined) continue + const perUnit = c.perUnit !== false + if (c.min !== undefined) { + const contrib = perUnit ? parentAmount * c.min : c.min + summedMin.set(c.id, (summedMin.get(c.id) ?? 0) + contrib) + } + if (c.max !== undefined) { + const contrib = perUnit ? parentAmount * c.max : c.max + summedMax.set(c.id, (summedMax.get(c.id) ?? 0) + contrib) + hasMax.add(c.id) + } + } + } + + // min: raise each target up to its summed floor (respecting the target's own max) + for (const [targetId, requiredMin] of summedMin) { + const current = this.getResourceRequest(resources, targetId) ?? 0 + if (current < requiredMin) { + const targetMax = this.getMaxMinResource(targetId, env, isFree).max + if (requiredMin > targetMax) { + throw new Error( + `Cannot satisfy aggregate constraint: requires at least ${requiredMin} ${targetId}, but max is ${targetMax}` + ) + } + this.setResourceAmount(resources, targetId, requiredMin) + } + } + + // max: reject if a target exceeds its summed ceiling (never auto-reduced) + for (const targetId of hasMax) { + const current = this.getResourceRequest(resources, targetId) ?? 0 + const requiredMax = summedMax.get(targetId) + if (current > requiredMax) { + throw new Error( + `Too much ${targetId} for the requested resources. Max allowed: ${requiredMax}, requested: ${current}` + ) + } + } + } + protected setResourceAmount( resources: ComputeResourceRequest[], id: ComputeResourceType, @@ -314,6 +497,70 @@ export abstract class C2DEngine { } } + // Returns the ids of every resource active in this env (paid or free list) whose `type` + // matches — the concrete members a `type` group constraint aggregates over. + protected getGroupResourceIds( + env: ComputeEnvironment, + type: string, + isFree: boolean + ): string[] { + const envResources = isFree ? (env.free?.resources ?? []) : (env.resources ?? []) + return envResources.filter((r) => r.type === type).map((r) => r.id) + } + + // Sum of requested amounts across a set of resource ids (missing => 0). + protected getGroupRequestedAmount( + resources: ComputeResourceRequest[], + ids: string[] + ): number { + let total = 0 + for (const id of ids) total += this.getResourceRequest(resources, id) ?? 0 + return total + } + + // Aggregate per-job ceiling of a group: sum of each member's max. + protected getGroupMax(env: ComputeEnvironment, ids: string[], isFree: boolean): number { + let total = 0 + for (const id of ids) total += this.getMaxMinResource(id, env, isFree).max + return total + } + + // Raise members of a `type` group until their combined requested amount grows by `deficit` + // units. Prefers members with the most availability (lowest inUse), then config order, and + // never exceeds any member's own max. Relies on checkAndFillMissingResources' pre-fill loop + // having already inserted an entry for every declared resource, so setResourceAmount always + // finds its target. inUse here is a point-in-time hint to reduce false deferrals; the + // authoritative availability gate is checkIfResourcesAreAvailable. + // NOTE: inUse is read from env.resources (not env.free.resources) on purpose — group + // constraints target discrete resources (GPUs), whose inUse there is the GLOBAL count + // (paid + free, across all envs), i.e. the binding availability signal. The free list holds + // free-only usage and would under-count paid usage of the same physical device. + protected bumpGroupToFloor( + resources: ComputeResourceRequest[], + env: ComputeEnvironment, + ids: string[], + deficit: number, + isFree: boolean + ): void { + let remaining = deficit + const candidates = ids + .map((id) => { + const current = this.getResourceRequest(resources, id) ?? 0 + const { max } = this.getMaxMinResource(id, env, isFree) + const inUse = this.getResource(env.resources, id)?.inUse ?? 0 + return { id, current, headroom: max - current, inUse } + }) + .filter((c) => c.headroom > 0) + .sort((a, b) => a.inUse - b.inUse) + + for (const c of candidates) { + if (remaining <= 0) break + const bump = Math.min(c.headroom, remaining) + this.setResourceAmount(resources, c.id, c.current + bump) + remaining -= bump + } + } + public async getUsedResources(env: ComputeEnvironment): Promise { const usedResources: { [x: string]: any } = {} const usedFreeResources: { [x: string]: any } = {} @@ -364,10 +611,10 @@ export abstract class C2DEngine { for (const resource of job.resources) { const envRes = envResourceMap.get(resource.id) if (envRes) { - // GPUs are shared-exclusive: inUse tracked globally across all envs - // Everything else (cpu, ram, disk) is per-env exclusive - const isSharedExclusive = envRes.type === 'gpu' - if (!isSharedExclusive && !isThisEnv) continue + // discrete resources (GPUs, FPGAs, NICs) tracked globally across all envs + // fungible resources (cpu, ram, disk) are per-env exclusive + const isGloballyTracked = envRes.kind === 'discrete' + if (!isGloballyTracked && !isThisEnv) continue if (!(resource.id in usedResources)) usedResources[resource.id] = 0 usedResources[resource.id] += resource.amount if (job.isFree) { @@ -378,6 +625,32 @@ export abstract class C2DEngine { } } } + + // Fold in on-demand services: they share the same physical resource pool as + // compute jobs, so a running service must occupy resources too. Services are + // always paid (no free tier) and always "running" while in the DB's running set, + // so we only tally their resources — job-slot/queue metrics stay compute-only. + // Do NOT swallow this failure: getUsedResources feeds the strict resource-availability + // gate (checkIfResourcesAreAvailable). Under-counting running services would let the + // engine overcommit shared GPU/CPU/RAM. Let it propagate so the allocation path defers + // the job (the caller already wraps getComputeEnvironments in try/catch) rather than + // proceeding with missing service data. + const serviceJobs: ServiceJob[] = await this.db.getRunningServiceJobs( + this.getC2DConfig().hash + ) + for (const svc of serviceJobs) { + const isThisEnv = svc.environment === env.id + for (const resource of svc.resources) { + const envRes = envResourceMap.get(resource.id) + if (!envRes) continue + // discrete resources (GPUs, FPGAs, NICs) tracked globally across all envs; + // fungible resources (cpu, ram, disk) are per-env exclusive. + const isGloballyTracked = envRes.kind === 'discrete' + if (!isGloballyTracked && !isThisEnv) continue + if (!(resource.id in usedResources)) usedResources[resource.id] = 0 + usedResources[resource.id] += resource.amount + } + } return { totalJobs, totalFreeJobs, @@ -401,13 +674,21 @@ export abstract class C2DEngine { ) { let globalUsed = 0 let globalTotal = 0 + let discreteInUse: number | undefined for (const e of allEnvironments) { const res = this.getResource(e.resources, resourceId) if (res) { globalTotal += res.total || 0 - globalUsed += res.inUse || 0 + if (res.kind === 'discrete') { + // getUsedResources already aggregates discrete inUse globally across all envs, + // so each env carries the same global value — take the max to avoid N-fold counting. + discreteInUse = Math.max(discreteInUse ?? 0, res.inUse || 0) + } else { + globalUsed += res.inUse || 0 + } } } + if (discreteInUse !== undefined) globalUsed += discreteInUse const physicalLimit = this.physicalLimits.get(resourceId) if (physicalLimit !== undefined && globalTotal > physicalLimit) { globalTotal = physicalLimit @@ -434,12 +715,19 @@ export abstract class C2DEngine { for (const request of activeResources) { let envResource = this.getResource(env.resources, request.id) if (!envResource) throw new Error(`No such resource ${request.id}`) - if (envResource.total - envResource.inUse < request.amount) - throw new Error(`Not enough available ${request.id}`) - // Global check for non-GPU resources (cpu, ram, disk are per-env exclusive) - // GPUs are shared-exclusive so their inUse already reflects global usage - if (allEnvironments && envResource.type !== 'gpu') { + const isFungible = envResource.kind === 'fungible' + const isShareableDiscrete = + envResource.kind === 'discrete' && envResource.shareable === true + + // Gate 1 (per-env ceiling) — fungible resources only. + // envResource.total = env aggregate ceiling (from EnvironmentResourceRef.total). + if (isFungible && envResource.total - (envResource.inUse ?? 0) < request.amount) + throw new Error(`Not enough available ${request.id} in this environment`) + + // Gate 2 (engine-wide pool ceiling) — fungible + exclusive discrete. + // shareable discrete: tracked for visibility but never blocks allocation. + if (!isShareableDiscrete && allEnvironments) { this.checkGlobalResourceAvailability(allEnvironments, request.id, request.amount) } diff --git a/src/components/c2d/compute_engine_docker.ts b/src/components/c2d/compute_engine_docker.ts index 1cba837e6..b930751e8 100755 --- a/src/components/c2d/compute_engine_docker.ts +++ b/src/components/c2d/compute_engine_docker.ts @@ -1,6 +1,5 @@ /* eslint-disable security/detect-non-literal-fs-filename */ import { Readable, PassThrough } from 'stream' -import os from 'os' import path from 'path' import { C2DStatusNumber, @@ -22,8 +21,10 @@ import type { ComputeResourceRequest, ComputeEnvFees, ComputeResource, + ComputeResourceKind, C2DEnvironmentConfig, - ComputeResourcesPricingInfo + ComputeResourcesPricingInfo, + EnvironmentResourceRef } from '../../@types/C2D/C2D.js' import { BASE_CHAIN_ID, USDC_TOKEN_ADDRESS_BASE } from '../../utils/config.js' import { C2DEngine } from './compute_engine_base.js' @@ -65,13 +66,44 @@ import { isPersistentStorageType } from '../../@types/fileObject.js' import { getAddress, ZeroAddress } from 'ethers' -import { AccessList } from '../../@types/AccessList.js' +import { + ServiceStatusNumber, + ServiceStatusText, + SERVICE_START_PENDING_STATUSES +} from '../../@types/C2D/ServiceOnDemand.js' +import type { ServiceJob } from '../../@types/C2D/ServiceOnDemand.js' +import { resolveServiceImage } from './serviceResourceMatching.js' +import { + allocateHostPort, + releaseHostPort, + reserveHostPort, + seedAllocatedPorts, + userDataToEnv, + decryptUserData +} from '../core/service/utils.js' const C2D_CONTAINER_UID = 1000 const C2D_CONTAINER_GID = 1000 const trivyImage = 'aquasec/trivy:0.69.3' // Use pinned versions for safety +// Cross-process service lifecycle lock timing: locks are heartbeated every +// SERVICE_LOCK_HEARTBEAT_MS while an operation runs, and a lock not refreshed within +// SERVICE_LOCK_STALE_MS belongs to a crashed process and may be stolen. Staleness must +// be a comfortable multiple of the heartbeat so one missed beat (GC pause, busy DB) +// doesn't get a live operation's resources torn down under it. +const SERVICE_LOCK_HEARTBEAT_MS = 30_000 +const SERVICE_LOCK_STALE_MS = 120_000 + +// Identifies one engine instance as a service-lock holder across processes. pid alone is +// not enough: pids are reused, and one process can host several engine instances. +const makeServiceLockHolderId = () => + `${process.pid}:${Math.random().toString(36).slice(2, 10)}` + +// "Already gone" Docker errors (404 missing, 304 already stopped) are the desired end +// state of a teardown, so treat them as success. +const isBenignDockerError = (e: any) => e?.statusCode === 404 || e?.statusCode === 304 + export class C2DEngineDocker extends C2DEngine { private envs: ComputeEnvironment[] = [] @@ -80,6 +112,30 @@ export class C2DEngineDocker extends C2DEngine { private cronTime: number = 2000 private jobImageSizes: Map = new Map() private isInternalLoopRunning: boolean = false + // Set true by stop() so a stopped engine cannot reschedule or run another InternalLoop pass. + // Without this, an in-flight loop's finally → setNewTimer() resurrects the timer on a stopped + // instance, leaving two engines (old + new, e.g. across a test restart) racing the same DB + // and double-processing a job (→ Docker 409 "container name already in use"). + private stopped: boolean = false + // The currently-running InternalLoop pass, so stop() can drain it before returning. + private internalLoopPromise: Promise | null = null + // Per-service lifecycle lock: serviceIds with an exclusive operation in flight — the + // loop-driven start pipeline (processServiceStart), restartService or stopService. At + // most one such operation may run per service: the InternalLoop skips locked ids + // (including its expiry sweep), restart/stop throw. Without this, the loop's + // orphan-recovery tears down the network a concurrent restart just created (the + // "network ocean-svc- not found" failure) and start/stop/restart clobber each + // other's docker resources and job status. The in-memory set serializes within this + // process; the service_locks DB lease (acquired alongside it) extends the guarantee + // across processes sharing the same DB file + Docker daemon. + private serviceOpsInFlight: Set = new Set() + private readonly serviceLockHolderId: string = makeServiceLockHolderId() + private serviceLockHeartbeatTimer: NodeJS.Timeout | null = null + // The in-flight service lifecycle promises — processServiceStart() launched + // fire-and-forget by InternalLoop, plus handler-driven stopService()/restartService() + // calls — so stop() can drain them before returning. Otherwise an op could outlive + // stop() and race a restarted engine on the same shared DB. + private serviceOpPromises: Set> = new Set() private imageCleanupTimer: NodeJS.Timeout | null = null private paymentClaimTimer: NodeJS.Timeout | null = null private scanDBUpdateTimer: NodeJS.Timeout | null = null @@ -92,6 +148,9 @@ export class C2DEngineDocker extends C2DEngine { private trivyCachePath: string private cpuAllocations: Map = new Map() private envCpuCoresMap: Map = new Map() + // Host core IDs jobs may be pinned to, expanded from the cpu resource's `cpuList` + // config. null = no restriction (all host cores). + private configuredCpuList: number[] | null = null private activeBuildAborts: Map = new Map() public constructor( @@ -193,42 +252,157 @@ export class C2DEngineDocker extends C2DEngine { return this.getC2DConfig().tempFolder + this.getC2DConfig().hash } - private createBenchmarkEnvironment(sysinfo: any, envConfig: any): void { - const ramGB = this.physicalLimits.get('ram') || 0 + private resolveResourceKind(res: ComputeResource): ComputeResourceKind { + if (res.kind) return res.kind + if (res.init) return 'discrete' + return 'fungible' + } + + private resolveConnectionResourcePool( + sysinfo: any, + configResources: ComputeResource[] | undefined + ): Map { + const pool = new Map() + + const physicalCpu = sysinfo.NCPU + const physicalRamGB = Math.floor(sysinfo.MemTotal / 1024 / 1024 / 1024) const physicalDiskGB = this.physicalLimits.get('disk') || 0 - const gpuMap = new Map() - for (const env of envConfig.environments) { - if (env.resources) { - for (const res of env.resources) { - if (res.id !== 'cpu' && res.id !== 'ram' && res.id !== 'disk') { - if (!gpuMap.has(res.id)) { - gpuMap.set(res.id, res) - } - } + pool.set('cpu', { + id: 'cpu', + type: 'cpu', + kind: 'fungible', + total: physicalCpu, + max: physicalCpu, + min: 1 + }) + pool.set('ram', { + id: 'ram', + type: 'ram', + kind: 'fungible', + total: physicalRamGB, + max: physicalRamGB, + min: 1 + }) + pool.set('disk', { + id: 'disk', + type: 'disk', + kind: 'fungible', + total: physicalDiskGB, + max: physicalDiskGB, + min: 0 + }) + + for (const res of configResources ?? []) { + const resolvedKind = this.resolveResourceKind(res) + if (['cpu', 'ram', 'disk'].includes(res.id)) { + const base = pool.get(res.id) + const cap = this.physicalLimits.get(res.id) ?? res.total + if (res.total > cap) + CORE_LOGGER.warn( + `Resource "${res.id}": configured total ${res.total} exceeds physical ${cap}, capping` + ) + if (res.total !== undefined) base.total = Math.min(res.total, cap) + // cpuList replaces total for the cpu resource (mutually exclusive by schema): + // the effective total is the number of allowed cores. + if (res.id === 'cpu' && res.cpuList) + base.total = Math.min(this.parseCpusetString(res.cpuList).length, cap) + base.max = res.max !== undefined ? Math.min(res.max, base.total) : base.total + if (res.min !== undefined) base.min = res.min + if (res.constraints !== undefined) + base.constraints = structuredClone(res.constraints) + } else { + // Warn if a GPU resource has multiple DeviceIDs — each physical GPU should be its own resource. + if (res.init?.deviceRequests?.DeviceIDs?.length > 1) { + CORE_LOGGER.warn( + `Resource "${res.id}": DeviceIDs has ${res.init.deviceRequests.DeviceIDs.length} entries. ` + + `Each physical GPU should be its own resource with a single DeviceID.` + ) + } + const custom: ComputeResource = { + ...res, + kind: resolvedKind, + max: res.max ?? res.total, + min: res.min ?? 0 + } + pool.set(res.id, custom) + // Register in physicalLimits so checkGlobalResourceAvailability caps correctly. + this.physicalLimits.set(res.id, res.total) + } + } + return pool + } + + private resolveEnvironmentResources( + envDef: C2DEnvironmentConfig, + pool: Map + ): ComputeResource[] { + const configuredRefs = envDef.resources || [] + // cpu/ram/disk are baseline resources every environment has (unlike opt-in discrete + // resources such as GPUs) — guarantee they're always resolved, using pool defaults + // when the config doesn't reference them, so a config that forgets one doesn't + // silently lose all tracking/enforcement for it. + const configuredIds = new Set(configuredRefs.map((r) => r.id)) + const baselineRefs: EnvironmentResourceRef[] = (['cpu', 'ram', 'disk'] as const) + .filter((id) => !configuredIds.has(id)) + .map((id) => ({ id })) + const refs: EnvironmentResourceRef[] = [...configuredRefs, ...baselineRefs] + const result: ComputeResource[] = [] + for (const ref of refs) { + const poolRes = pool.get(ref.id) + if (!poolRes) { + CORE_LOGGER.warn(`resource "${ref.id}" not in pool, skipping`) + continue + } + const resolved: ComputeResource = { + ...poolRes, + init: poolRes.init ? structuredClone(poolRes.init) : undefined, + constraints: poolRes.constraints + ? structuredClone(poolRes.constraints) + : undefined + } + + if (poolRes.kind === 'fungible') { + resolved.total = + ref.total !== undefined ? Math.min(ref.total, poolRes.total) : poolRes.total + } + if (ref.max !== undefined) { + if (ref.max > resolved.total) { + CORE_LOGGER.warn( + `Environment "${envDef.description || envDef.id || 'unknown'}": resource "${ref.id}" has max (${ref.max}) greater than total (${resolved.total}) — clamping max to ${resolved.total}. Fix your config so max <= total for this resource.` + ) } + resolved.max = Math.min(ref.max, resolved.total) } + if (ref.min !== undefined) resolved.min = ref.min + if (ref.constraints !== undefined) + resolved.constraints = structuredClone(ref.constraints) + result.push(resolved) } - const gpuResources: ComputeResource[] = Array.from(gpuMap.values()) + return result + } + + private createBenchmarkEnvironment(sysinfo: any, envConfig: any): void { + // Collect all discrete accelerators (GPUs, FPGAs, etc.) from the connection-level resources. + const discreteResources: ComputeResource[] = (envConfig.resources ?? []).filter( + (res: ComputeResource) => this.resolveResourceKind(res) === 'discrete' + ) const benchmarkPrices: ComputeResourcesPricingInfo[] = - gpuResources.length > 0 ? [{ id: gpuResources[0].id, price: 1 }] : [] + discreteResources.length > 0 ? [{ id: discreteResources[0].id, price: 1 }] : [] const benchmarkFees: ComputeEnvFeesStructure = { [BASE_CHAIN_ID]: [{ feeToken: USDC_TOKEN_ADDRESS_BASE, prices: benchmarkPrices }] } + // Benchmark env uses resource refs: cpu/ram/disk are auto-detected; discrete accelerators listed by id. + const gpuRefs = discreteResources.map((r) => ({ id: r.id })) const benchmarkEnv: C2DEnvironmentConfig = { description: 'Auto-generated benchmark environment', storageExpiry: 604800, maxJobDuration: 180, minJobDuration: 0, - resources: [ - { id: 'cpu', total: sysinfo.NCPU, min: 1, max: sysinfo.NCPU }, - { id: 'ram', total: ramGB, min: 1, max: ramGB }, - { id: 'disk', total: physicalDiskGB, min: 0, max: physicalDiskGB }, - ...gpuResources - ], + resources: [{ id: 'cpu' }, { id: 'ram' }, { id: 'disk' }, ...gpuRefs], access: { addresses: [], accessLists: [ @@ -262,6 +436,8 @@ export class C2DEngineDocker extends C2DEngine { this.physicalLimits.set('cpu', sysinfo.NCPU) this.physicalLimits.set('ram', Math.floor(sysinfo.MemTotal / 1024 / 1024 / 1024)) + + this.resolveConfiguredCpuList(envConfig, sysinfo.NCPU) try { const diskStats = statfsSync(this.getC2DConfig().tempFolder) const diskGB = Math.floor((diskStats.bsize * diskStats.blocks) / 1024 / 1024 / 1024) @@ -294,65 +470,16 @@ export class C2DEngineDocker extends C2DEngine { } } + const connectionPool = this.resolveConnectionResourcePool( + sysinfo, + envConfig.resources + ) + for (let envIdx = 0; envIdx < envConfig.environments.length; envIdx++) { const envDef: C2DEnvironmentConfig = envConfig.environments[envIdx] const fees = this.processFeesForEnvironment(envDef.fees, supportedChains) - - const envResources: ComputeResource[] = [] - const cpuResources = { - id: 'cpu', - type: 'cpu', - total: sysinfo.NCPU, - max: sysinfo.NCPU, - min: 1, - description: os.cpus()[0].model - } - const ramResources = { - id: 'ram', - type: 'ram', - total: Math.floor(sysinfo.MemTotal / 1024 / 1024 / 1024), - max: Math.floor(sysinfo.MemTotal / 1024 / 1024 / 1024), - min: 1 - } - const physicalDiskGB = this.physicalLimits.get('disk') || 0 - const diskResources = { - id: 'disk', - type: 'disk', - total: physicalDiskGB, - max: physicalDiskGB, - min: 0 - } - - if (envDef.resources) { - for (const res of envDef.resources) { - // allow user to add other resources - if (res.id === 'cpu') { - if (res.total) cpuResources.total = res.total - if (res.max) cpuResources.max = res.max - if (res.min) cpuResources.min = res.min - } - if (res.id === 'ram') { - if (res.total) ramResources.total = res.total - if (res.max) ramResources.max = res.max - if (res.min) ramResources.min = res.min - } - if (res.id === 'disk') { - if (res.total) diskResources.total = res.total - if (res.max) diskResources.max = res.max - if (res.min !== undefined) diskResources.min = res.min - } - - if (res.id !== 'cpu' && res.id !== 'ram' && res.id !== 'disk') { - if (!res.max) res.max = res.total - if (!res.min) res.min = 0 - envResources.push(res) - } - } - } - envResources.push(cpuResources) - envResources.push(ramResources) - envResources.push(diskResources) + const envResources = this.resolveEnvironmentResources(envDef, connectionPool) const env: ComputeEnvironment = { id: '', @@ -368,7 +495,11 @@ export class C2DEngineDocker extends C2DEngine { queMaxWaitTimeFree: 0, runMaxWaitTime: 0, runMaxWaitTimeFree: 0, - enableNetwork: envDef.enableNetwork + enableNetwork: envDef.enableNetwork, + features: { + computeJobs: envDef.features?.computeJobs ?? true, + services: envDef.features?.services ?? true + } } if (envDef.storageExpiry !== undefined) env.storageExpiry = envDef.storageExpiry @@ -389,9 +520,15 @@ export class C2DEngineDocker extends C2DEngine { if (envDef.free.maxJobDuration !== undefined) env.free.maxJobDuration = envDef.free.maxJobDuration if (envDef.free.maxJobs !== undefined) env.free.maxJobs = envDef.free.maxJobs - if (envDef.free.resources) env.free.resources = envDef.free.resources if (envDef.free.allowImageBuild !== undefined) env.free.allowImageBuild = envDef.free.allowImageBuild + // Resolve free resource refs → full ComputeResource[] using the same connection pool. + if (envDef.free.resources) { + env.free.resources = this.resolveEnvironmentResources( + { resources: envDef.free.resources } as C2DEnvironmentConfig, + connectionPool + ) + } } const envIdSuffix = envDef.id || String(envIdx) @@ -406,41 +543,40 @@ export class C2DEngineDocker extends C2DEngine { ) } + // CPU affinity: all environments share the same core pool — the configured cpuList + // when present, otherwise the full physical core range. + // allocateCpus() dynamically assigns free cores per job across all envs. const physicalCpuCount = this.physicalLimits.get('cpu') || 0 - let cpuOffset = 0 + const allCores = + this.configuredCpuList ?? Array.from({ length: physicalCpuCount }, (_, i) => i) for (const env of this.envs) { const cpuRes = this.getResource(env.resources ?? [], 'cpu') if (cpuRes && cpuRes.total > 0) { - let isBenchmarkEnv = false - if (env.access?.accessLists) { - const baseAccessList = env.access?.accessLists?.[0] as AccessList - if (baseAccessList && baseAccessList[BASE_CHAIN_ID]) { - isBenchmarkEnv = baseAccessList[BASE_CHAIN_ID].includes( - getAddress('0xcb7Db55Ca9Aa9C3b25F5Bc266da63317fa02086a') - ) - } - } - - if (isBenchmarkEnv) { - const total = physicalCpuCount > 0 ? physicalCpuCount : cpuRes.total - const cores = Array.from({ length: total }, (_, i) => i) - this.envCpuCoresMap.set(env.id, cores) - CORE_LOGGER.info( - `CPU affinity: benchmark environment ${env.id} cores 0-${cores[cores.length - 1]}` - ) - } else { - const cores = Array.from({ length: cpuRes.total }, (_, i) => cpuOffset + i) - this.envCpuCoresMap.set(env.id, cores) - CORE_LOGGER.info( - `CPU affinity: environment ${env.id} cores ${cores[0]}-${cores[cores.length - 1]}` - ) - cpuOffset += cpuRes.total - } + // Each env gets its own copy: the pool array must never alias + // this.configuredCpuList (or another env's pool), so a mutation of one can't + // corrupt the others. + this.envCpuCoresMap.set(env.id, [...allCores]) + CORE_LOGGER.info( + `CPU affinity: environment ${env.id} shares pool of ${allCores.length} cores${ + this.configuredCpuList + ? ` (restricted by cpuList to [${allCores.join(',')}])` + : '' + }` + ) } } // Rebuild CPU allocations from running containers (handles node restart) await this.rebuildCpuAllocations() + await this.rebuildServiceCpuAllocations() + + // Re-seed the in-memory allocated-host-port set from running service jobs (handles node restart) + await seedAllocatedPorts(this.db, this.getC2DConfig().hash).catch((e) => { + CORE_LOGGER.warn(`Could not seed allocated service ports: ${e.message}`) + }) + + // (re)starting: clear the stopped flag so the loop can schedule again + this.stopped = false // only now set the timer if (!this.cronTimer) { @@ -455,6 +591,17 @@ export class C2DEngineDocker extends C2DEngine { return } + // Heartbeat the DB rows of every service lifecycle lock this instance holds, so a + // multi-minute image pull/build isn't stolen as a stale lock by another process. + if (!this.serviceLockHeartbeatTimer) { + this.serviceLockHeartbeatTimer = setInterval(() => { + if (this.serviceOpsInFlight.size === 0) return + this.db.refreshServiceLocks(this.serviceLockHolderId).catch((e) => { + CORE_LOGGER.warn(`service lock heartbeat failed: ${e.message}`) + }) + }, SERVICE_LOCK_HEARTBEAT_MS) + } + // Start image cleanup timer if (this.cleanupInterval) { if (this.imageCleanupTimer) { @@ -528,12 +675,30 @@ export class C2DEngineDocker extends C2DEngine { } } - public override stop(): Promise { + public override async stop(): Promise { + // Mark stopped FIRST so the in-flight loop's finally won't reschedule and a queued timer + // becomes a no-op. This keeps a stopped engine from racing a freshly-started one on the + // same shared DB (which caused the same job to be processed twice). + this.stopped = true // Clear the timer and reset the flag if (this.cronTimer) { clearTimeout(this.cronTimer) this.cronTimer = null } + // Drain a currently-running InternalLoop pass so it fully completes before we return, + // so the caller (e.g. addC2DEngines / tearDownAll) can start a new engine knowing the old + // one is quiescent. + if (this.internalLoopPromise) { + await this.internalLoopPromise.catch(() => {}) + this.internalLoopPromise = null + } + // Drain any in-flight service lifecycle ops — start pipelines launched by the loop + // AND handler-driven stopService/restartService calls — so none continue (and touch + // escrow/Docker on the shared DB) after the engine is considered stopped. + if (this.serviceOpPromises.size > 0) { + await Promise.allSettled([...this.serviceOpPromises]) + this.serviceOpPromises.clear() + } this.isInternalLoopRunning = false // Stop image cleanup timer if (this.imageCleanupTimer) { @@ -546,7 +711,10 @@ export class C2DEngineDocker extends C2DEngine { this.paymentClaimTimer = null CORE_LOGGER.debug('Payment claim timer stopped') } - return Promise.resolve() + if (this.serviceLockHeartbeatTimer) { + clearInterval(this.serviceLockHeartbeatTimer) + this.serviceLockHeartbeatTimer = null + } } public async updateImageUsage(image: string): Promise { @@ -1587,18 +1755,31 @@ export class C2DEngineDocker extends C2DEngine { } private setNewTimer() { + // never reschedule once stopped (prevents an in-flight loop's finally from resurrecting + // the timer on a stopped engine) + if (this.stopped) { + return + } if (this.cronTimer) { return } // don't set the cron if we don't have compute environments if (this.envs.length > 0) - this.cronTimer = setTimeout(this.InternalLoop.bind(this), this.cronTime) + this.cronTimer = setTimeout(() => { + // track the running pass so stop() can drain it + this.internalLoopPromise = this.InternalLoop() + }, this.cronTime) } private async InternalLoop() { // this is the internal loop of docker engine // gets list of all running jobs and process them one by one + // a queued timer may fire after stop(); a stopped engine must not process anything + if (this.stopped) { + return + } + // Prevent concurrent execution if (this.isInternalLoopRunning) { CORE_LOGGER.debug( @@ -1633,6 +1814,100 @@ export class C2DEngineDocker extends C2DEngine { // wait for all promises, there is no return await Promise.all(promises) } + + // Service-on-Demand health check: catch Running services whose container died on its + // own (crash, OOM, or Docker daemon down) instead of only noticing at expiresAt. + await this.checkRunningServices() + + // Service-on-Demand starts: advance pending service jobs through the start pipeline. + // Fire-and-forget (NOT awaited): an image pull can take minutes and must not block the + // loop (compute jobs + expiry must keep advancing). The in-progress guard prevents a + // second pipeline for the same service across overlapping ticks. processServiceStart + // never throws, so the unawaited promise is safe. + const pendingStarts = await this.db.getPendingServiceStarts( + this.getC2DConfig().hash + ) + for (const svc of pendingStarts) { + // eslint-disable-next-line no-await-in-loop + if (!(await this.tryAcquireServiceLifecycleLock(svc.serviceId))) continue + // Track the promise so stop() can drain it; clean both trackers when it settles. + const startPromise = this.processServiceStart(svc).finally(() => { + this.serviceOpPromises.delete(startPromise) + return this.releaseServiceLifecycleLock(svc.serviceId) + }) + this.serviceOpPromises.add(startPromise) + // processServiceStart persists every outcome, but its up-front DB re-read can + // still reject — consume it so the unawaited promise can't surface as an + // unhandled rejection (stop()'s allSettled drain never observes rejections). + startPromise.catch((e) => + CORE_LOGGER.error( + `processServiceStart ${svc.serviceId} failed unexpectedly: ${e.message}` + ) + ) + } + + // Service-on-Demand expiry: stop services whose paid window has elapsed. + const expiredServices = await this.db.getExpiredServiceJobs( + this.getC2DConfig().hash + ) + for (const svc of expiredServices) { + CORE_LOGGER.info(`Service ${svc.serviceId} expired — stopping`) + let stopped: ServiceJob | null + try { + // onlyIfExpired: doStopService re-checks expiresAt on the FRESH row under the + // lifecycle lock — a SERVICE_EXTEND that landed after our expiredServices + // snapshot must not have its service torn down. + stopped = await this.stopService(svc.serviceId, svc.owner, true) + } catch (e: any) { + // Typically the lifecycle lock (a restart/stop already in flight). Marking + // Expired without teardown would leak the container/ports, so defer the whole + // sweep for this service to the next tick. + CORE_LOGGER.error( + `Failed to stop expired service ${svc.serviceId}: ${e.message} — retrying next tick` + ) + continue + } + // Extended mid-sweep (expiresAt is in the future again on the fresh row) — not + // expired anymore, regardless of status. Leave it alone. + if (stopped && Date.now() < stopped.expiresAt) { + CORE_LOGGER.debug( + `Service ${svc.serviceId} was extended after the expiry snapshot — skipping expiry` + ) + continue + } + // Flip to Expired ONLY once teardown actually completed. Expired is terminal — + // it is never swept again — so marking it while the stop failed mid-way (the + // job comes back as Error "stop failed: …" with its container possibly still + // running and its ports still reserved) would leak those resources forever. + // Left as Error, the job stays in the expirable set and is retried next tick. + if ( + !stopped || + (stopped.status !== ServiceStatusNumber.Stopped && + stopped.status !== ServiceStatusNumber.Expired) + ) { + CORE_LOGGER.error( + `Expired service ${svc.serviceId} teardown did not complete ` + + `(status "${stopped?.statusText ?? 'gone'}") — retrying next tick` + ) + continue + } + // mark the (now stopped) record as Expired so it is not picked up again + const [stoppedJob] = await this.db.getServiceJob(svc.serviceId, svc.owner) + if (stoppedJob) { + stoppedJob.status = ServiceStatusNumber.Expired + stoppedJob.statusText = ServiceStatusText[ServiceStatusNumber.Expired] + await this.db.updateServiceJob(stoppedJob) + // Expired is the ONLY transition that releases the reservation: amounts stop + // counting (Expired is not an active status) and the host ports are freed + // here. Everywhere else — including an explicit user stop — the paid-for + // reservation survives so the service can be restarted on the same endpoints. + for (const ep of stoppedJob.endpoints ?? []) releaseHostPort(ep.hostPort) + CORE_LOGGER.debug( + `Service ${svc.serviceId} marked Expired — all resources released ` + + `(ports [${(stoppedJob.endpoints ?? []).map((ep) => ep.hostPort).join(',')}])` + ) + } + } } catch (e) { CORE_LOGGER.error(`Error in C2D InternalLoop: ${e.message}`) } finally { @@ -2301,6 +2576,30 @@ export class C2DEngineDocker extends C2DEngine { return cores } + /** + * Expands the cpu resource's cpuList config (format already schema-validated) into + * `configuredCpuList` and checks it against this host's CPU count. Throws when the + * list references cores the host doesn't have — a config error; fail fast (abort node + * startup, or surface the error to the admin on a config push) rather than silently + * pin elsewhere or leave a half-initialized engine behind. + */ + private resolveConfiguredCpuList( + envConfig: { resources?: ComputeResource[] }, + ncpu: number + ): void { + this.configuredCpuList = null + const cpuConfigEntry = (envConfig.resources ?? []).find((r) => r.id === 'cpu') + if (!cpuConfigEntry?.cpuList) return + const cores = this.parseCpusetString(cpuConfigEntry.cpuList) + const outOfRange = cores.filter((c) => c >= ncpu) + if (outOfRange.length > 0) { + throw new Error( + `C2D Engine ${this.getC2DConfig().hash}: invalid cpuList "${cpuConfigEntry.cpuList}" — core IDs [${outOfRange.join(',')}] don't exist on this host (${ncpu} CPUs, valid IDs: 0-${ncpu - 1})` + ) + } + this.configuredCpuList = cores + } + private allocateCpus(jobId: string, count: number, envId: string): string | null { const envCores = this.envCpuCoresMap.get(envId) if (!envCores || envCores.length === 0 || count <= 0) return null @@ -2381,6 +2680,43 @@ export class C2DEngineDocker extends C2DEngine { } } + /** + * On startup, inspects running Docker service containers to rebuild the CPU allocation + * map. Statuses without a container yet (Starting/Locking/PullImage/BuildImage/Claiming) + * are skipped; Running/Error services with a still-present (even if exited) container + * keep their cores reserved, matching the "reserved until explicit restart/stop/expiry" + * semantics that getRunningServiceJobs already relies on. + */ + private async rebuildServiceCpuAllocations(): Promise { + if (this.envCpuCoresMap.size === 0) return + try { + const services = await this.db.getRunningServiceJobs(this.getC2DConfig().hash) + for (const job of services) { + if (!job.containerId) continue + try { + const container = this.docker.getContainer(job.containerId) + const info = await container.inspect() + const cpuset = info.HostConfig?.CpusetCpus + if (cpuset) { + const cores = this.parseCpusetString(cpuset) + if (cores.length > 0) { + this.cpuAllocations.set(job.serviceId, cores) + CORE_LOGGER.info( + `CPU affinity: recovered allocation [${cpuset}] for running service ${job.serviceId}` + ) + } + } + } catch (e) { + // Container may be gone + } + } + } catch (e) { + CORE_LOGGER.error( + `CPU affinity: failed to rebuild service allocations: ${e.message}` + ) + } + } + private async cleanupJob(job: DBComputeJob) { // cleaning up // - claim payment or release lock @@ -2838,6 +3174,1198 @@ export class C2DEngineDocker extends C2DEngine { } } + // ── Service on Demand ───────────────────────────────────────────────── + + // Pulls a plain image reference with the same registry-auth + Trivy scan + // guarantees as pullImage(). Service code MUST go through this — never docker.pull() raw. + private async pullImageRef( + imageRef: string, + encryptedDockerRegistryAuth?: string, + logFile?: string + ): Promise { + const controller = new AbortController() + const timer = setTimeout(() => controller.abort(), this.getImagePullTimeoutMs()) + try { + const { registry } = this.parseImage(imageRef) + let dockerRegistryAuthForPull: any + if (encryptedDockerRegistryAuth) { + const decryptedDockerRegistryAuth = await this.keyManager.decrypt( + Uint8Array.from(Buffer.from(encryptedDockerRegistryAuth, 'hex')), + EncryptMethod.ECIES + ) + dockerRegistryAuthForPull = JSON.parse(decryptedDockerRegistryAuth.toString()) + } else { + dockerRegistryAuthForPull = this.getDockerRegistryAuth(registry) + } + + const pullOptions: any = { abortSignal: controller.signal } + if (dockerRegistryAuthForPull) { + const registryUrl = new URL(registry) + const serveraddress = + registryUrl.hostname + (registryUrl.port ? `:${registryUrl.port}` : '') + const authString = dockerRegistryAuthForPull.auth + ? dockerRegistryAuthForPull.auth + : Buffer.from( + `${dockerRegistryAuthForPull.username}:${dockerRegistryAuthForPull.password}` + ).toString('base64') + pullOptions.authconfig = { + serveraddress, + ...(dockerRegistryAuthForPull.auth + ? { auth: authString } + : { + username: dockerRegistryAuthForPull.username, + password: dockerRegistryAuthForPull.password + }) + } + } + + const pullStream = await this.docker.pull(imageRef, pullOptions) + await new Promise((resolve, reject) => { + this.docker.modem.followProgress( + pullStream, + (err: any) => { + if (err) { + if (logFile) appendFileSync(logFile, String(err.message)) + return reject(err) + } + resolve() + }, + (progress: any) => { + if (logFile) appendFileSync(logFile, (progress.status ?? '') + '\n') + } + ) + }) + this.updateImageUsage(imageRef).catch((e) => { + CORE_LOGGER.debug(`Failed to track image usage: ${e.message}`) + }) + + // Image scanning — same Trivy check used by the compute path. + if (this.scanImages) { + const scanResult = await this.checkImageVulnerability(imageRef) + if (scanResult.vulnerable) { + await this.docker + .getImage(imageRef) + .remove({ force: true }) + .catch(() => {}) + throw new Error( + `Image "${imageRef}" failed security scan: ${JSON.stringify(scanResult.summary)}` + ) + } + } + } finally { + clearTimeout(timer) + } + } + + // Builds an image from a plain Dockerfile string with the same build mechanics as + // buildImage(). Service code MUST go through this — never docker.buildImage() raw. + private async buildImageFromSpec( + imageRef: string, + dockerfile: string, + additionalDockerFiles: Record, + maxDurationSeconds: number, + memoryGB?: number, + cpuCount?: number + ): Promise { + const controller = new AbortController() + const timer = setTimeout(() => controller.abort(), maxDurationSeconds * 1000) + try { + const pack = tarStream.pack() + pack.entry({ name: 'Dockerfile' }, dockerfile) + for (const [name, content] of Object.entries(additionalDockerFiles)) + pack.entry({ name }, content) + pack.finalize() + + const buildOptions: Dockerode.ImageBuildOptions = { + t: imageRef, + nocache: true, + abortSignal: controller.signal + } + if (memoryGB) { + buildOptions.memory = memoryGB * 1024 ** 3 + buildOptions.memswap = memoryGB * 1024 ** 3 + } + if (cpuCount) { + buildOptions.cpuquota = cpuCount * 100000 + buildOptions.cpuperiod = 100000 + } + + const buildStream = (await this.docker.buildImage(pack, buildOptions)) as Readable + await new Promise((resolve, reject) => { + this.docker.modem.followProgress(buildStream, (err: any) => + err ? reject(err) : resolve() + ) + }) + // Verify image exists after build + await this.docker.getImage(imageRef).inspect() + + // Image scanning — same Trivy gate used by pullImageRef(). A built image must clear + // it too; on failure remove the built image so it cannot be used to start a service. + if (this.scanImages) { + const scanResult = await this.checkImageVulnerability(imageRef) + if (scanResult.vulnerable) { + await this.docker + .getImage(imageRef) + .remove({ force: true }) + .catch(() => {}) + throw new Error( + `Image "${imageRef}" failed security scan: ${JSON.stringify(scanResult.summary)}` + ) + } + } + } finally { + clearTimeout(timer) + } + } + + // Builds Docker HostConfig resource constraints (memory, cpu, GPU device requests) + // from a service resource request, resolved against the connection-level resource pool. + private buildServiceResourceConstraints( + resources: ComputeResourceRequest[], + serviceId: string, + environment: string + ): { + Memory?: number + NanoCpus?: number + DeviceRequests?: any[] + CpusetCpus?: string + } { + const connResources: ComputeResource[] = + this.getC2DConfig().connection?.resources ?? [] + const ram = resources.find((r) => r.id === 'ram')?.amount + const cpu = resources.find((r) => r.id === 'cpu')?.amount + const deviceRequests = this.getDockerDeviceRequest(resources, connResources) ?? [] + const cpusetStr = + cpu && cpu > 0 ? this.allocateCpus(serviceId, cpu, environment) : null + return { + Memory: ram ? ram * 1024 ** 3 : undefined, + NanoCpus: cpu ? cpu * 1e9 : undefined, + DeviceRequests: deviceRequests.length ? deviceRequests : undefined, + CpusetCpus: cpusetStr ?? undefined + } + } + + // Handler-facing: persist the initial Starting record and return immediately so the HTTP + // response carries the serviceId without waiting for escrow/image/container. The background + // loop then calls processServiceStart() to advance it. Persisting Starting also reserves the + // service's resources (getRunningServiceJobs counts Starting), preventing oversubscription. + public override async createServiceJob( + environment: string, + image: string, + tag: string | undefined, + checksum: string | undefined, + dockerfile: string | undefined, + additionalDockerFiles: Record | undefined, + dockerCmd: string[] | undefined, + dockerEntrypoint: string[] | undefined, + exposedPorts: number[], + resources: ComputeResourceRequest[], + duration: number, + owner: string, + payment: DBComputeJobPayment, + serviceId: string, + userData?: string + ): Promise { + const containerImage = resolveServiceImage( + image, + tag, + checksum, + dockerfile, + serviceId + ) + const job: ServiceJob = { + serviceId, + clusterHash: this.getC2DConfig().hash, + environment, + owner, + image, + tag, + checksum, + dockerfile, + additionalDockerFiles, + dockerCmd, + dockerEntrypoint, + containerImage, + containerId: '', + networkId: '', + status: ServiceStatusNumber.Starting, + statusText: ServiceStatusText[ServiceStatusNumber.Starting], + dateCreated: new Date().toISOString(), + expiresAt: Date.now() + duration * 1000, + duration, + exposedPorts, + endpoints: [], + userData, // stored as received (ECIES-encrypted); decrypted transiently at container start + resources: resources.map((r) => ({ id: r.id, amount: r.amount })), + payment + } + await this.db.newServiceJob(job) + return job + } + + // Background pipeline (driven by InternalLoop): Starting → Locking → (Pull|Build)Image → + // Claiming → Running. Escrow is reordered vs. the old sync flow: createLock first, claimLock + // only AFTER the image succeeds, cancelLock (refund) if the image step fails. Never throws — + // every terminal outcome is persisted as the job status so clients see it via serviceStatus. + public override async processServiceStart(snapshot: ServiceJob): Promise { + // Re-read the job under the lifecycle lock: the loop's pendingStarts snapshot was + // queried BEFORE the lock was acquired, so it can be stale — e.g. captured while a + // restart was mid-pull and processed after that restart completed and released the + // lock. Acting on the stale row would tear down the freshly created container + + // network (and redo escrow ops) of a service that is actually fine. + const [job] = await this.db.getServiceJob(snapshot.serviceId, snapshot.owner) + if (!job || !SERVICE_START_PENDING_STATUSES.includes(job.status)) { + CORE_LOGGER.debug( + `processServiceStart ${snapshot.serviceId}: stale snapshot ` + + `(snapshot status "${snapshot.statusText}", fresh status ` + + `"${job?.statusText ?? 'row gone'}") — nothing to do` + ) + return + } + + const { serviceId } = job + const { chainId, token } = job.payment + CORE_LOGGER.debug( + `processServiceStart ${serviceId}: picked up in status "${job.statusText}" ` + + `(image=${job.containerImage}, owner=${job.owner})` + ) + + // Orphan recovery: a previous process died mid-start/mid-restart (after a node + // restart the in-memory loop guard is empty, so an intermediate-state record reaches + // here). Refund any unclaimed lock, tear down partial docker, and mark Error — never + // resume on-chain ops. A PAID (claimed) job keeps its host ports reserved: it stays + // restartable on the same endpoints until expiresAt (the expiry sweep releases them, + // and seedAllocatedPorts re-seeds them at boot). An unpaid one frees them. + if (job.status !== ServiceStatusNumber.Starting) { + CORE_LOGGER.error( + `processServiceStart: orphaned service ${serviceId} in state "${job.statusText}" — cleaning up` + ) + await this.logServiceDockerState( + `orphan recovery ${serviceId} (state "${job.statusText}")`, + serviceId + ) + if (job.payment.lockTx && !job.payment.claimTx && !job.payment.cancelTx) { + job.payment.cancelTx = await this.safeCancelLock( + chainId, + serviceId, + token, + job.owner + ) + } + if (job.containerId) + await this.cleanupServiceDocker( + this.docker.getContainer(job.containerId), + null, + serviceId + ) + await this.removeServiceNetwork(serviceId, job.networkId).catch((e) => { + CORE_LOGGER.debug(`orphan recovery ${serviceId}: network removal: ${e.message}`) + }) + if (!job.payment.claimTx) { + for (const ep of job.endpoints) releaseHostPort(ep.hostPort) + } + job.status = ServiceStatusNumber.Error + job.statusText = 'Service start aborted (node restarted mid-start)' + await this.db.updateServiceJob(job) + return + } + + const sod = this.getC2DConfig().connection?.serviceOnDemand + // Live docker handles, tracked so the catch can tear down a half-created service. + let network: Dockerode.Network | null = null + let container: Dockerode.Container | null = null + try { + // 1. LOCKING — lock the consumer's funds in escrow (refundable until claimed). + job.status = ServiceStatusNumber.Locking + job.statusText = ServiceStatusText[ServiceStatusNumber.Locking] + await this.db.updateServiceJob(job) + const lockTx = await this.escrow.createLock( + chainId, + serviceId, + token, + job.owner, + job.payment.cost, + this.escrow.getMinLockTime(job.duration) + ) + if (!lockTx) throw new Error('Escrow lock failed') + await this.escrow.waitForTransaction(chainId, lockTx) + job.payment.lockTx = lockTx + await this.db.updateServiceJob(job) + + // 2. IMAGE — pull or build the image (vulnerability scan runs inside these helpers). + let imageError: Error | null = null + try { + if (job.dockerfile) { + if (!sod?.allowImageBuild) + throw new Error( + 'Dockerfile-based services are not allowed on this environment (allowImageBuild=false)' + ) + job.status = ServiceStatusNumber.BuildImage + job.statusText = ServiceStatusText[ServiceStatusNumber.BuildImage] + await this.db.updateServiceJob(job) + const ram = job.resources.find((r) => r.id === 'ram')?.amount + const cpu = job.resources.find((r) => r.id === 'cpu')?.amount + await this.buildImageFromSpec( + job.containerImage, + job.dockerfile, + job.additionalDockerFiles ?? {}, + sod?.maxDurationSeconds ?? job.duration, + ram, + cpu + ) + } else { + job.status = ServiceStatusNumber.PullImage + job.statusText = ServiceStatusText[ServiceStatusNumber.PullImage] + await this.db.updateServiceJob(job) + await this.pullImageRef(job.containerImage) + } + } catch (e: any) { + imageError = e + } + + // 3. PAYMENT — claim the lock on success, or cancel it (refund the consumer) if the + // image step failed. + job.status = ServiceStatusNumber.Claiming + job.statusText = ServiceStatusText[ServiceStatusNumber.Claiming] + await this.db.updateServiceJob(job) + + if (imageError) { + job.payment.cancelTx = await this.safeCancelLock( + chainId, + serviceId, + token, + job.owner + ) + job.status = job.dockerfile + ? ServiceStatusNumber.BuildImageFailed + : ServiceStatusNumber.PullImageFailed + job.statusText = String(imageError.message) + await this.db.updateServiceJob(job) + CORE_LOGGER.error( + `startService ${serviceId} image step failed (lock refunded): ${imageError.message}` + ) + return + } + + const claimTx = await this.escrow.claimLock( + chainId, + serviceId, + token, + job.owner, + job.payment.cost, + `service-start:${serviceId}` + ) + if (!claimTx) { + job.payment.cancelTx = await this.safeCancelLock( + chainId, + serviceId, + token, + job.owner + ) + throw new Error('Escrow claim failed — lock cancelled') + } + job.payment.claimTx = claimTx + await this.db.updateServiceJob(job) + + // 4. CONTINUE — allocate host ports → network → create + start container → Running. + // Sequential allocation with rollback (job.endpoints isn't populated yet, so a mid-way + // failure would otherwise strand reserved ports in the in-memory allocatedPorts set). + const [rangeStart, rangeEnd] = sod?.hostPortRange ?? [30000, 32767] + const hostPorts: number[] = [] + try { + for (let i = 0; i < job.exposedPorts.length; i++) { + // eslint-disable-next-line no-await-in-loop + hostPorts.push(await allocateHostPort(rangeStart, rangeEnd)) + } + } catch (e) { + for (const port of hostPorts) releaseHostPort(port) + throw e + } + + const nodeHost = sod?.nodeHost ?? 'localhost' + job.endpoints = job.exposedPorts.map((cp, i) => ({ + containerPort: cp, + hostPort: hostPorts[i], + url: `http://${nodeHost}:${hostPorts[i]}` + })) + + network = await this.createServiceNetwork(serviceId) + + // Container env from the decrypted (in-memory) userData; command/entrypoint from the request. + const decryptedUserData = await decryptUserData(job.userData, this.keyManager) + const env = userDataToEnv(decryptedUserData) + const cmd = job.dockerCmd?.length ? job.dockerCmd : undefined + const entrypoint = job.dockerEntrypoint?.length ? job.dockerEntrypoint : undefined + + const PortBindings: Record> = {} + const ExposedPorts: Record> = {} + job.exposedPorts.forEach((cp, i) => { + PortBindings[`${cp}/tcp`] = [{ HostPort: String(hostPorts[i]) }] + ExposedPorts[`${cp}/tcp`] = {} + }) + + const { Memory, NanoCpus, DeviceRequests, CpusetCpus } = + this.buildServiceResourceConstraints( + job.resources.map((r) => ({ id: r.id, amount: r.amount })), + job.serviceId, + job.environment + ) + + container = await this.docker.createContainer({ + Image: job.containerImage, + Cmd: cmd, + Entrypoint: entrypoint, + Env: Object.entries(env).map(([k, v]) => `${k}=${v}`), + ExposedPorts, + HostConfig: { + Memory, + NanoCpus, + DeviceRequests, + CpusetCpus, + PortBindings, + NetworkMode: network.id, + SecurityOpt: ['no-new-privileges'], // security plan #5 + CapDrop: ['ALL'], + PidsLimit: 512 + } + }) + CORE_LOGGER.debug( + `start ${serviceId}: created container ${container.id} on network ${network.id} — starting` + ) + await container.start() + + job.containerId = container.id + job.networkId = network.id + job.status = ServiceStatusNumber.Running + job.statusText = ServiceStatusText[ServiceStatusNumber.Running] + await this.db.updateServiceJob(job) + CORE_LOGGER.info( + `start ${serviceId}: completed — container ${container.id} Running on ` + + `ports [${job.endpoints.map((ep) => ep.hostPort).join(',')}]` + ) + } catch (err: any) { + await this.logServiceDockerState( + `start ${serviceId}: FAILED (${err.message}) — docker state at failure`, + serviceId + ) + await this.cleanupServiceDocker(container, network, serviceId) + // Refund if funds were locked but never claimed (e.g. container creation failed). + if (job.payment.lockTx && !job.payment.claimTx && !job.payment.cancelTx) { + job.payment.cancelTx = await this.safeCancelLock( + chainId, + serviceId, + token, + job.owner + ) + } + if (!job.payment.claimTx) { + // Never paid (lock refunded): the job is not restartable, so free its ports. + for (const ep of job.endpoints) releaseHostPort(ep.hostPort) + } + // Paid (claimed) failures keep their host ports reserved: the consumer paid until + // expiresAt and may SERVICE_RESTART on the same endpoints anytime — the expiry + // sweep releases them once the window elapses. + job.status = ServiceStatusNumber.Error + job.statusText = String(err.message) + await this.db.updateServiceJob(job) + CORE_LOGGER.error(`startService ${serviceId} failed: ${err.message}`) + } + } + + // Best-effort escrow refund used by the start pipeline's failure paths. Returns the cancel + // tx hash, or '' if there was nothing to cancel / the cancel itself failed (never throws). + private async safeCancelLock( + chainId: number, + serviceId: string, + token: string, + owner: string + ): Promise { + try { + return (await this.escrow.cancelExpiredLock(chainId, serviceId, token, owner)) ?? '' + } catch (e: any) { + CORE_LOGGER.error(`cancelExpiredLock failed for ${serviceId}: ${e.message}`) + return '' + } + } + + // DEBUG-level dump of the Docker daemon state relevant to services: every container + // (docker ps -a, compacted) and every ocean-svc-* network. Pure diagnostics so a + // lifecycle failure on a remote node can be reconstructed from its logs — never throws. + private async logServiceDockerState(context: string, serviceId?: string) { + try { + const [containers, networks] = await Promise.all([ + this.docker.listContainers({ all: true }), + this.docker.listNetworks() + ]) + const cs = containers.map((c: any) => ({ + id: c.Id?.slice(0, 12), + names: c.Names, + state: c.State, + status: c.Status, + networks: Object.keys(c.NetworkSettings?.Networks ?? {}) + })) + const ns = networks + .filter( + (n: any) => + n.Name?.startsWith('ocean-svc-') || (serviceId && n.Name?.includes(serviceId)) + ) + .map((n: any) => ({ id: n.Id?.slice(0, 12), name: n.Name })) + CORE_LOGGER.debug( + `[docker state] ${context}: ocean-svc networks=${JSON.stringify(ns)} ` + + `containers=${JSON.stringify(cs)}` + ) + } catch (e: any) { + CORE_LOGGER.debug(`[docker state] ${context}: unavailable (${e.message})`) + } + } + + // Best-effort teardown of a half-created service container + network. Used by the + // startService / restartService failure paths to avoid leaking Docker networks + // (which would exhaust the daemon's IPAM CIDR pool over repeated failures). The network + // is removed by deterministic name too, so it is cleaned even when createNetwork itself + // threw and no live handle exists. + private async cleanupServiceDocker( + container: Dockerode.Container | null, + network: Dockerode.Network | null, + serviceId: string + ): Promise { + this.releaseCpus(serviceId) + if (container) { + await container.stop({ t: 5 }).catch(() => {}) + await container.remove({ force: true }).catch(() => {}) + } + await this.removeServiceNetwork(serviceId, network?.id).catch(() => {}) + } + + // Removes a service's Docker network by BOTH the persisted id (when set) and the + // deterministic name `ocean-svc-`: job.networkId is only persisted after the + // container starts, so a node crash mid-start leaves networkId='' in the DB while the + // named network still exists — removal must never depend solely on the persisted field. + // Any container still attached at this point is a stale leftover of the same service + // (every caller tears down the known container first), so it is force-removed to + // unblock network.remove(). Benign "already gone" errors resolve silently; anything + // else propagates so callers can decide whether teardown failure matters. + private async removeServiceNetwork( + serviceId: string, + networkId?: string + ): Promise { + const refs = new Set([`ocean-svc-${serviceId}`]) + if (networkId) refs.add(networkId) + for (const ref of refs) { + const network = this.docker.getNetwork(ref) // dockerode accepts a name or an id + let info: any + try { + // eslint-disable-next-line no-await-in-loop + info = await network.inspect() + } catch (e: any) { + if (isBenignDockerError(e)) { + CORE_LOGGER.debug(`removeServiceNetwork ${serviceId}: "${ref}" already gone`) + continue + } + throw e + } + const attached = Object.keys(info?.Containers ?? {}) + CORE_LOGGER.debug( + `removeServiceNetwork ${serviceId}: removing "${ref}" (id=${info?.Id?.slice(0, 12)}, ` + + `attached containers=[${attached.map((c) => c.slice(0, 12)).join(',')}])` + ) + for (const containerId of attached) { + // eslint-disable-next-line no-await-in-loop + await this.docker + .getContainer(containerId) + .remove({ force: true }) + .catch((e) => { + if (!isBenignDockerError(e)) throw e + }) + } + // eslint-disable-next-line no-await-in-loop + await network.remove().catch((e) => { + if (!isBenignDockerError(e)) throw e + }) + CORE_LOGGER.debug(`removeServiceNetwork ${serviceId}: "${ref}" removed`) + } + } + + // createNetwork with self-healing on a 409 name conflict: a conflict means a previous + // incarnation of this service leaked its network (e.g. the node died before networkId + // was persisted) — remove the stale one and retry once instead of failing the + // start/restart forever. + private async createServiceNetwork(serviceId: string): Promise { + const Name = `ocean-svc-${serviceId}` + try { + const network = await this.docker.createNetwork({ Name }) + CORE_LOGGER.debug(`createServiceNetwork ${serviceId}: created "${Name}"`) + return network + } catch (e: any) { + if (e?.statusCode !== 409) throw e + CORE_LOGGER.error( + `createNetwork conflict for ${Name} — removing stale network and retrying` + ) + await this.logServiceDockerState( + `createServiceNetwork ${serviceId}: 409 conflict`, + serviceId + ) + await this.removeServiceNetwork(serviceId) + return await this.docker.createNetwork({ Name }) + } + } + + // Checked every InternalLoop tick (same cadence as compute jobs) so a service whose + // container died on its own (crash, OOM, or the whole Docker daemon going down) is + // detected within ~cronTime instead of only at expiresAt. + private async checkRunningServices(): Promise { + const services = await this.db.getRunningServiceJobs(this.getC2DConfig().hash) + // Skip services with a lifecycle op in flight: a restart intentionally kills the + // container mid-way, which must not be reported as an unexpected death. + const runningOnly = services.filter( + (svc) => + svc.status === ServiceStatusNumber.Running && + !this.serviceOpsInFlight.has(svc.serviceId) + ) + await Promise.all(runningOnly.map((svc) => this.checkServiceContainerHealth(svc))) + } + + private async checkServiceContainerHealth(job: ServiceJob): Promise { + let details + try { + const container = this.docker.getContainer(job.containerId) + details = await container.inspect() + } catch (e: any) { + await this.markServiceFailed(job, `container lost: ${e.message}`) + return + } + if (details.State.Running === false) { + const reason = details.State.OOMKilled + ? 'OOM killed' + : details.State.Error + ? `error: ${details.State.Error}` + : `exited with code ${details.State.ExitCode}` + await this.markServiceFailed(job, reason) + } + } + + // Flips a service to Error after its container died unexpectedly. Deliberately does not + // touch Docker or release host ports/network — the consumer already paid for those and + // restartService() reuses them (and does its own best-effort teardown of the dead + // container/network first). + private async markServiceFailed(job: ServiceJob, reason: string): Promise { + // Take the SAME lifecycle lease every start/stop/restart holds, so the Error write + // is serialized with them — a check-then-write here could otherwise overwrite a + // Restarting state persisted between our lease check and our update. Failing to + // acquire (op in flight locally or in another process) fails CLOSED: skip this + // tick; a genuinely dead container is re-detected on the next one. + if (!(await this.tryAcquireServiceLifecycleLock(job.serviceId))) { + CORE_LOGGER.debug( + `markServiceFailed ${job.serviceId}: skipped — could not take the lifecycle lease` + ) + return + } + try { + // Re-read UNDER the lease: this snapshot cannot go stale before our write. + const [fresh] = await this.db.getServiceJob(job.serviceId, job.owner) + if (!fresh || fresh.status !== ServiceStatusNumber.Running) { + // already moved on (stopped/restarted/expired) by another path — don't clobber it + return + } + if (fresh.containerId !== job.containerId) { + // the container we observed dead is no longer the job's container (a restart + // replaced it since our snapshot) — the current one is not known to be dead + return + } + fresh.status = ServiceStatusNumber.Error + fresh.statusText = `service container exited unexpectedly: ${reason}` + await this.db.updateServiceJob(fresh) + CORE_LOGGER.error(`Service ${job.serviceId} container died — ${reason}`) + } finally { + await this.releaseServiceLifecycleLock(job.serviceId) + } + } + + // Tries to take the per-service lifecycle lock: the in-memory set serializes callers + // inside this process, the service_locks DB lease serializes across processes sharing + // the DB + Docker daemon. Returns false when someone else owns the service. + private async tryAcquireServiceLifecycleLock(serviceId: string): Promise { + // A stopped engine must not begin new lifecycle work: a replacement engine (config + // push, restart) may already be running against the same DB and daemon. + if (this.stopped) return false + if (this.serviceOpsInFlight.has(serviceId)) return false + // Reserve in-memory first — the synchronous has()→add() pair keeps concurrent local + // callers out while the async DB acquire below is in flight. + this.serviceOpsInFlight.add(serviceId) + let acquired = false + try { + acquired = await this.db.acquireServiceLock( + serviceId, + this.serviceLockHolderId, + SERVICE_LOCK_STALE_MS + ) + } catch (e: any) { + // Fail CLOSED: proceeding without the SQLite lease would disable cross-process + // exclusion exactly when it matters most — SQLITE_BUSY-style errors are most + // likely precisely when two processes contend. Deferring costs little: every + // lifecycle operation writes to the same SQLite file moments later anyway, so it + // could not complete with a broken DB regardless. Callers reject with "retry + // shortly" (handlers) or retry on the next tick (the loop). + CORE_LOGGER.warn( + `service lock DB acquire failed for ${serviceId} — deferring the operation (fail closed): ${e.message}` + ) + this.serviceOpsInFlight.delete(serviceId) + return false + } + if (!acquired) { + this.serviceOpsInFlight.delete(serviceId) + CORE_LOGGER.debug( + `service lock ${serviceId}: DB lease held by another process — not acquired` + ) + return false + } + // Re-check stopped: the engine may have been stopped (and drained) while the async + // DB acquire was in flight — beginning work now would race the replacement engine. + if (this.stopped) { + CORE_LOGGER.debug( + `service lock ${serviceId}: engine stopped during acquire — releasing` + ) + await this.releaseServiceLifecycleLock(serviceId) + return false + } + CORE_LOGGER.debug( + `service lock ${serviceId}: acquired by ${this.serviceLockHolderId}` + ) + return true + } + + // Takes the per-service lifecycle lock or throws: a stop must not run while the loop's + // start pipeline or a restart owns the service — its by-name network removal would tear + // down the docker resources the other operation is creating (and vice versa). + private async acquireServiceLifecycleLock(serviceId: string): Promise { + if (this.stopped) { + throw new Error( + `C2D engine is stopped — cannot run a service operation for ${serviceId}, retry shortly` + ) + } + if (!(await this.tryAcquireServiceLifecycleLock(serviceId))) { + throw new Error( + `Service ${serviceId} has a start/stop/restart operation in progress — retry shortly` + ) + } + } + + // Releases both lock layers. Never throws: a failed DB delete only leaves a row that + // expires via SERVICE_LOCK_STALE_MS. + private async releaseServiceLifecycleLock(serviceId: string): Promise { + this.serviceOpsInFlight.delete(serviceId) + try { + await this.db.releaseServiceLock(serviceId, this.serviceLockHolderId) + CORE_LOGGER.debug( + `service lock ${serviceId}: released by ${this.serviceLockHolderId}` + ) + } catch (e: any) { + CORE_LOGGER.warn(`service lock DB release failed for ${serviceId}: ${e.message}`) + } + } + + public override async stopService( + serviceId: string, + owner: string, + onlyIfExpired: boolean = false + ): Promise { + await this.acquireServiceLifecycleLock(serviceId) + // Tracked like loop-launched starts so engine stop() drains an in-flight stop too. + const op = this.doStopService(serviceId, owner, onlyIfExpired).finally(() => { + this.serviceOpPromises.delete(op) + return this.releaseServiceLifecycleLock(serviceId) + }) + this.serviceOpPromises.add(op) + return await op + } + + // Runs fn under the per-service lifecycle lock (in-memory + cross-process DB lease), + // serializing it with the start pipeline, restart, stop and the expiry sweep. Used by + // handlers whose read-mutate-write flows (e.g. SERVICE_EXTEND) must not interleave + // with a concurrent teardown — throws "operation in progress" when the lock is busy. + public override async runExclusive( + serviceId: string, + fn: () => Promise + ): Promise { + await this.acquireServiceLifecycleLock(serviceId) + const op = fn().finally(() => { + this.serviceOpPromises.delete(op) + return this.releaseServiceLifecycleLock(serviceId) + }) + this.serviceOpPromises.add(op) + return await op + } + + // The actual stop. Must only run while holding the service lifecycle lock (see + // stopService above). + private async doStopService( + serviceId: string, + owner: string, + onlyIfExpired: boolean = false + ): Promise { + const [job] = await this.db.getServiceJob(serviceId, owner) + if (!job) { + CORE_LOGGER.debug(`stopService ${serviceId}: no job found for owner ${owner}`) + return null + } + // Expiry-sweep mode: re-validate expiry on the FRESH row now that we hold the lock. + // The sweep's expiredServices snapshot predates the lock, so a SERVICE_EXTEND may + // have pushed expiresAt into the future since — tearing down then would destroy a + // service the consumer just paid to prolong. The caller sees the untouched job and + // skips (its own expiresAt check). + if (onlyIfExpired && Date.now() < job.expiresAt) { + CORE_LOGGER.debug( + `stopService ${serviceId}: expiry teardown requested but the job was extended (expiresAt in the future) — skipping` + ) + return job + } + if ( + job.status === ServiceStatusNumber.Stopped || + job.status === ServiceStatusNumber.Expired + ) { + CORE_LOGGER.debug( + `stopService ${serviceId}: already "${job.statusText}" — nothing to tear down` + ) + return job + } + + CORE_LOGGER.debug( + `stopService ${serviceId}: stopping (status=${job.statusText}, ` + + `containerId=${job.containerId || '-'}, networkId=${job.networkId || '-'})` + ) + await this.logServiceDockerState(`stop ${serviceId}: before teardown`, serviceId) + job.status = ServiceStatusNumber.Stopping + job.statusText = ServiceStatusText[ServiceStatusNumber.Stopping] + await this.db.updateServiceJob(job) + + // Benign "already gone" errors count as success. Any other error means teardown + // genuinely failed — record it and keep the job OUT of Stopped so the persisted + // state stays accurate. + let cleanupError: Error | null = null + try { + if (job.containerId) { + const c = this.docker.getContainer(job.containerId) + await c.stop({ t: 10 }).catch((e) => { + if (!isBenignDockerError(e)) throw e + }) + await c.remove({ force: true }).catch((e) => { + if (!isBenignDockerError(e)) throw e + }) + } + await this.removeServiceNetwork(serviceId, job.networkId) + // Host ports deliberately NOT released: the consumer paid for the reservation + // until expiresAt and a Stopped service must be restartable anytime on the SAME + // endpoints. Only the expiry sweep releases them (after marking Expired). + } catch (err: any) { + cleanupError = err + CORE_LOGGER.error(`stopService ${serviceId} cleanup error: ${err.message}`) + } + + if (cleanupError) { + await this.logServiceDockerState( + `stop ${serviceId}: FAILED (${cleanupError.message}) — docker state at failure`, + serviceId + ) + job.status = ServiceStatusNumber.Error + job.statusText = `stop failed: ${cleanupError.message}` + } else { + job.status = ServiceStatusNumber.Stopped + job.statusText = ServiceStatusText[ServiceStatusNumber.Stopped] + // The docker objects are confirmed gone — drop the stale ids so nothing later + // mistakes them for live resources (a restart re-creates and re-persists both). + job.containerId = '' + job.networkId = '' + // Release the CPU pinning only now that the container is confirmed gone: on a + // failed teardown the old container may still be running on those cores, and + // handing them to another job would double-pin them. + this.releaseCpus(serviceId) + CORE_LOGGER.debug(`stopService ${serviceId}: stopped, container + network removed`) + } + await this.db.updateServiceJob(job) + return job + } + + public override async restartService( + serviceId: string, + owner: string, + newUserData?: string, + newDockerCmd?: string[], + newDockerEntrypoint?: string[] + ): Promise { + // Lifecycle lock: without it the InternalLoop's orphan-recovery (which sees the + // intermediate Restarting/PullImage/BuildImage status this method persists) tears + // down the network created here mid-restart → container.start() fails with + // "network ocean-svc- not found". + await this.acquireServiceLifecycleLock(serviceId) + + // Validate + persist Restarting synchronously so the caller gets immediate errors + // (not found / expired / refunded), then continue in the background: the image + // pull/build can take minutes and must not block the HTTP/P2P response. Clients + // poll SERVICE_GET_STATUS and watch Restarting → … → Running (or Error). + let job: ServiceJob + try { + ;[job] = await this.db.getServiceJob(serviceId, owner) + if (!job) { + CORE_LOGGER.debug(`restartService ${serviceId}: no job found for owner ${owner}`) + // early return bypasses both the catch below and the background op's finally — + // the lock must be handed back explicitly or it leaks forever + await this.releaseServiceLifecycleLock(serviceId) + return null + } + CORE_LOGGER.debug( + `restartService ${serviceId}: accepted (status=${job.statusText}, ` + + `containerId=${job.containerId || '-'}, networkId=${job.networkId || '-'}, ` + + `expiresAt=${new Date(job.expiresAt).toISOString()})` + ) + // Reject on the expiry timestamp too, not just the status: the expiry cron flips the + // status asynchronously, so a service can be past its paid window before it reads + // Expired. Restarting then would silently extend the service beyond what was paid for. + if (job.status === ServiceStatusNumber.Expired || Date.now() >= job.expiresAt) + throw new Error('Cannot restart an expired service') + // Only a CLAIMED payment makes a job restartable. Every legitimately restartable + // job (Running, container-died Error, Stopped) has claimTx set — claiming happens + // before the first container start. A job without it was never paid for: either + // the escrow lock failed outright (e.g. insufficient funds — all payment fields + // empty) or the lock was refunded (cancelTx set). Restarting either would run the + // service for free. The consumer must start a new service instead. + if (!job.payment?.claimTx) + throw new Error( + 'Cannot restart a service whose payment was never claimed (unpaid or refunded) — start a new service' + ) + // Persist Restarting BEFORE returning: status polls flip immediately, and a crash + // from here on leaves a pending-status record the boot loop orphan-recovers + // (Restarting is in SERVICE_START_PENDING_STATUSES) instead of a bare Starting + // record that would be double-started with a second escrow lock. + job.status = ServiceStatusNumber.Restarting + job.statusText = ServiceStatusText[ServiceStatusNumber.Restarting] + await this.db.updateServiceJob(job) + } catch (e) { + await this.releaseServiceLifecycleLock(serviceId) + throw e + } + + // Tracked like loop-launched starts so engine stop() drains an in-flight restart too. + const op = this.doRestartService( + job, + newUserData, + newDockerCmd, + newDockerEntrypoint + ).finally(() => { + this.serviceOpPromises.delete(op) + return this.releaseServiceLifecycleLock(serviceId) + }) + this.serviceOpPromises.add(op) + // The outcome is persisted on the job status (Running or Error + statusText); nobody + // awaits the background op, so swallow its rejection to avoid an unhandled rejection. + op.catch((e) => + CORE_LOGGER.debug(`restartService ${serviceId}: background op failed: ${e.message}`) + ) + // Snapshot for the immediate response — the DB record already says Restarting. + return job + } + + // The actual restart (teardown onwards). Runs in the background after restartService + // returned; must only run while holding the service lifecycle lock, with the job + // already validated and persisted as Restarting (see restartService above). + private async doRestartService( + job: ServiceJob, + newUserData?: string, + newDockerCmd?: string[], + newDockerEntrypoint?: string[] + ): Promise { + const { serviceId } = job + + // Re-reserve the job's host ports in the in-memory allocator: a Stopped service + // released them, so without this a concurrent start could be handed the same port + // this restart is about to re-bind. For a Running service the adds are no-ops (the + // ports are still reserved). + for (const ep of job.endpoints) reserveHostPort(ep.hostPort) + + await this.logServiceDockerState(`restart ${serviceId}: before teardown`, serviceId) + + // 1. Tear down existing container + network (best-effort). The job is already + // persisted as Restarting, so a crash anywhere in this method leaves a pending-status + // record that the boot loop orphan-recovers (refund-safe: the original payment's + // claimTx is set, so recovery never touches escrow for a restart). + if (job.containerId) { + CORE_LOGGER.debug(`restart ${serviceId}: removing old container ${job.containerId}`) + const c = this.docker.getContainer(job.containerId) + await c.stop({ t: 10 }).catch((e) => { + CORE_LOGGER.debug(`restart ${serviceId}: old container stop: ${e.message}`) + }) + await c.remove({ force: true }).catch((e) => { + CORE_LOGGER.debug(`restart ${serviceId}: old container remove: ${e.message}`) + }) + } + await this.removeServiceNetwork(serviceId, job.networkId).catch((e) => { + CORE_LOGGER.debug(`restart ${serviceId}: old network removal: ${e.message}`) + }) + await this.logServiceDockerState(`restart ${serviceId}: after teardown`, serviceId) + + job.containerId = '' + job.networkId = '' + await this.db.updateServiceJob(job) + + // Live Docker handles for the newly-created container/network, tracked so the + // catch block can tear them down on failure (otherwise the network leaks). + let network: Dockerode.Network | null = null + let container: Dockerode.Container | null = null + try { + const sod = this.getC2DConfig().connection?.serviceOnDemand + + // 2. Pull or rebuild image based on how the service was originally started + // (the original Dockerfile + build files are stored on the job). + if (job.dockerfile) { + job.status = ServiceStatusNumber.BuildImage + job.statusText = ServiceStatusText[ServiceStatusNumber.BuildImage] + await this.db.updateServiceJob(job) + const ram = job.resources.find((r) => r.id === 'ram')?.amount + const cpu = job.resources.find((r) => r.id === 'cpu')?.amount + await this.buildImageFromSpec( + job.containerImage, + job.dockerfile, + job.additionalDockerFiles ?? {}, + sod?.maxDurationSeconds ?? job.duration, + ram, + cpu + ) + } else { + job.status = ServiceStatusNumber.PullImage + job.statusText = ServiceStatusText[ServiceStatusNumber.PullImage] + await this.db.updateServiceJob(job) + await this.pullImageRef(job.containerImage) + } + + // 3. Effective overrides: each REPLACES the stored value when supplied (even with an + // empty value), and falls back to the stored one when omitted (undefined). + const effectiveUserData = newUserData ?? job.userData + const effectiveDockerCmd = newDockerCmd !== undefined ? newDockerCmd : job.dockerCmd + const effectiveDockerEntrypoint = + newDockerEntrypoint !== undefined ? newDockerEntrypoint : job.dockerEntrypoint + const decryptedUserData = await decryptUserData(effectiveUserData, this.keyManager) + + // 4. Rebuild env (from userData) + command/entrypoint + const env = userDataToEnv(decryptedUserData) + const cmd = effectiveDockerCmd?.length ? effectiveDockerCmd : undefined + const entrypoint = effectiveDockerEntrypoint?.length + ? effectiveDockerEntrypoint + : undefined + + // 5. Rebuild port bindings — reuse already-allocated host ports + const PortBindings: Record> = {} + const ExposedPorts: Record> = {} + job.endpoints.forEach((ep) => { + PortBindings[`${ep.containerPort}/tcp`] = [{ HostPort: String(ep.hostPort) }] + ExposedPorts[`${ep.containerPort}/tcp`] = {} + }) + + // 6. New per-service network + network = await this.createServiceNetwork(serviceId) + CORE_LOGGER.debug(`restart ${serviceId}: created network ${network.id}`) + + // 7. Resource constraints + const { Memory, NanoCpus, DeviceRequests, CpusetCpus } = + this.buildServiceResourceConstraints( + job.resources.map((r) => ({ id: r.id, amount: r.amount })), + serviceId, + job.environment + ) + + // 8. Create and start new container + container = await this.docker.createContainer({ + Image: job.containerImage, + Cmd: cmd, + Entrypoint: entrypoint, + Env: Object.entries(env).map(([k, v]) => `${k}=${v}`), + ExposedPorts, + HostConfig: { + Memory, + NanoCpus, + DeviceRequests, + CpusetCpus, + PortBindings, + NetworkMode: network.id, + SecurityOpt: ['no-new-privileges'], + CapDrop: ['ALL'], + PidsLimit: 512 + } + }) + CORE_LOGGER.debug( + `restart ${serviceId}: created container ${container.id} on network ${network.id} — starting` + ) + await container.start() + CORE_LOGGER.debug(`restart ${serviceId}: container ${container.id} started`) + + // 9. Update record — same expiresAt, same payment, new container/network. + job.containerId = container.id + job.networkId = network.id + job.userData = effectiveUserData + job.dockerCmd = effectiveDockerCmd + job.dockerEntrypoint = effectiveDockerEntrypoint + job.status = ServiceStatusNumber.Running + job.statusText = ServiceStatusText[ServiceStatusNumber.Running] + await this.db.updateServiceJob(job) + CORE_LOGGER.info( + `restart ${serviceId}: completed — container ${container.id} Running on ` + + `ports [${job.endpoints.map((ep) => ep.hostPort).join(',')}]` + ) + return job + } catch (err: any) { + // Dump the daemon state BEFORE cleanup, so the log shows what the failure saw + // (e.g. whether ocean-svc- actually existed when container.start() ran). + await this.logServiceDockerState( + `restart ${serviceId}: FAILED (${err.message}) — docker state at failure`, + serviceId + ) + await this.cleanupServiceDocker(container, network, serviceId) + // Ports deliberately stay reserved: the consumer paid until expiresAt and may + // restart again — the expiry sweep releases them when the window elapses. + job.status = ServiceStatusNumber.Error + job.statusText = String(err.message) + await this.db.updateServiceJob(job) + CORE_LOGGER.error(`restartService ${serviceId} failed: ${err.message}`) + throw err + } + } + + public override async getServiceStatus( + consumerAddress?: string, + serviceId?: string + ): Promise { + const jobs = await this.db.getServiceJob(serviceId, consumerAddress) + return jobs.filter((j) => j.clusterHash === this.getC2DConfig().hash) + } + + public override async getServiceStreamableLogs( + serviceId: string, + owner: string, + since?: number + ): Promise { + const [job] = await this.db.getServiceJob(serviceId, owner) + if (!job) return null + if ( + job.status !== ServiceStatusNumber.Running && + job.status !== ServiceStatusNumber.Error + ) { + return null + } + try { + const container = this.docker.getContainer(job.containerId) + await container.inspect() // throws if the container no longer exists + return (await container.logs({ + stdout: true, + stderr: true, + follow: true, + ...(since !== undefined ? { since } : {}) + })) as unknown as NodeJS.ReadableStream + } catch (e: any) { + CORE_LOGGER.error( + `getServiceStreamableLogs failed for ${serviceId}: ${e?.message ?? e}` + ) + return null + } + } + private addUserDataToFilesObject( filesObject: any, userData: { [key: string]: any } diff --git a/src/components/c2d/compute_engines.ts b/src/components/c2d/compute_engines.ts index beab7002a..296ff22f3 100644 --- a/src/components/c2d/compute_engines.ts +++ b/src/components/c2d/compute_engines.ts @@ -10,15 +10,22 @@ import { C2DDatabase } from '../database/C2DDatabase.js' import { Escrow } from '../core/utils/escrow.js' import { KeyManager } from '../KeyManager/index.js' import { CORE_LOGGER } from '../../utils/logging/common.js' +import type { + ServiceTemplate, + ServiceTemplatePublic +} from '../../@types/C2D/ServiceOnDemand.js' +import { loadServiceTemplates } from '../core/service/templateLoader.js' export class C2DEngines { public engines: C2DEngine[] + private config: OceanNodeConfig public constructor( config: OceanNodeConfig, db: C2DDatabase, escrow: Escrow, keyManager: KeyManager ) { + this.config = config const crons = { imageCleanup: false, scanDBUpdate: false @@ -130,6 +137,23 @@ export class C2DEngines { throw new Error(`C2D Engine not found by id: ${envId}`) } + // Called by SERVICE_GET_TEMPLATES handler. Loads + validates templates from the + // serviceTemplatesPath folder and returns them sanitized (envVars values stripped to keys). + // Choosing a compatible compute environment is the client's responsibility — the node + // exposes environments + their resources via GET_COMPUTE_ENVIRONMENTS. + async fetchServiceTemplates(): Promise { + const templates: ServiceTemplate[] = await loadServiceTemplates( + this.config?.serviceTemplatesPath + ) + return templates.map((tmpl) => { + const { envVars, ...rest } = tmpl + return { + ...rest, + ...(envVars ? { envVarKeys: Object.keys(envVars) } : {}) + } + }) + } + async fetchEnvironments( chainId?: number, engine?: C2DEngine diff --git a/src/components/c2d/serviceResourceMatching.ts b/src/components/c2d/serviceResourceMatching.ts new file mode 100644 index 000000000..6303a6ad7 --- /dev/null +++ b/src/components/c2d/serviceResourceMatching.ts @@ -0,0 +1,13 @@ +// Resolves the final Docker image reference from a service start spec. +// Priority: dockerfile > checksum > tag > default "latest". +export function resolveServiceImage( + image: string, + tag?: string, + checksum?: string, + dockerfile?: string, + serviceId?: string +): string { + if (dockerfile) return `${serviceId!.toLowerCase()}-svc-image:latest` + if (checksum) return `${image}@${checksum}` + return `${image}:${tag ?? 'latest'}` +} diff --git a/src/components/core/compute/startCompute.ts b/src/components/core/compute/startCompute.ts index 5d4029a5d..ba3e7863e 100644 --- a/src/components/core/compute/startCompute.ts +++ b/src/components/core/compute/startCompute.ts @@ -136,6 +136,15 @@ export class PaidComputeStartHandler extends CommonComputeHandler { } } } + if (env.features?.computeJobs === false) { + return { + stream: null, + status: { + httpStatus: 403, + error: 'Compute jobs are not enabled on this environment' + } + } + } if (!task.maxJobDuration || task.maxJobDuration > env.maxJobDuration) { task.maxJobDuration = env.maxJobDuration } @@ -950,6 +959,15 @@ export class FreeComputeStartHandler extends CommonComputeHandler { } } } + if (env.features?.computeJobs === false) { + return { + stream: null, + status: { + httpStatus: 403, + error: 'Compute jobs are not enabled on this environment' + } + } + } try { const accessGranted = await validateAccess( task.consumerAddress, @@ -1062,7 +1080,7 @@ export class FreeComputeStartHandler extends CommonComputeHandler { } } -async function validateAccess( +export async function validateAccess( consumerAddress: string, access: ComputeAccessList | undefined, oceanNode: OceanNode diff --git a/src/components/core/handler/authHandler.ts b/src/components/core/handler/authHandler.ts index 2a53a6aa1..62731a559 100644 --- a/src/components/core/handler/authHandler.ts +++ b/src/components/core/handler/authHandler.ts @@ -8,6 +8,7 @@ import { ReadableString } from '../../P2P/handlers.js' import { Command } from '../../../@types/commands.js' import { Readable } from 'stream' import { checkNonce, NonceResponse } from '../utils/nonceHandler.js' +import jwt from 'jsonwebtoken' export interface AuthMessage { address: string @@ -57,9 +58,17 @@ export class CreateAuthTokenHandler extends CommandHandler { } const createdAt = Date.now() + const issuerPeerId = this.getOceanNode().getKeyManager().getPeerIdString() const jwtToken = await this.getOceanNode() .getAuth() - .getJWTToken(task.address, task.nonce, createdAt) + .getJWTToken( + task.address, + task.nonce, + createdAt, + signature, + issuerPeerId, + task.chainId + ) await this.getOceanNode() .getAuth() @@ -92,7 +101,7 @@ export class InvalidateAuthTokenHandler extends CommandHandler { } try { - const isValid = await checkNonce( + const nonceCheckResult = await checkNonce( this.getOceanNode().getConfig(), nonceDb, address, @@ -101,7 +110,7 @@ export class InvalidateAuthTokenHandler extends CommandHandler { task.command, task.chainId ) - if (!isValid) { + if (!nonceCheckResult.valid) { return { stream: null, status: { httpStatus: 400, error: 'Invalid signature' } @@ -122,3 +131,47 @@ export class InvalidateAuthTokenHandler extends CommandHandler { } } } + +export interface ValidateAuthTokenCommand extends Command { + token: string +} + +export class ValidateAuthTokenHandler extends CommandHandler { + validate(command: ValidateAuthTokenCommand): ValidateParams { + return validateCommandParameters(command, ['token']) + } + + async handle(task: ValidateAuthTokenCommand): Promise { + const validationResponse = await this.verifyParamsAndRateLimits(task) + if (this.shouldDenyTaskHandling(validationResponse)) { + return validationResponse + } + + try { + const auth = this.getOceanNode().getAuth() + let verified = true + try { + jwt.verify(task.token, auth.getJwtSecret()) + } catch { + verified = false + } + const row = verified ? await auth.getLocalToken(task.token) : null + const body = row + ? { + valid: true, + validUntil: row.validUntil == null ? null : new Date(row.validUntil).getTime() + } + : { valid: false } + + return { + stream: Readable.from(JSON.stringify(body)), + status: { httpStatus: 200, error: null } + } + } catch (error) { + return { + stream: null, + status: { httpStatus: 500, error: `Error validating auth token: ${error}` } + } + } + } +} diff --git a/src/components/core/handler/coreHandlersRegistry.ts b/src/components/core/handler/coreHandlersRegistry.ts index 55663c96f..c9c322b25 100644 --- a/src/components/core/handler/coreHandlersRegistry.ts +++ b/src/components/core/handler/coreHandlersRegistry.ts @@ -46,7 +46,11 @@ import { GetP2PNetworkStatsHandler, FindPeerHandler } from './p2p.js' -import { CreateAuthTokenHandler, InvalidateAuthTokenHandler } from './authHandler.js' +import { + CreateAuthTokenHandler, + InvalidateAuthTokenHandler, + ValidateAuthTokenHandler +} from './authHandler.js' import { GetJobsHandler } from './getJobs.js' import { PersistentStorageCreateBucketHandler, @@ -59,6 +63,16 @@ import { } from './persistentStorage.js' import { GetAccessListHandler, SearchAccessListHandler } from './accessListHandler.js' import { EscrowEventsHandler } from './escrowHandler.js' +import { + ServiceGetTemplatesHandler, + ServiceStartHandler, + ServiceStopHandler, + ServiceExtendHandler, + ServiceRestartHandler, + ServiceGetStatusHandler, + GetServicesHandler, + ServiceGetStreamableLogsHandler +} from '../service/index.js' export type HandlerRegistry = { handlerName: string // name of the handler @@ -146,6 +160,32 @@ export class CoreHandlersRegistry { PROTOCOL_COMMANDS.COMPUTE_INITIALIZE, new ComputeInitializeHandler(node) ) + this.registerCoreHandler( + PROTOCOL_COMMANDS.SERVICE_GET_TEMPLATES, + new ServiceGetTemplatesHandler(node) + ) + this.registerCoreHandler( + PROTOCOL_COMMANDS.SERVICE_START, + new ServiceStartHandler(node) + ) + this.registerCoreHandler(PROTOCOL_COMMANDS.SERVICE_STOP, new ServiceStopHandler(node)) + this.registerCoreHandler( + PROTOCOL_COMMANDS.SERVICE_EXTEND, + new ServiceExtendHandler(node) + ) + this.registerCoreHandler( + PROTOCOL_COMMANDS.SERVICE_RESTART, + new ServiceRestartHandler(node) + ) + this.registerCoreHandler( + PROTOCOL_COMMANDS.SERVICE_GET_STATUS, + new ServiceGetStatusHandler(node) + ) + this.registerCoreHandler(PROTOCOL_COMMANDS.SERVICE_LIST, new GetServicesHandler(node)) + this.registerCoreHandler( + PROTOCOL_COMMANDS.SERVICE_GET_STREAMABLE_LOGS, + new ServiceGetStreamableLogsHandler(node) + ) this.registerCoreHandler(PROTOCOL_COMMANDS.STOP_NODE, new StopNodeHandler(node)) this.registerCoreHandler(PROTOCOL_COMMANDS.STOP_JOB, new StopJobHandler(node)) this.registerCoreHandler(PROTOCOL_COMMANDS.REINDEX_TX, new ReindexTxHandler(node)) @@ -176,6 +216,10 @@ export class CoreHandlersRegistry { PROTOCOL_COMMANDS.INVALIDATE_AUTH_TOKEN, new InvalidateAuthTokenHandler(node) ) + this.registerCoreHandler( + PROTOCOL_COMMANDS.VALIDATE_AUTH_TOKEN, + new ValidateAuthTokenHandler(node) + ) this.registerCoreHandler(PROTOCOL_COMMANDS.FETCH_CONFIG, new FetchConfigHandler(node)) this.registerCoreHandler(PROTOCOL_COMMANDS.PUSH_CONFIG, new PushConfigHandler(node)) this.registerCoreHandler(PROTOCOL_COMMANDS.GET_LOGS, new GetLogsHandler(node)) diff --git a/src/components/core/handler/handler.ts b/src/components/core/handler/handler.ts index 4f6cd2503..3c0b1daeb 100644 --- a/src/components/core/handler/handler.ts +++ b/src/components/core/handler/handler.ts @@ -174,22 +174,13 @@ export abstract class CommandHandler } } - async getAddressFromToken(authToken: string): Promise { - const auth = this.getOceanNode().getAuth() - if (!auth) { - throw new Error('Auth not configured') - } - - return (await auth.validateToken(authToken)).address - } - async validateTokenOrSignature( authToken: string, address: string, nonce: string, signature: string, command: string - ): Promise { + ): Promise { const oceanNode = this.getOceanNode() const auth = oceanNode.getAuth() if (!auth) { @@ -214,7 +205,8 @@ export abstract class CommandHandler return { stream: null, - status: { httpStatus: 200 } + status: { httpStatus: 200 }, + consumerAddress: isAuthRequestValid.address } } } diff --git a/src/components/core/handler/persistentStorage.ts b/src/components/core/handler/persistentStorage.ts index 02b89fe0a..8f650f2ff 100644 --- a/src/components/core/handler/persistentStorage.ts +++ b/src/components/core/handler/persistentStorage.ts @@ -88,7 +88,7 @@ export class PersistentStorageCreateBucketHandler extends CommandHandler { config.persistentStorage?.accessLists.length > 0 ) { const isAllowedCreate = await checkAddressOnAccessList( - task.consumerAddress, + isAuthRequestValid.consumerAddress, config.persistentStorage?.accessLists, node ) @@ -105,9 +105,7 @@ export class PersistentStorageCreateBucketHandler extends CommandHandler { let ownerNormalized: string try { - ownerNormalized = task.consumerAddress - ? getAddress(task.consumerAddress) - : getAddress(await this.getAddressFromToken(task.authorization)) + ownerNormalized = getAddress(isAuthRequestValid.consumerAddress) } catch { return { stream: null, @@ -159,9 +157,7 @@ export class PersistentStorageUpdateBucketHandler extends CommandHandler { try { const storage = requirePersistentStorage(this) - const ownerNormalized = task.consumerAddress - ? getAddress(task.consumerAddress) - : getAddress(await this.getAddressFromToken(task.authorization)) + const ownerNormalized = getAddress(isAuthRequestValid.consumerAddress) const label = await storage.updateBucketLabel( task.bucketId, task.label, @@ -267,9 +263,7 @@ export class PersistentStorageListFilesHandler extends CommandHandler { try { const storage = requirePersistentStorage(this) - const ownerNormalized = task.consumerAddress - ? getAddress(task.consumerAddress) - : getAddress(await this.getAddressFromToken(task.authorization)) + const ownerNormalized = getAddress(isAuthRequestValid.consumerAddress) const result = await storage.listFiles(task.bucketId, ownerNormalized) return { stream: Readable.from(JSON.stringify(result)), @@ -311,9 +305,7 @@ export class PersistentStorageGetFileObjectHandler extends CommandHandler { try { const storage = requirePersistentStorage(this) - const ownerNormalized = task.consumerAddress - ? getAddress(task.consumerAddress) - : getAddress(await this.getAddressFromToken(task.authorization)) + const ownerNormalized = getAddress(isAuthRequestValid.consumerAddress) const obj = await storage.getFileObject( task.bucketId, task.fileName, @@ -369,9 +361,7 @@ export class PersistentStorageUploadFileHandler extends CommandHandler { status: { httpStatus: 403, error: 'Upload stream error' } } } - const ownerNormalized = task.consumerAddress - ? getAddress(task.consumerAddress) - : getAddress(await this.getAddressFromToken(task.authorization)) + const ownerNormalized = getAddress(isAuthRequestValid.consumerAddress) const result = await storage.uploadFile( task.bucketId, task.fileName, @@ -419,9 +409,7 @@ export class PersistentStorageDeleteFileHandler extends CommandHandler { try { const storage = requirePersistentStorage(this) - const ownerNormalized = task.consumerAddress - ? getAddress(task.consumerAddress) - : getAddress(await this.getAddressFromToken(task.authorization)) + const ownerNormalized = getAddress(isAuthRequestValid.consumerAddress) await storage.deleteFile(task.bucketId, task.fileName, ownerNormalized) return { stream: Readable.from(JSON.stringify({ success: true })), diff --git a/src/components/core/service/extendService.ts b/src/components/core/service/extendService.ts new file mode 100644 index 000000000..ec34e71ad --- /dev/null +++ b/src/components/core/service/extendService.ts @@ -0,0 +1,354 @@ +import { Readable } from 'stream' +import { P2PCommandResponse } from '../../../@types/index.js' +import { ServiceExtendCommand } from '../../../@types/commands.js' +import { CommandHandler } from '../handler/handler.js' +import { + ValidateParams, + validateCommandParameters, + buildInvalidParametersResponse, + buildInvalidRequestMessage +} from '../../httpRoutes/validateCommands.js' +import { CORE_LOGGER } from '../../../utils/logging/common.js' +import type { ComputeEnvironment } from '../../../@types/C2D/C2D.js' +import { ServiceStatusNumber } from '../../../@types/C2D/ServiceOnDemand.js' +import { validateAccess } from '../compute/startCompute.js' +import { findServiceJobAndEngine, toPublicServiceJob } from './utils.js' + +export class ServiceExtendHandler extends CommandHandler { + validate(command: ServiceExtendCommand): ValidateParams { + const commandValidation = validateCommandParameters(command, [ + 'consumerAddress', + 'serviceId', + 'additionalDuration', + 'payment' + ]) + if (commandValidation.valid) { + if (parseInt(String(command.additionalDuration)) <= 0) + return buildInvalidRequestMessage('Invalid additionalDuration') + } + return commandValidation + } + + async handle(task: ServiceExtendCommand): Promise { + const validationResponse = await this.verifyParamsAndRateLimits(task) + if (this.shouldDenyTaskHandling(validationResponse)) return validationResponse + + const auth = await this.validateTokenOrSignature( + task.authorization, + task.consumerAddress, + task.nonce, + task.signature, + task.command + ) + if (auth.status.httpStatus !== 200) return auth + + const engines = this.getOceanNode().getC2DEngines() + if (!engines) + return { + stream: null, + status: { httpStatus: 503, error: 'Compute engines not configured' } + } + + // Find the job and the engine that owns it (by clusterHash — see helper) + const { job, engine } = await findServiceJobAndEngine( + engines, + task.serviceId, + task.consumerAddress + ) + if (!job) + return buildInvalidParametersResponse( + buildInvalidRequestMessage('Service job not found: ' + task.serviceId) + ) + if (!engine) + return { + stream: null, + status: { + httpStatus: 500, + error: `No compute engine owns service ${task.serviceId} (cluster ${job.clusterHash}) — the node's compute configuration may have changed` + } + } + + // Ownership check + if (job.owner.toLowerCase() !== task.consumerAddress.toLowerCase()) + return { stream: null, status: { httpStatus: 401, error: 'Not the service owner' } } + + // Resolve the environment the service actually runs on. This MUST exist: both the + // access gate and pricing key off it. A missing env would otherwise let validateAccess + // auto-allow (undefined access → true) and pricing fall back to an unrelated env. + const runEnv: ComputeEnvironment | undefined = ( + await engine.getComputeEnvironments() + ).find((e) => e.id === job!.environment) + if (!runEnv) + return buildInvalidParametersResponse( + buildInvalidRequestMessage(`Service environment "${job.environment}" not found`) + ) + + // Access-list gate (mirrors paid compute → 403). Re-checked here because access + // lists are mutable and extending prolongs use of the restricted environment. + const accessGranted = await validateAccess( + task.consumerAddress, + runEnv.access, + this.getOceanNode() + ) + if (!accessGranted) + return { stream: null, status: { httpStatus: 403, error: 'Access denied' } } + + // Everything from the state check to the final write runs under the per-service + // lifecycle lock: extend is a read-mutate-write of the job row, and without the lock + // its final updateServiceJob could overwrite a concurrent stop/restart/Expired state + // (and the expiry sweep could tear the service down between our payment and write). + // The lock is taken BEFORE any escrow operation, so a busy lock costs nothing. + try { + return await engine.runExclusive( + task.serviceId, + async (): Promise => { + // Re-read under the lock — every other mutator holds the same lock, so this + // snapshot cannot go stale before our write. + const [freshJob] = await engine.db.getServiceJob( + task.serviceId, + task.consumerAddress + ) + if (!freshJob) + return buildInvalidParametersResponse( + buildInvalidRequestMessage('Service job not found: ' + task.serviceId) + ) + + // State check — only Starting or Running can be extended + if ( + freshJob.status !== ServiceStatusNumber.Starting && + freshJob.status !== ServiceStatusNumber.Running + ) + return buildInvalidParametersResponse( + buildInvalidRequestMessage( + `Cannot extend a service in state "${freshJob.statusText}". Only Starting or Running services can be extended.` + ) + ) + + // Extension must not push total beyond maxDurationSeconds + const sod = engine.getC2DConfig().connection?.serviceOnDemand + const maxDuration = sod?.maxDurationSeconds ?? 86400 + const remainingSeconds = Math.max( + 0, + Math.floor((freshJob.expiresAt - Date.now()) / 1000) + ) + const newTotalDuration = remainingSeconds + task.additionalDuration + if (newTotalDuration > maxDuration) + return buildInvalidParametersResponse( + buildInvalidRequestMessage( + `Extension would result in ${newTotalDuration}s remaining, exceeding maximum ${maxDuration}s` + ) + ) + + // Cost — same price formula as the start, priced off the env the service runs + // on. No fallback: pricing must use runEnv (resolved above); + // calculateResourcesCost returns null if that env has no pricing for the token. + const costExtend = engine.calculateResourcesCost( + freshJob.resources.map((r) => ({ id: r.id, amount: r.amount })), + runEnv, + task.payment.chainId, + task.payment.token, + task.additionalDuration + ) + if (costExtend === null) + return buildInvalidParametersResponse( + buildInvalidRequestMessage( + `No pricing configured for token ${task.payment.token} on chain ${task.payment.chainId}` + ) + ) + + // An extendPayments entry with a lockTx but neither claimTx nor cancelTx is an + // UNRESOLVED intent from a previous crash (see below — the intent is persisted + // before claiming). Resolve it before charging again: try to cancel (refund) + // the old lock; if that fails the lock was most likely already claimed and a + // human must reconcile — reject rather than risk a double charge. + const unresolved = (freshJob.extendPayments ?? []).find( + (p) => p.lockTx && !p.claimTx && !p.cancelTx + ) + if (unresolved) { + try { + const cancelTx = await engine.escrow.cancelExpiredLock( + unresolved.chainId, + task.serviceId, + unresolved.token, + task.consumerAddress + ) + if (!cancelTx) throw new Error('cancelExpiredLock returned no tx') + unresolved.cancelTx = cancelTx + await engine.db.updateServiceJob(freshJob) + CORE_LOGGER.logMessage( + `Service ${task.serviceId}: refunded unresolved extension lock ${unresolved.lockTx} (${cancelTx})`, + true + ) + } catch (e: any) { + CORE_LOGGER.error( + `Service ${task.serviceId}: unresolved extension intent (lock ${unresolved.lockTx}) could not be refunded: ${e.message}` + ) + return { + stream: null, + status: { + httpStatus: 409, + error: `A previous extension of this service is unresolved (lock ${unresolved.lockTx}) and could not be auto-refunded — retry later or contact the node operator` + } + } + } + } + + // Escrow lock + immediate claim + let lockTx: string | null + try { + lockTx = await engine.escrow.createLock( + task.payment.chainId, + task.serviceId, + task.payment.token, + task.consumerAddress, + costExtend, + engine.escrow.getMinLockTime(task.additionalDuration) + ) + } catch (e: any) { + CORE_LOGGER.error(`Service extend createLock failed: ${e.message}`) + return { stream: null, status: { httpStatus: 402, error: e.message } } + } + if (!lockTx) + return { + stream: null, + status: { httpStatus: 402, error: 'Escrow lock failed for extend' } + } + + // Wait for the lock tx to be mined before claiming (same-signer back-to-back txs). + try { + await engine.escrow.waitForTransaction(task.payment.chainId, lockTx) + } catch (e: any) { + CORE_LOGGER.error(`Service extend lock not confirmed: ${e.message}`) + await engine.escrow + .cancelExpiredLock( + task.payment.chainId, + task.serviceId, + task.payment.token, + task.consumerAddress + ) + .catch((err) => + CORE_LOGGER.error(`cancelExpiredLock failed: ${err.message}`) + ) + return { + stream: null, + status: { + httpStatus: 402, + error: 'Escrow lock not confirmed — lock cancelled' + } + } + } + + // Persist the extension INTENT (lockTx recorded, claim pending) BEFORE + // claiming: a crash between claim and the final write is then auditable — the + // consumer's money can never be taken without a durable record of why — and a + // retry finds the intent (unresolved branch above) instead of charging twice. + const intent = { + chainId: task.payment.chainId, + token: task.payment.token, + lockTx, + claimTx: '', + cancelTx: '', + cost: costExtend + } + freshJob.extendPayments = [...(freshJob.extendPayments ?? []), intent] + try { + await engine.db.updateServiceJob(freshJob) + } catch (persistErr: any) { + // The lock is mined but we could NOT make it durable — without a record, the + // unresolved-intent recovery would never find it. Compensate: refund the + // lock now. If even that fails, funds are stranded on-chain → 409 so a + // human reconciles; nothing was claimed either way. + CORE_LOGGER.error( + `Service extend: intent persistence failed (${persistErr.message}) — refunding lock ${lockTx}` + ) + freshJob.extendPayments = freshJob.extendPayments.filter((p) => p !== intent) + const cancelTx = await engine.escrow + .cancelExpiredLock( + task.payment.chainId, + task.serviceId, + task.payment.token, + task.consumerAddress + ) + .catch((e): string | null => { + CORE_LOGGER.error(`cancelExpiredLock failed: ${e.message}`) + return null + }) + if (!cancelTx) + return { + stream: null, + status: { + httpStatus: 409, + error: `Extension aborted: the payment intent could not be persisted AND the escrow lock ${lockTx} could not be refunded — contact the node operator` + } + } + return { + stream: null, + status: { + httpStatus: 402, + error: + 'Extension aborted before charging: the payment intent could not be persisted — the escrow lock was refunded' + } + } + } + + let claimTx: string | null + try { + claimTx = await engine.escrow.claimLock( + task.payment.chainId, + task.serviceId, + task.payment.token, + task.consumerAddress, + costExtend, + `service-extend:${task.serviceId}` + ) + } catch (e: any) { + claimTx = null + CORE_LOGGER.error(`Service extend claimLock failed: ${e.message}`) + } + if (!claimTx) { + const cancelTx = await engine.escrow + .cancelExpiredLock( + task.payment.chainId, + task.serviceId, + task.payment.token, + task.consumerAddress + ) + .catch((e): string | null => { + CORE_LOGGER.error(`cancelExpiredLock failed: ${e.message}`) + return null + }) + // close the intent (refunded) — or leave it unresolved for the next attempt + if (cancelTx) { + intent.cancelTx = cancelTx + await engine.db.updateServiceJob(freshJob) + } + return { + stream: null, + status: { httpStatus: 402, error: 'Escrow claim failed — lock cancelled' } + } + } + + // Payment successful — finalize the intent and push expiresAt forward + intent.claimTx = claimTx + freshJob.expiresAt += task.additionalDuration * 1000 + freshJob.duration += task.additionalDuration + await engine.db.updateServiceJob(freshJob) + + CORE_LOGGER.logMessage( + `Service ${task.serviceId} extended by ${task.additionalDuration}s, new expiresAt: ${freshJob.expiresAt}`, + true + ) + return { + stream: Readable.from(JSON.stringify([toPublicServiceJob(freshJob)])), + status: { httpStatus: 200 } + } + } + ) + } catch (error: any) { + // Lifecycle lock busy (a stop/restart/expiry owns the service) or engine stopped — + // nothing was charged yet, the client simply retries shortly. + CORE_LOGGER.error(`ServiceExtend ${task.serviceId} rejected: ${error.message}`) + return { stream: null, status: { httpStatus: 400, error: error.message } } + } + } +} diff --git a/src/components/core/service/getServices.ts b/src/components/core/service/getServices.ts new file mode 100644 index 000000000..c5fdb3e4c --- /dev/null +++ b/src/components/core/service/getServices.ts @@ -0,0 +1,135 @@ +import { Readable } from 'stream' +import { P2PCommandResponse } from '../../../@types/index.js' +import { GetServicesCommand } from '../../../@types/commands.js' +import { CommandHandler } from '../handler/handler.js' +import { + ValidateParams, + validateCommandParameters, + buildInvalidRequestMessage +} from '../../httpRoutes/validateCommands.js' +import { + ServiceStatusNumber, + type ServiceJob +} from '../../../@types/C2D/ServiceOnDemand.js' +import { toListedServiceJob } from './utils.js' + +// Parses the `fromTimestamp` filter into Unix milliseconds. Accepts an ISO date string +// or a Unix timestamp (seconds or milliseconds) given as a string / number-like string. +// Returns undefined for "no filter" and null for an unparseable value (caller → 400). +export function parseFromTimestamp(value?: string): number | undefined | null { + if (value === undefined || value === null || value === '') return undefined + if (/^\d+$/.test(String(value))) { + const n = Number(value) + // 1e12 ms ≈ Sep 2001; any plausible seconds value is far below it + return n > 1e12 ? n : n * 1000 + } + const t = Date.parse(String(value)) + return Number.isNaN(t) ? null : t +} + +// SERVICE_LIST: the node-wide service listing, shaped like GetJobsHandler. Default (no +// filters) returns exactly what the engines count against the shared resource pools +// (getRunningServiceJobs): Running/Restarting/Stopping, the mid-start pipeline states, +// paid Error (container died, restartable), and explicitly Stopped (reservation kept +// until expiresAt). `status` narrows to ONE specific status (any, incl. Expired); +// `includeAllStatuses` returns everything; `fromTimestamp` keeps only services created +// at/after that moment. Unlike SERVICE_GET_STATUS this is NOT owner-scoped: any +// authenticated caller sees every consumer's services, so the output is listing-grade +// sanitized (no userData, no CMD/ENTRYPOINT overrides, no Dockerfile). +export class GetServicesHandler extends CommandHandler { + validate(command: GetServicesCommand): ValidateParams { + // consumerAddress is required: it is the identity the signature/token is verified + // against (same contract as the other service commands). + const validation = validateCommandParameters(command, ['consumerAddress']) + if (!validation.valid) return validation + if ( + command.status !== undefined && + ServiceStatusNumber[command.status] === undefined + ) { + return buildInvalidRequestMessage( + `Parameter "status" is not a valid service status number: ${command.status}` + ) + } + if (command.fromTimestamp !== undefined) { + if (typeof command.fromTimestamp !== 'string') + return buildInvalidRequestMessage( + 'Parameter "fromTimestamp" is not a valid string' + ) + if (!Number.isFinite(parseFromTimestamp(command.fromTimestamp))) + return buildInvalidRequestMessage( + `Parameter "fromTimestamp" is not a valid date: "${command.fromTimestamp}" — use an ISO date or a Unix timestamp` + ) + } + if (command.updatedSince !== undefined) { + if (typeof command.updatedSince !== 'string') + return buildInvalidRequestMessage( + 'Parameter "updatedSince" is not a valid string' + ) + if (!Number.isFinite(parseFromTimestamp(command.updatedSince))) + return buildInvalidRequestMessage( + `Parameter "updatedSince" is not a valid date: "${command.updatedSince}" — use an ISO date or a Unix timestamp` + ) + } + return validation + } + + async handle(task: GetServicesCommand): Promise { + const validationResponse = await this.verifyParamsAndRateLimits(task) + if (this.shouldDenyTaskHandling(validationResponse)) return validationResponse + + const auth = await this.validateTokenOrSignature( + task.authorization, + task.consumerAddress, + task.nonce, + task.signature, + task.command + ) + if (auth.status.httpStatus !== 200) return auth + + const engines = this.getOceanNode().getC2DEngines() + if (!engines) + return { + stream: null, + status: { httpStatus: 503, error: 'Compute engines not configured' } + } + + const jobs: ServiceJob[] = [] + for (const eng of engines.getAllEngines()) { + const { hash } = eng.getC2DConfig() + if ( + task.updatedSince !== undefined || + task.status !== undefined || + task.includeAllStatuses + ) { + const all = (await eng.db.getServiceJob()).filter( + (j: ServiceJob) => j.clusterHash === hash + ) + jobs.push( + ...(task.status !== undefined + ? all.filter((j: ServiceJob) => j.status === task.status) + : all) + ) + } else { + // default: the cluster's resource-holding set + jobs.push(...(await eng.db.getRunningServiceJobs(hash))) + } + } + + const fromMs = parseFromTimestamp(task.fromTimestamp) + const updatedSinceMs = parseFromTimestamp(task.updatedSince) + const filtered = jobs.filter( + (j) => + (fromMs === undefined || fromMs === null + ? true + : Date.parse(j.dateCreated) >= fromMs) && + (updatedSinceMs === undefined || updatedSinceMs === null + ? true + : (j.updatedAt ?? 0) >= updatedSinceMs) + ) + + return { + stream: Readable.from(JSON.stringify(filtered.map(toListedServiceJob))), + status: { httpStatus: 200 } + } + } +} diff --git a/src/components/core/service/getStatus.ts b/src/components/core/service/getStatus.ts new file mode 100644 index 000000000..a141b3e40 --- /dev/null +++ b/src/components/core/service/getStatus.ts @@ -0,0 +1,53 @@ +import { Readable } from 'stream' +import { P2PCommandResponse } from '../../../@types/index.js' +import { ServiceGetStatusCommand } from '../../../@types/commands.js' +import { CommandHandler } from '../handler/handler.js' +import { + ValidateParams, + validateCommandParameters +} from '../../httpRoutes/validateCommands.js' +import type { ServiceJob } from '../../../@types/C2D/ServiceOnDemand.js' +import { toPublicServiceJob } from './utils.js' + +export class ServiceGetStatusHandler extends CommandHandler { + validate(command: ServiceGetStatusCommand): ValidateParams { + // consumerAddress is required: it is the owner scope AND the identity the + // signature/token is verified against. + return validateCommandParameters(command, ['consumerAddress']) + } + + async handle(task: ServiceGetStatusCommand): Promise { + const validationResponse = await this.verifyParamsAndRateLimits(task) + if (this.shouldDenyTaskHandling(validationResponse)) return validationResponse + + // Status exposes live endpoint URLs / payment data, so the caller must prove + // control of consumerAddress; results are then scoped to that owner. + const auth = await this.validateTokenOrSignature( + task.authorization, + task.consumerAddress, + task.nonce, + task.signature, + task.command + ) + if (auth.status.httpStatus !== 200) return auth + + const engines = this.getOceanNode().getC2DEngines() + if (!engines) + return { + stream: null, + status: { httpStatus: 503, error: 'Compute engines not configured' } + } + + // Aggregate across engines; each engine returns only its own cluster's jobs, + // and the query ANDs owner + serviceId so only the authenticated owner's jobs match. + const jobs: ServiceJob[] = [] + for (const eng of engines.getAllEngines()) { + jobs.push(...(await eng.getServiceStatus(task.consumerAddress, task.serviceId))) + } + + return { + stream: Readable.from(JSON.stringify(jobs.map(toPublicServiceJob))), + status: { httpStatus: 200 } + } + } +} diff --git a/src/components/core/service/getStreamableLogs.ts b/src/components/core/service/getStreamableLogs.ts new file mode 100644 index 000000000..fb6ebcfdf --- /dev/null +++ b/src/components/core/service/getStreamableLogs.ts @@ -0,0 +1,91 @@ +import { Stream } from 'stream' +import { P2PCommandResponse } from '../../../@types/index.js' +import { ServiceGetStreamableLogsCommand } from '../../../@types/commands.js' +import { CommandHandler } from '../handler/handler.js' +import { + ValidateParams, + validateCommandParameters, + buildInvalidParametersResponse, + buildInvalidRequestMessage +} from '../../httpRoutes/validateCommands.js' +import { CORE_LOGGER } from '../../../utils/logging/common.js' +import { findServiceJobAndEngine, parseSinceParam } from './utils.js' + +export class ServiceGetStreamableLogsHandler extends CommandHandler { + validate(command: ServiceGetStreamableLogsCommand): ValidateParams { + const validation = validateCommandParameters(command, [ + 'consumerAddress', + 'serviceId' + ]) + if (!validation.valid) return validation + if (command.since) { + try { + parseSinceParam(command.since) + } catch (error: any) { + return buildInvalidRequestMessage(error.message) + } + } + return validation + } + + async handle(task: ServiceGetStreamableLogsCommand): Promise { + const validationResponse = await this.verifyParamsAndRateLimits(task) + if (this.shouldDenyTaskHandling(validationResponse)) return validationResponse + + const auth = await this.validateTokenOrSignature( + task.authorization, + task.consumerAddress, + task.nonce, + task.signature, + task.command + ) + if (auth.status.httpStatus !== 200) return auth + + const engines = this.getOceanNode().getC2DEngines() + if (!engines) + return { + stream: null, + status: { httpStatus: 503, error: 'Compute engines not configured' } + } + + // Find the job and the engine that owns it (by clusterHash — see helper) + const { job, engine } = await findServiceJobAndEngine( + engines, + task.serviceId, + task.consumerAddress + ) + if (!job) + return buildInvalidParametersResponse( + buildInvalidRequestMessage('Service job not found: ' + task.serviceId) + ) + if (!engine) + return { + stream: null, + status: { + httpStatus: 500, + error: `No compute engine owns service ${task.serviceId} (cluster ${job.clusterHash}) — the node's compute configuration may have changed` + } + } + if (job.owner.toLowerCase() !== task.consumerAddress.toLowerCase()) + return { stream: null, status: { httpStatus: 401, error: 'Not the service owner' } } + + try { + const respStream = await engine.getServiceStreamableLogs( + task.serviceId, + task.consumerAddress, + parseSinceParam(task.since) + ) + if (!respStream) { + return { + stream: null, + status: { httpStatus: 404, error: 'Service not found or not running' } + } + } + return { stream: respStream as unknown as Stream, status: { httpStatus: 200 } } + } catch (error) { + const message = (error as Error)?.message ?? String(error) + CORE_LOGGER.error(message) + return { stream: null, status: { httpStatus: 500, error: message } } + } + } +} diff --git a/src/components/core/service/getTemplates.ts b/src/components/core/service/getTemplates.ts new file mode 100644 index 000000000..6cfbd68fa --- /dev/null +++ b/src/components/core/service/getTemplates.ts @@ -0,0 +1,41 @@ +import { Readable } from 'stream' +import { P2PCommandResponse } from '../../../@types/index.js' +import { ServiceGetTemplatesCommand } from '../../../@types/commands.js' +import { CommandHandler } from '../handler/handler.js' +import { + ValidateParams, + validateCommandParameters +} from '../../httpRoutes/validateCommands.js' +import { CORE_LOGGER } from '../../../utils/logging/common.js' + +export class ServiceGetTemplatesHandler extends CommandHandler { + validate(command: ServiceGetTemplatesCommand): ValidateParams { + return validateCommandParameters(command, []) + } + + async handle(task: ServiceGetTemplatesCommand): Promise { + const validationResponse = await this.verifyParamsAndRateLimits(task) + if (this.shouldDenyTaskHandling(validationResponse)) return validationResponse + try { + const engines = this.getOceanNode().getC2DEngines() + if (!engines) + return { + stream: null, + status: { httpStatus: 503, error: 'Compute engines not configured' } + } + + const templates = await engines.fetchServiceTemplates() + CORE_LOGGER.logMessage( + `ServiceGetTemplates: returning ${templates.length} template(s)`, + true + ) + return { + stream: Readable.from(JSON.stringify(templates)), + status: { httpStatus: 200 } + } + } catch (error: any) { + CORE_LOGGER.error(error.message) + return { stream: null, status: { httpStatus: 500, error: error.message } } + } + } +} diff --git a/src/components/core/service/index.ts b/src/components/core/service/index.ts new file mode 100644 index 000000000..b568b2f51 --- /dev/null +++ b/src/components/core/service/index.ts @@ -0,0 +1,10 @@ +export { ServiceGetTemplatesHandler } from './getTemplates.js' +export { ServiceStartHandler } from './startService.js' +export { ServiceStopHandler } from './stopService.js' +export { ServiceExtendHandler } from './extendService.js' +export { ServiceRestartHandler } from './restartService.js' +export { ServiceGetStatusHandler } from './getStatus.js' +export { GetServicesHandler } from './getServices.js' +export { ServiceGetStreamableLogsHandler } from './getStreamableLogs.js' +export * from './utils.js' +export * from './templateLoader.js' diff --git a/src/components/core/service/restartService.ts b/src/components/core/service/restartService.ts new file mode 100644 index 000000000..ed5bcd9c3 --- /dev/null +++ b/src/components/core/service/restartService.ts @@ -0,0 +1,132 @@ +import { Readable } from 'stream' +import { P2PCommandResponse } from '../../../@types/index.js' +import { ServiceRestartCommand } from '../../../@types/commands.js' +import { CommandHandler } from '../handler/handler.js' +import { + ValidateParams, + validateCommandParameters, + buildInvalidParametersResponse, + buildInvalidRequestMessage +} from '../../httpRoutes/validateCommands.js' +import { CORE_LOGGER } from '../../../utils/logging/common.js' +import type { ComputeEnvironment } from '../../../@types/C2D/C2D.js' +import { ServiceStatusNumber } from '../../../@types/C2D/ServiceOnDemand.js' +import { validateAccess } from '../compute/startCompute.js' +import { decryptUserData, findServiceJobAndEngine, toPublicServiceJob } from './utils.js' + +export class ServiceRestartHandler extends CommandHandler { + validate(command: ServiceRestartCommand): ValidateParams { + return validateCommandParameters(command, ['consumerAddress', 'serviceId']) + } + + async handle(task: ServiceRestartCommand): Promise { + const validationResponse = await this.verifyParamsAndRateLimits(task) + if (this.shouldDenyTaskHandling(validationResponse)) return validationResponse + + const auth = await this.validateTokenOrSignature( + task.authorization, + task.consumerAddress, + task.nonce, + task.signature, + task.command + ) + if (auth.status.httpStatus !== 200) return auth + + const node = this.getOceanNode() + const engines = node.getC2DEngines() + if (!engines) + return { + stream: null, + status: { httpStatus: 503, error: 'Compute engines not configured' } + } + + // Find the job and the engine that owns it (by clusterHash — see helper) + const { job, engine } = await findServiceJobAndEngine( + engines, + task.serviceId, + task.consumerAddress + ) + if (!job) + return buildInvalidParametersResponse( + buildInvalidRequestMessage('Service job not found: ' + task.serviceId) + ) + if (!engine) + return { + stream: null, + status: { + httpStatus: 500, + error: `No compute engine owns service ${task.serviceId} (cluster ${job.clusterHash}) — the node's compute configuration may have changed` + } + } + + // Ownership check + if (job.owner.toLowerCase() !== task.consumerAddress.toLowerCase()) + return { stream: null, status: { httpStatus: 401, error: 'Not the service owner' } } + + // Resolve the environment the service runs on. This MUST exist: the services gate and + // access gate both key off it, and restarting resumes the container on it. + const runEnv: ComputeEnvironment | undefined = ( + await engine.getComputeEnvironments() + ).find((e) => e.id === job!.environment) + if (!runEnv) + return buildInvalidParametersResponse( + buildInvalidRequestMessage(`Service environment "${job.environment}" not found`) + ) + + // Services capability gate (mirrors the start path → 403). features.services is mutable, + // so an environment that no longer offers services must not have its services resumed. + if (runEnv.features?.services === false) + return { + stream: null, + status: { httpStatus: 403, error: 'Services are not enabled on this environment' } + } + + // Access-list gate (mirrors paid compute → 403). Re-checked here because access + // lists are mutable and restarting resumes use of the restricted environment. + const accessGranted = await validateAccess(task.consumerAddress, runEnv.access, node) + if (!accessGranted) + return { stream: null, status: { httpStatus: 403, error: 'Access denied' } } + + // State check — cannot restart an expired service + if (job.status === ServiceStatusNumber.Expired) + return buildInvalidParametersResponse( + buildInvalidRequestMessage('Cannot restart an expired service') + ) + + // If newUserData is provided it REPLACES the stored userData (must be the complete set). + // Decrypt it as a validity check before touching the container. + if (task.userData) { + try { + await decryptUserData(task.userData, node.getKeyManager()) + } catch { + return buildInvalidParametersResponse( + buildInvalidRequestMessage( + 'userData could not be decrypted — it must be ECIES-encrypted to the node public key' + ) + ) + } + } + + try { + // Asynchronous, like SERVICE_START: engine.restartService validates, persists the + // job as Restarting and returns immediately — the teardown + image pull/build + + // new container run in the background (an image pull can take minutes and must + // not block the HTTP/P2P response). Clients poll SERVICE_GET_STATUS and watch + // Restarting → … → Running (or an Error status with the failure reason). + const restarted = await engine.restartService( + task.serviceId, + task.consumerAddress, + task.userData, + task.dockerCmd, + task.dockerEntrypoint + ) + return { + stream: Readable.from(JSON.stringify([toPublicServiceJob(restarted)])), + status: { httpStatus: 200 } + } + } catch (error: any) { + CORE_LOGGER.error(`ServiceRestart ${task.serviceId} failed: ${error.message}`) + return { stream: null, status: { httpStatus: 500, error: error.message } } + } + } +} diff --git a/src/components/core/service/startService.ts b/src/components/core/service/startService.ts new file mode 100644 index 000000000..585ce69bd --- /dev/null +++ b/src/components/core/service/startService.ts @@ -0,0 +1,252 @@ +import { Readable } from 'stream' +import { P2PCommandResponse } from '../../../@types/index.js' +import { ServiceStartCommand } from '../../../@types/commands.js' +import { CommandHandler } from '../handler/handler.js' +import { + ValidateParams, + validateCommandParameters, + buildInvalidParametersResponse, + buildInvalidRequestMessage +} from '../../httpRoutes/validateCommands.js' +import { CORE_LOGGER } from '../../../utils/logging/common.js' +import { isAddress } from 'ethers' +import type { C2DEngine } from '../../c2d/compute_engine_base.js' +import type { + ComputeEnvironment, + DBComputeJobPayment as Payment +} from '../../../@types/C2D/C2D.js' +import { generateUniqueID } from '../compute/utils.js' +import { validateAccess } from '../compute/startCompute.js' +import { decryptUserData, toPublicServiceJob } from './utils.js' + +export class ServiceStartHandler extends CommandHandler { + validate(command: ServiceStartCommand): ValidateParams { + const commandValidation = validateCommandParameters(command, [ + 'consumerAddress', + 'environment', + 'image', + 'duration', + 'payment' + ]) + if (commandValidation.valid) { + if (!isAddress(command.consumerAddress)) + return buildInvalidRequestMessage( + 'Parameter : "consumerAddress" is not a valid web3 address' + ) + if (parseInt(String(command.duration)) <= 0) + return buildInvalidRequestMessage('Invalid duration') + const imageModes = [command.tag, command.checksum, command.dockerfile].filter( + Boolean + ).length + if (imageModes > 1) + return buildInvalidRequestMessage( + 'Provide at most one of "tag", "checksum", "dockerfile"' + ) + } + return commandValidation + } + + async handle(task: ServiceStartCommand): Promise { + const validationResponse = await this.verifyParamsAndRateLimits(task) + if (this.shouldDenyTaskHandling(validationResponse)) return validationResponse + + const auth = await this.validateTokenOrSignature( + task.authorization, + task.consumerAddress, + task.nonce, + task.signature, + task.command + ) + if (auth.status.httpStatus !== 200) return auth + + const node = this.getOceanNode() + const engines = node.getC2DEngines() + if (!engines) + return { + stream: null, + status: { httpStatus: 503, error: 'Compute engines not configured' } + } + + try { + // 1. Resolve engine + environment (environment is mandatory) + let engine: C2DEngine + try { + engine = await engines.getC2DByEnvId(task.environment) + } catch { + return buildInvalidParametersResponse( + buildInvalidRequestMessage(`Unknown environment "${task.environment}"`) + ) + } + const env: ComputeEnvironment | undefined = ( + await engine.getComputeEnvironments() + ).find((e) => e.id === task.environment) + if (!env) + return buildInvalidParametersResponse( + buildInvalidRequestMessage(`Unknown environment "${task.environment}"`) + ) + + // 1a. Services capability gate (mirrors compute F4/F5 gates → 403) + if (env.features?.services === false) + return { + stream: null, + status: { + httpStatus: 403, + error: 'Services are not enabled on this environment' + } + } + + // 1b. Access-list gate (mirrors paid compute → 403). The signature only proves + // control of consumerAddress, not allowlist membership, so this must be + // enforced here before any escrow/charge logic. + const accessGranted = await validateAccess(task.consumerAddress, env.access, node) + if (!accessGranted) + return { + stream: null, + status: { + httpStatus: 403, + error: 'Access denied' + } + } + + // 2. Decrypt userData (pre-escrow validity check, so undecryptable input isn't charged). + // The decrypted object becomes the container's env-var map inside the engine. + if (task.userData) { + try { + await decryptUserData(task.userData, node.getKeyManager()) + } catch { + return buildInvalidParametersResponse( + buildInvalidRequestMessage( + 'userData could not be decrypted — it must be ECIES-encrypted to the node public key' + ) + ) + } + } + + // 4. Duration limit + const sod = engine.getC2DConfig().connection?.serviceOnDemand + const maxDuration = sod?.maxDurationSeconds ?? 86400 + if (task.duration > maxDuration) + return buildInvalidParametersResponse( + buildInvalidRequestMessage( + `Duration ${task.duration}s exceeds maximum ${maxDuration}s` + ) + ) + + // 5. Resolve resources (fill cpu/ram/disk defaults the same way compute jobs do) + let resources + try { + resources = await engine.checkAndFillMissingResources( + task.resources ?? [], + env, + false + ) + await engine.checkIfResourcesAreAvailable( + resources, + env, + false, + await engine.getComputeEnvironments() + ) + } catch (e: any) { + return buildInvalidParametersResponse( + buildInvalidRequestMessage(e?.message || String(e)) + ) + } + + // 6. Server-side cost (used to size the escrow lock the background loop will create). + const cost = engine.calculateResourcesCost( + resources, + env, + task.payment.chainId, + task.payment.token, + task.duration + ) + if (cost === null) + return buildInvalidParametersResponse( + buildInvalidRequestMessage( + `No pricing configured for token ${task.payment.token} on chain ${task.payment.chainId}` + ) + ) + + // 6b. Fail fast when the consumer's escrow visibly can't cover the cost, instead + // of returning a serviceId doomed to fail asynchronously at the Locking step. + // Best-effort UX only: balances can change before the background createLock runs, + // so the authoritative check stays in the pipeline — and an RPC hiccup here must + // not block starts (the pipeline check will catch a genuine shortfall anyway). + try { + const [availableWei, costWei] = await Promise.all([ + engine.escrow.getUserAvailableFunds( + task.payment.chainId, + task.consumerAddress, + task.payment.token + ), + engine.escrow.getPaymentAmountInWei( + cost, + task.payment.chainId, + task.payment.token + ) + ]) + if (BigInt(availableWei.toString()) < BigInt(costWei.toString())) { + return buildInvalidParametersResponse( + buildInvalidRequestMessage( + `Insufficient escrow funds for token ${task.payment.token} on chain ` + + `${task.payment.chainId}: available ${availableWei}, required ${costWei} ` + + `wei — deposit and authorize escrow funds before starting the service` + ) + ) + } + } catch (e: any) { + CORE_LOGGER.debug( + `SERVICE_START: escrow funds pre-check skipped (${e.message}) — the background Locking step will verify` + ) + } + + const serviceId = generateUniqueID({ + owner: task.consumerAddress, + environment: task.environment, + image: task.image, + duration: task.duration, + nonce: task.nonce + }) + + // Escrow tx hashes are filled in later by the background pipeline (locking → payment). + const payment: Payment = { + chainId: task.payment.chainId, + token: task.payment.token, + lockTx: '', + claimTx: '', + cancelTx: '', + cost + } + + // 7. Persist the Starting record and return immediately with the serviceId. The + // engine's background loop (processServiceStart) then performs escrow lock → image + // pull/build → claim/cancel → container start. Clients poll SERVICE_GET_STATUS to + // watch the service progress to Running (or a *Failed / Error terminal status). + const job = await engine.createServiceJob( + task.environment, + task.image, + task.tag, + task.checksum, + task.dockerfile, + task.additionalDockerFiles, + task.dockerCmd, + task.dockerEntrypoint, + task.exposedPorts ?? [], + resources, + task.duration, + task.consumerAddress, + payment, + serviceId, + task.userData + ) + + return { + stream: Readable.from(JSON.stringify([toPublicServiceJob(job)])), + status: { httpStatus: 200 } + } + } catch (error: any) { + CORE_LOGGER.error(`ServiceStart failed: ${error.message}`) + return { stream: null, status: { httpStatus: 500, error: error.message } } + } + } +} diff --git a/src/components/core/service/stopService.ts b/src/components/core/service/stopService.ts new file mode 100644 index 000000000..52330ae1a --- /dev/null +++ b/src/components/core/service/stopService.ts @@ -0,0 +1,71 @@ +import { Readable } from 'stream' +import { P2PCommandResponse } from '../../../@types/index.js' +import { ServiceStopCommand } from '../../../@types/commands.js' +import { CommandHandler } from '../handler/handler.js' +import { + ValidateParams, + validateCommandParameters, + buildInvalidParametersResponse, + buildInvalidRequestMessage +} from '../../httpRoutes/validateCommands.js' +import { CORE_LOGGER } from '../../../utils/logging/common.js' +import { findServiceJobAndEngine, toPublicServiceJob } from './utils.js' + +export class ServiceStopHandler extends CommandHandler { + validate(command: ServiceStopCommand): ValidateParams { + return validateCommandParameters(command, ['consumerAddress', 'serviceId']) + } + + async handle(task: ServiceStopCommand): Promise { + const validationResponse = await this.verifyParamsAndRateLimits(task) + if (this.shouldDenyTaskHandling(validationResponse)) return validationResponse + + const auth = await this.validateTokenOrSignature( + task.authorization, + task.consumerAddress, + task.nonce, + task.signature, + task.command + ) + if (auth.status.httpStatus !== 200) return auth + + const engines = this.getOceanNode().getC2DEngines() + if (!engines) + return { + stream: null, + status: { httpStatus: 503, error: 'Compute engines not configured' } + } + + // Find the job and the engine that owns it (by clusterHash — see helper) + const { job, engine } = await findServiceJobAndEngine( + engines, + task.serviceId, + task.consumerAddress + ) + if (!job) + return buildInvalidParametersResponse( + buildInvalidRequestMessage('Service job not found: ' + task.serviceId) + ) + if (!engine) + return { + stream: null, + status: { + httpStatus: 500, + error: `No compute engine owns service ${task.serviceId} (cluster ${job.clusterHash}) — the node's compute configuration may have changed` + } + } + if (job.owner.toLowerCase() !== task.consumerAddress.toLowerCase()) + return { stream: null, status: { httpStatus: 401, error: 'Not the service owner' } } + + try { + const stopped = await engine.stopService(task.serviceId, task.consumerAddress) + return { + stream: Readable.from(JSON.stringify([toPublicServiceJob(stopped)])), + status: { httpStatus: 200 } + } + } catch (error: any) { + CORE_LOGGER.error(`ServiceStop ${task.serviceId} failed: ${error.message}`) + return { stream: null, status: { httpStatus: 500, error: error.message } } + } + } +} diff --git a/src/components/core/service/templateLoader.ts b/src/components/core/service/templateLoader.ts new file mode 100644 index 000000000..fc3c99abc --- /dev/null +++ b/src/components/core/service/templateLoader.ts @@ -0,0 +1,61 @@ +import { readdir, readFile } from 'fs/promises' +import { join } from 'path' +import type { ServiceTemplate } from '../../../@types/C2D/ServiceOnDemand.js' +import { ServiceTemplateSchema } from '../../../utils/config/schemas.js' +import { CORE_LOGGER } from '../../../utils/logging/common.js' + +// Re-reads on every call so operators can add/edit/remove template files without a restart. +// (If profiling ever shows this is hot, add an mtime-keyed cache — semantics stay identical.) +export async function loadServiceTemplates(dir?: string): Promise { + if (!dir) return [] // safety net; in practice the config schema always supplies the default + + let files: string[] + try { + files = (await readdir(dir)).filter((f) => f.toLowerCase().endsWith('.json')).sort() // deterministic order → stable duplicate resolution + } catch (e) { + // A missing folder is the normal "no templates" state — the default path + // (databases/serviceTemplates/) need not exist — so stay quiet on ENOENT. + if (e.code === 'ENOENT') { + CORE_LOGGER.debug( + `serviceTemplatesPath "${dir}" does not exist — no service templates loaded` + ) + } else { + CORE_LOGGER.error(`serviceTemplatesPath "${dir}" is not readable: ${e.message}`) + } + return [] + } + + const byId = new Map() + for (const file of files) { + let raw: unknown + try { + raw = JSON.parse(await readFile(join(dir, file), 'utf8')) + } catch (e) { + CORE_LOGGER.warn( + `Skipping service template file "${file}": invalid JSON (${e.message})` + ) + continue + } + // A file may be a single template object or an array of templates. + for (const candidate of Array.isArray(raw) ? raw : [raw]) { + const parsed = ServiceTemplateSchema.safeParse(candidate) + if (!parsed.success) { + CORE_LOGGER.warn( + `Skipping invalid template in "${file}": ${parsed.error.issues + .map((i) => i.message) + .join('; ')}` + ) + continue + } + const tmpl = parsed.data as ServiceTemplate + if (byId.has(tmpl.id)) { + CORE_LOGGER.warn( + `Duplicate service template id "${tmpl.id}" (in "${file}") — keeping the first occurrence` + ) + continue + } + byId.set(tmpl.id, tmpl) + } + } + return [...byId.values()] +} diff --git a/src/components/core/service/utils.ts b/src/components/core/service/utils.ts new file mode 100644 index 000000000..d47221ae6 --- /dev/null +++ b/src/components/core/service/utils.ts @@ -0,0 +1,165 @@ +import net from 'net' +import type { ServiceJob } from '../../../@types/C2D/ServiceOnDemand.js' +import { EncryptMethod } from '../../../@types/fileObject.js' +import type { KeyManager } from '../../KeyManager/index.js' +import type { C2DDatabase } from '../../database/C2DDatabase.js' +import type { C2DEngine } from '../../c2d/compute_engine_base.js' +import type { C2DEngines } from '../../c2d/compute_engines.js' + +// Looks up a service job and resolves the engine that OWNS it (by clusterHash). Every +// engine shares the same C2DDatabase, so any engine's db returns the job — taking the +// first engine that "finds" it (the old pattern) breaks on nodes with several docker +// engines: the wrong engine's lifecycle lock and InternalLoop would not protect the +// job, resurrecting the teardown-mid-restart race. engine === null with a non-null job +// means no configured engine matches the job's clusterHash (node config changed). +export async function findServiceJobAndEngine( + engines: C2DEngines, + serviceId: string, + owner?: string +): Promise<{ job: ServiceJob | null; engine: C2DEngine | null }> { + const all = engines.getAllEngines() + if (all.length === 0) return { job: null, engine: null } + const [job] = await all[0].db.getServiceJob(serviceId, owner) + if (!job) return { job: null, engine: null } + const engine = all.find((e) => e.getC2DConfig().hash === job.clusterHash) ?? null + return { job, engine } +} + +// Converts the decrypted userData object into a flat container env-var map (stringified values). +export function userDataToEnv(userData: Record): Record { + const env: Record = {} + for (const [k, v] of Object.entries(userData)) { + if (v !== undefined && v !== null) env[k] = String(v) + } + return env +} + +// Decrypts the ECIES userData string (encrypted by the client to the node's public key) +// and JSON-parses it. Called only transiently in memory — at SERVICE_START and SERVICE_RESTART +// to build the container env. Returns {} when no userData was supplied. +export async function decryptUserData( + encryptedUserData: string | undefined, + keyManager: KeyManager +): Promise> { + if (!encryptedUserData) return {} + const plain = await keyManager.decrypt( + Uint8Array.from(Buffer.from(encryptedUserData, 'hex')), + EncryptMethod.ECIES + ) + return JSON.parse(plain.toString()) +} + +// Strips the opaque encrypted userData blob from a ServiceJob before it enters an API +// response (it is node-only-decryptable and useless to callers). null-safe, so handlers +// can pass engine results straight through. EVERY handler returning service jobs +// (SERVICE_START / STOP / EXTEND / RESTART / GET_STATUS) must map results through this. +export function toPublicServiceJob( + job: ServiceJob | null +): Omit | null { + if (!job) return null + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const { userData, ...pub } = job + return pub +} + +// Listing-grade sanitization for SERVICE_LIST, which is NOT owner-scoped: on top of the +// always-stripped userData (the encrypted env blob), it removes everything that reveals +// HOW a service is configured — CMD/ENTRYPOINT overrides and any inline Dockerfile — +// keeping identity, status, resources, endpoints and payment metadata. The owner-scoped +// SERVICE_GET_STATUS keeps those fields (the owner set them). +export function toListedServiceJob( + job: ServiceJob | null +): Omit< + ServiceJob, + 'userData' | 'dockerCmd' | 'dockerEntrypoint' | 'dockerfile' | 'additionalDockerFiles' +> | null { + if (!job) return null + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const { + userData, + dockerCmd, + dockerEntrypoint, + dockerfile, + additionalDockerFiles, + ...pub + } = job + return pub +} + +const SINCE_DURATION_RE = /^(\d+)(s|m|h|d)$/ +const SINCE_DURATION_UNIT_SECONDS: Record = { + s: 1, + m: 60, + h: 3600, + d: 86400 +} + +// Parses the `since` param for SERVICE_GET_STREAMABLE_LOGS into a Unix timestamp (seconds), +// the format `container.logs({ since })` expects. Accepts either an absolute Unix timestamp +// (all-digit string, e.g. "1735689600") or a relative duration counted back from now +// (e.g. "30s", "45m", "2h", "7d") — the latter is a client convenience since the Docker +// Engine API itself only understands absolute timestamps. Returns undefined for "no filter" +// (parameter omitted). Throws on an unrecognized format so the caller can turn it into a 400. +export function parseSinceParam(since?: string): number | undefined { + if (!since) return undefined + if (/^\d+$/.test(since)) return parseInt(since, 10) + const match = since.match(SINCE_DURATION_RE) + if (!match) { + throw new Error( + `Invalid "since" parameter: "${since}". Use a Unix timestamp in seconds, or a relative ` + + 'duration like "30s", "45m", "2h", "7d".' + ) + } + const [, amountStr, unit] = match + const amount = parseInt(amountStr, 10) + return Math.floor(Date.now() / 1000) - amount * SINCE_DURATION_UNIT_SECONDS[unit] +} + +// Port allocation — in-memory set seeded from DB on engine restart +const allocatedPorts = new Set() + +export async function seedAllocatedPorts( + db: C2DDatabase, + clusterHash: string +): Promise { + const jobs = await db.getRunningServiceJobs(clusterHash) + for (const job of jobs) for (const ep of job.endpoints) allocatedPorts.add(ep.hostPort) +} + +export async function allocateHostPort( + rangeStart: number, + rangeEnd: number +): Promise { + const size = rangeEnd - rangeStart + 1 + for (let i = 0; i < Math.min(size, 50); i++) { + const candidate = rangeStart + Math.floor(Math.random() * size) + if (allocatedPorts.has(candidate)) continue + // Reserve before the async check to close the TOCTOU window: the synchronous + // has()->add() pair is atomic, so no concurrent caller can claim the same port + // while we await isPortFree(). Release the reservation if the OS port is busy. + allocatedPorts.add(candidate) + // eslint-disable-next-line no-await-in-loop + if (await isPortFree(candidate)) return candidate + allocatedPorts.delete(candidate) + } + throw new Error(`No free host port in range ${rangeStart}–${rangeEnd}`) +} + +export function releaseHostPort(port: number): void { + allocatedPorts.delete(port) +} + +// Marks an already-assigned port as reserved (idempotent). Used by restart, which +// re-binds the ports recorded on the job: after a stop (or an Error path) released +// them, they must go back into the set before the container binds them again. +export function reserveHostPort(port: number): void { + allocatedPorts.add(port) +} + +function isPortFree(port: number): Promise { + return new Promise((resolve) => { + const s = net.createServer() + s.once('error', () => resolve(false)) + s.listen(port, '0.0.0.0', () => s.close(() => resolve(true))) + }) +} diff --git a/src/components/core/utils/escrow.ts b/src/components/core/utils/escrow.ts index 313134c2f..4984cb5c3 100644 --- a/src/components/core/utils/escrow.ts +++ b/src/components/core/utils/escrow.ts @@ -41,6 +41,23 @@ export class Escrow { return maxJobDuration + this.claimDurationTimeout } + /** + * Waits for a submitted transaction to be mined. Used when two transactions are sent + * back-to-back from the node signer (e.g. the immediate createLock → claimLock sequence + * in Service-on-Demand) so the second tx picks up the advanced account nonce and acts on + * confirmed on-chain state. + */ + async waitForTransaction( + chain: number, + txHash: string, + confirmations: number = 1, + timeoutMs: number = 60000 + ): Promise { + const blockchain = this.getBlockchain(chain) + const provider = await blockchain.getProvider() + await provider.waitForTransaction(txHash, confirmations, timeoutMs) + } + /** * Get a Blockchain instance for the given chainId from BlockchainRegistry. * diff --git a/src/components/core/utils/nonceHandler.ts b/src/components/core/utils/nonceHandler.ts index 44ecdea5c..9e1cd41f8 100644 --- a/src/components/core/utils/nonceHandler.ts +++ b/src/components/core/utils/nonceHandler.ts @@ -195,7 +195,41 @@ async function validateNonceAndSignature( error: 'nonce: ' + nonce + ' is not a valid nonce' } } - const message = String(String(consumer) + String(nonce) + String(command)) + const issuerPeerId = + command === PROTOCOL_COMMANDS.CREATE_AUTH_TOKEN + ? OceanNode.getInstance().getKeyManager().getPeerIdString() + : '' + if ( + await verifyConsumerSignature( + consumer, + nonce, + signature, + issuerPeerId, + command, + config, + chainId + ) + ) { + return { valid: true } + } + return { + valid: false, + error: 'consumer address and nonce signature mismatch' + } +} + +export async function verifyConsumerSignature( + consumer: string, + nonce: string | number, + signature: string, + issuerPeerId: string, + command: string = null, + config?: OceanNodeConfig, + chainId?: string | null +): Promise { + const message = String( + String(consumer) + String(nonce) + String(command) + String(issuerPeerId) + ) const consumerMessage = ethers.solidityPackedKeccak256( ['bytes'], [ethers.hexlify(ethers.toUtf8Bytes(message))] @@ -212,7 +246,7 @@ async function validateNonceAndSignature( ethers.getAddress(addressFromBytesSignature)?.toLowerCase() === ethers.getAddress(consumer)?.toLowerCase() ) { - return { valid: true } + return true } } catch (error) { // Continue to smart account check @@ -228,23 +262,20 @@ async function validateNonceAndSignature( // Try custom hash format (for backward compatibility) if (await isERC1271Valid(consumer, consumerMessage, signature, provider)) { - return { valid: true } + return true } // Try EIP-191 prefixed hash (standard for smart wallets) const eip191Hash = ethers.hashMessage(message) if (await isERC1271Valid(consumer, eip191Hash, signature, provider)) { - return { valid: true } + return true } } } catch (error) { - // Smart account validation failed + CORE_LOGGER.error(`ERC-1271 signature validation error: ${error?.message}`) } - return { - valid: false, - error: 'consumer address and nonce signature mismatch' - } + return false } // Smart account validation diff --git a/src/components/database/AuthTokenDatabase.ts b/src/components/database/AuthTokenDatabase.ts index 662489a41..b1f6c860a 100644 --- a/src/components/database/AuthTokenDatabase.ts +++ b/src/components/database/AuthTokenDatabase.ts @@ -1,8 +1,6 @@ import { DATABASE_LOGGER } from '../../utils/logging/common.js' import { AbstractDatabase } from './BaseDatabase.js' import { OceanNodeDBConfig } from '../../@types/OceanNode.js' -import path from 'path' -import * as fs from 'fs' import { SQLiteAuthToken } from './sqliteAuthToken.js' export interface AuthToken { @@ -24,10 +22,7 @@ export class AuthTokenDatabase extends AbstractDatabase { static async create(config: OceanNodeDBConfig): Promise { DATABASE_LOGGER.info('Creating AuthTokenDatabase with SQLite') - const dbDir = path.dirname('databases/authTokenDatabase.sqlite') - if (!fs.existsSync(dbDir)) { - fs.mkdirSync(dbDir, { recursive: true }) - } + // SqliteClient creates the parent directory on construction. const provider = new SQLiteAuthToken('databases/authTokenDatabase.sqlite') await provider.createTable() return new AuthTokenDatabase(config, provider) diff --git a/src/components/database/C2DDatabase.ts b/src/components/database/C2DDatabase.ts index b1af4db02..b19d30dea 100755 --- a/src/components/database/C2DDatabase.ts +++ b/src/components/database/C2DDatabase.ts @@ -5,6 +5,7 @@ import { DBComputeJob, C2DStatusNumber } from '../../@types/C2D/C2D.js' +import { ServiceJob } from '../../@types/C2D/ServiceOnDemand.js' import { SQLiteCompute } from './sqliteCompute.js' import { DATABASE_LOGGER } from '../../utils/logging/common.js' import { OceanNodeDBConfig } from '../../@types/OceanNode.js' @@ -30,6 +31,8 @@ export class C2DDatabase extends AbstractDatabase { this.provider = new SQLiteCompute('databases/c2dDatabase.sqlite') await this.provider.createTable() await this.provider.createImageTable() + await this.provider.createServiceTable() + await this.provider.createServiceLocksTable() return this })() as unknown as C2DDatabase @@ -71,6 +74,54 @@ export class C2DDatabase extends AbstractDatabase { return await this.provider.getRunningJobs(engine, environment) } + // ── Service-on-Demand jobs ────────────────────────────────────────── + + async newServiceJob(job: ServiceJob): Promise { + return await this.provider.newServiceJob(job) + } + + async getServiceJob(serviceId?: string, owner?: string): Promise { + return await this.provider.getServiceJob(serviceId, owner) + } + + async updateServiceJob(job: ServiceJob): Promise { + return await this.provider.updateServiceJob(job) + } + + async getRunningServiceJobs(clusterHash?: string): Promise { + return await this.provider.getRunningServiceJobs(clusterHash) + } + + async getExpiredServiceJobs(clusterHash?: string): Promise { + return await this.provider.getExpiredServiceJobs(clusterHash) + } + + async getPendingServiceStarts(clusterHash?: string): Promise { + return await this.provider.getPendingServiceStarts(clusterHash) + } + + // ── Service lifecycle locks (cross-process; see sqliteCompute.ts) ─────── + + async acquireServiceLock( + serviceId: string, + holder: string, + staleMs: number + ): Promise { + return await this.provider.acquireServiceLock(serviceId, holder, staleMs) + } + + async releaseServiceLock(serviceId: string, holder: string): Promise { + return await this.provider.releaseServiceLock(serviceId, holder) + } + + async refreshServiceLocks(holder: string): Promise { + return await this.provider.refreshServiceLocks(holder) + } + + async isServiceLocked(serviceId: string, staleMs: number): Promise { + return await this.provider.isServiceLocked(serviceId, staleMs) + } + async deleteJob(jobId: string): Promise { return await this.provider.deleteJob(jobId) } diff --git a/src/components/database/ElasticSchemas.ts b/src/components/database/ElasticSchemas.ts index b2b09965a..ac5c8ab23 100644 --- a/src/components/database/ElasticSchemas.ts +++ b/src/components/database/ElasticSchemas.ts @@ -165,7 +165,10 @@ export const elasticSchemas: ElasticsearchSchemas = { proof: { type: 'text' }, maxLockedAmount: { type: 'text' }, maxLockSeconds: { type: 'text' }, - maxLockCounts: { type: 'text' } + maxLockCounts: { type: 'text' }, + oldAmount: { type: 'text' }, + newAmount: { type: 'text' }, + newExpiry: { type: 'text' } } } } diff --git a/src/components/database/SQLLiteConfigDatabase.ts b/src/components/database/SQLLiteConfigDatabase.ts index 72f9f0942..540f991e2 100644 --- a/src/components/database/SQLLiteConfigDatabase.ts +++ b/src/components/database/SQLLiteConfigDatabase.ts @@ -1,5 +1,3 @@ -import fs from 'fs' -import path from 'path' import { DATABASE_LOGGER } from '../../utils/logging/common.js' import { GENERIC_EMOJIS, LOG_LEVELS_STR } from '../../utils/logging/Logger.js' import { SQLiteProvider } from './sqlite.js' @@ -11,11 +9,7 @@ export class SQLLiteConfigDatabase { return (async (): Promise => { DATABASE_LOGGER.info('Config Database initiated with SQLite provider') - // Ensure the directory exists before instantiating SQLiteProvider - const dbDir = path.dirname('databases/config.sqlite') - if (!fs.existsSync(dbDir)) { - fs.mkdirSync(dbDir, { recursive: true }) - } + // SqliteClient creates the parent directory on construction. this.provider = new SQLiteProvider('databases/config.sqlite') await this.provider.createTableForConfig() diff --git a/src/components/database/SQLLiteNonceDatabase.ts b/src/components/database/SQLLiteNonceDatabase.ts index 06334c6e7..039518050 100644 --- a/src/components/database/SQLLiteNonceDatabase.ts +++ b/src/components/database/SQLLiteNonceDatabase.ts @@ -1,5 +1,3 @@ -import fs from 'fs' -import path from 'path' import { OceanNodeDBConfig } from '../../@types/OceanNode.js' import { DATABASE_LOGGER } from '../../utils/logging/common.js' import { GENERIC_EMOJIS, LOG_LEVELS_STR } from '../../utils/logging/Logger.js' @@ -15,11 +13,7 @@ export class SQLLiteNonceDatabase extends AbstractNonceDatabase { return (async (): Promise => { DATABASE_LOGGER.info('Nonce Database initiated with SQLite provider') - // Ensure the directory exists before instantiating SQLiteProvider - const dbDir = path.dirname('databases/nonceDatabase.sqlite') - if (!fs.existsSync(dbDir)) { - fs.mkdirSync(dbDir, { recursive: true }) - } + // SqliteClient creates the parent directory on construction. this.provider = new SQLiteProvider('databases/nonceDatabase.sqlite') await this.provider.createTableForNonce() diff --git a/src/components/database/TypesenseSchemas.ts b/src/components/database/TypesenseSchemas.ts index 292aef8f8..6f91ddd80 100644 --- a/src/components/database/TypesenseSchemas.ts +++ b/src/components/database/TypesenseSchemas.ts @@ -166,7 +166,11 @@ export const typesenseSchemas: TypesenseSchemas = { { name: 'proof', type: 'string', optional: true, index: false }, { name: 'maxLockedAmount', type: 'string', optional: true }, { name: 'maxLockSeconds', type: 'string', optional: true }, - { name: 'maxLockCounts', type: 'string', optional: true } + { name: 'maxLockCounts', type: 'string', optional: true }, + // ReLock event fields (uint256 kept as raw strings) + { name: 'oldAmount', type: 'string', optional: true }, + { name: 'newAmount', type: 'string', optional: true }, + { name: 'newExpiry', type: 'string', optional: true } ] } } diff --git a/src/components/database/sqlite.ts b/src/components/database/sqlite.ts index 61231688e..4a081e55d 100644 --- a/src/components/database/sqlite.ts +++ b/src/components/database/sqlite.ts @@ -1,5 +1,5 @@ import { TypesenseSchema, typesenseSchemas } from './TypesenseSchemas.js' -import sqlite3 from 'sqlite3' +import { SqliteClient } from './sqliteClient.js' interface DatabaseProvider { createNonce(address: string, nonce: number): Promise<{ id: string; nonce: number }> @@ -9,46 +9,34 @@ interface DatabaseProvider { } export class SQLiteProvider implements DatabaseProvider { - private db: sqlite3.Database + private db: SqliteClient private schemaNonce: TypesenseSchema private configSchema: string constructor(dbFilePath: string) { - this.db = new sqlite3.Database(dbFilePath) + this.db = new SqliteClient(dbFilePath) this.schemaNonce = typesenseSchemas.nonceSchemas this.configSchema = 'config' } // eslint-disable-next-line require-await async createTableForNonce() { - const createTableSQL = ` + this.db.exec(` CREATE TABLE IF NOT EXISTS ${this.schemaNonce.name} ( id TEXT PRIMARY KEY, nonce INTEGER ); - ` - return new Promise((resolve, reject) => { - this.db.run(createTableSQL, (err) => { - if (err) reject(err) - else resolve() - }) - }) + `) } // eslint-disable-next-line require-await async createTableForConfig() { - const createTableSQL = ` + this.db.exec(` CREATE TABLE IF NOT EXISTS ${this.configSchema} ( key TEXT NOT NULL PRIMARY KEY, value TEXT ); - ` - return new Promise((resolve, reject) => { - this.db.run(createTableSQL, (err) => { - if (err) reject(err) - else resolve() - }) - }) + `) } // eslint-disable-next-line require-await @@ -58,12 +46,8 @@ export class SQLiteProvider implements DatabaseProvider { VALUES (?, ?) ON CONFLICT(id) DO UPDATE SET nonce=excluded.nonce; ` - return new Promise<{ id: string; nonce: number }>((resolve, reject) => { - this.db.run(insertSQL, [address, nonce], (err) => { - if (err) reject(err) - else resolve({ id: address, nonce }) - }) - }) + this.db.run(insertSQL, [address, nonce]) + return { id: address, nonce } } // eslint-disable-next-line require-await @@ -73,12 +57,8 @@ export class SQLiteProvider implements DatabaseProvider { VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value; ` - return new Promise<{ key: string; value: string }>((resolve, reject) => { - this.db.run(insertSQL, [key, value], (err) => { - if (err) reject(err) - else resolve({ key, value }) - }) - }) + this.db.run(insertSQL, [key, value]) + return { key, value } } // eslint-disable-next-line require-await @@ -86,13 +66,8 @@ export class SQLiteProvider implements DatabaseProvider { const selectSQL = ` SELECT * FROM ${this.schemaNonce.name} WHERE id = ? ` - return new Promise<{ id: string; nonce: number | null }>((resolve, reject) => { - this.db.get(selectSQL, [address], (err, row: { nonce: number } | undefined) => { - if (err) reject(err) - else - resolve(row ? { id: address, nonce: row.nonce } : { id: address, nonce: null }) - }) - }) + const row = this.db.get<{ nonce: number }>(selectSQL, [address]) + return row ? { id: address, nonce: row.nonce } : { id: address, nonce: null } } // eslint-disable-next-line require-await @@ -100,12 +75,9 @@ export class SQLiteProvider implements DatabaseProvider { const selectSQL = ` SELECT value FROM ${this.configSchema} WHERE key = ?; ` - return new Promise<{ value: string | null }>((resolve, reject) => { - this.db.get(selectSQL, [key], (err, row: { value: string } | undefined) => { - if (err) reject(err) - else resolve(row ? { value: row.value } : { value: null }) // Returns null if no version exists - }) - }) + const row = this.db.get<{ value: string }>(selectSQL, [key]) + // Returns null if no version exists + return row ? { value: row.value } : { value: null } } // eslint-disable-next-line require-await @@ -123,16 +95,10 @@ export class SQLiteProvider implements DatabaseProvider { DELETE FROM ${this.schemaNonce.name} WHERE id = ? ` - return new Promise<{ id: string; nonce: number | null }>((resolve, reject) => { - this.db.get(selectSQL, [address], (err, row: { nonce: number } | undefined) => { - if (err) return reject(err) - if (!row) return resolve({ id: address, nonce: null }) + const row = this.db.get<{ nonce: number }>(selectSQL, [address]) + if (!row) return { id: address, nonce: null } - this.db.run(deleteSQL, [address], (err) => { - if (err) reject(err) - else resolve({ id: address, nonce: row.nonce }) - }) - }) - }) + this.db.run(deleteSQL, [address]) + return { id: address, nonce: row.nonce } } } diff --git a/src/components/database/sqliteAuthToken.ts b/src/components/database/sqliteAuthToken.ts index 5f607dafd..bdc8ba596 100644 --- a/src/components/database/sqliteAuthToken.ts +++ b/src/components/database/sqliteAuthToken.ts @@ -1,5 +1,5 @@ import { AuthToken } from './AuthTokenDatabase.js' -import sqlite3 from 'sqlite3' +import { SqliteClient } from './sqliteClient.js' import { DATABASE_LOGGER } from '../../utils/logging/common.js' interface AuthTokenDatabaseProvider { @@ -15,14 +15,15 @@ interface AuthTokenDatabaseProvider { } export class SQLiteAuthToken implements AuthTokenDatabaseProvider { - private db: sqlite3.Database + private db: SqliteClient constructor(dbFilePath: string) { - this.db = new sqlite3.Database(dbFilePath) + this.db = new SqliteClient(dbFilePath) } + // eslint-disable-next-line require-await async createTable(): Promise { - await this.db.exec(` + this.db.exec(` CREATE TABLE IF NOT EXISTS authTokens ( token TEXT PRIMARY KEY, address TEXT NOT NULL, @@ -33,16 +34,20 @@ export class SQLiteAuthToken implements AuthTokenDatabaseProvider { ) `) - // Migration: Add chainId column if it doesn't exist - return new Promise((resolve) => { - this.db.run(`ALTER TABLE authTokens ADD COLUMN chainId TEXT`, (_err) => { - // Ignore error if column already exists - resolve() - }) - }) + // Migration for DBs created before the chainId column existed: add it if missing. + // A fresh table already has the column, so ALTER throws "duplicate column name" — + // that's the only expected failure, so swallow it. With the synchronous exec above this + // now runs strictly after the CREATE TABLE, unlike the old callback-based `await exec` + // which never actually waited and let the CREATE race the ALTER. + try { + this.db.exec(`ALTER TABLE authTokens ADD COLUMN chainId TEXT`) + } catch { + // column already exists + } } - createToken( + // eslint-disable-next-line require-await + async createToken( token: string, address: string, createdAt: number, @@ -52,73 +57,60 @@ export class SQLiteAuthToken implements AuthTokenDatabaseProvider { const insertSQL = ` INSERT INTO authTokens (token, address, createdAt, validUntil, chainId) VALUES (?, ?, ?, ?, ?) ` - return new Promise((resolve, reject) => { - this.db.run(insertSQL, [token, address, createdAt, validUntil, chainId], (err) => { - if (err) { - DATABASE_LOGGER.error(`Error creating auth token: ${err}`) - reject(err) - } else { - resolve() - } - }) - }) + try { + this.db.run(insertSQL, [token, address, createdAt, validUntil, chainId]) + } catch (err) { + DATABASE_LOGGER.error(`Error creating auth token: ${err}`) + throw err + } } - validateTokenEntry(token: string): Promise { + async validateTokenEntry(token: string): Promise { const selectSQL = ` SELECT * FROM authTokens WHERE token = ? ` - return new Promise((resolve, reject) => { - this.db.get(selectSQL, [token], async (err, row: AuthToken) => { - if (err) { - DATABASE_LOGGER.error(`Error validating auth token: ${err}`) - reject(err) - return - } + let row: AuthToken | undefined + try { + row = this.db.get(selectSQL, [token]) + } catch (err) { + DATABASE_LOGGER.error(`Error validating auth token: ${err}`) + throw err + } - if (!row) { - resolve(null) - return - } + if (!row) { + return null + } - if (!row.isValid) { - resolve(null) - return - } + if (!row.isValid) { + return null + } - if (row.validUntil === null) { - resolve(row) - return - } + if (row.validUntil === null) { + return row + } - const validUntilDate = new Date(row.validUntil).getTime() - const now = Date.now() + const validUntilDate = new Date(row.validUntil).getTime() + const now = Date.now() - if (validUntilDate < now) { - resolve(null) - DATABASE_LOGGER.info(`Auth token ${token} is invalid`) - await this.invalidateTokenEntry(token) - return - } + if (validUntilDate < now) { + DATABASE_LOGGER.info(`Auth token ${token} is invalid`) + await this.invalidateTokenEntry(token) + return null + } - resolve(row) - }) - }) + return row } - invalidateTokenEntry(token: string): Promise { + // eslint-disable-next-line require-await + async invalidateTokenEntry(token: string): Promise { const deleteSQL = ` UPDATE authTokens SET isValid = FALSE WHERE token = ? ` - return new Promise((resolve, reject) => { - this.db.run(deleteSQL, [token], (err) => { - if (err) { - DATABASE_LOGGER.error(`Error invalidating auth token: ${err}`) - reject(err) - } else { - resolve() - } - }) - }) + try { + this.db.run(deleteSQL, [token]) + } catch (err) { + DATABASE_LOGGER.error(`Error invalidating auth token: ${err}`) + throw err + } } } diff --git a/src/components/database/sqliteClient.ts b/src/components/database/sqliteClient.ts new file mode 100644 index 000000000..bd9a31f84 --- /dev/null +++ b/src/components/database/sqliteClient.ts @@ -0,0 +1,59 @@ +import { DatabaseSync } from 'node:sqlite' +import fs from 'fs' +import path from 'path' + +type Bindable = string | number | bigint | Uint8Array | null + +// node:sqlite refuses `undefined` and boolean bindings (throws ERR_INVALID_ARG_TYPE); +// the old `sqlite3` package silently coerced them. Keep that lenient behaviour centrally +// so every call site binds the same way it did before the engine swap. +function sanitize(v: unknown): Bindable { + if (v === undefined) return null + if (typeof v === 'boolean') return v ? 1 : 0 + return v as Bindable +} + +/** + * Thin synchronous wrapper around Node's built-in `node:sqlite` (`DatabaseSync`). + * Owns a single database handle per file and the sqlite3-compatibility concerns + * (bind sanitization, eager parent-directory creation). This replaces the former + * `sqlite3` native addon; all embedded DBs (nonce, config, C2D jobs, auth tokens, + * persistent-storage registry) go through this client. + * + * The queries in this project are all single-row or small local-table operations, so + * running them synchronously on the main thread is acceptable. Do not point this client + * at large or unbounded datasets without reconsidering. + */ +export class SqliteClient { + private db: DatabaseSync + + constructor(dbFilePath: string) { + // DatabaseSync opens eagerly and throws synchronously if the parent directory is + // missing (unlike the old sqlite3.Database, which deferred opening). mkdir here so + // every DB path is covered uniformly and callers no longer need to mkdir themselves. + fs.mkdirSync(path.dirname(dbFilePath), { recursive: true }) + this.db = new DatabaseSync(dbFilePath) + // Long-running server process: wait briefly rather than fail if the DB is momentarily + // locked (e.g. an external `sqlite3` CLI reading a live file). + this.db.exec('PRAGMA busy_timeout = 5000;') + } + + exec(sql: string): void { + this.db.exec(sql) + } + + run(sql: string, params: unknown[] = []): { changes: number } { + const result = this.db.prepare(sql).run(...params.map(sanitize)) + // readBigInts is not enabled, so `changes` is a JS number already; coerce to keep the + // public type a plain number (all counts here are tiny, well below Number.MAX_SAFE_INTEGER). + return { changes: Number(result.changes) } + } + + get(sql: string, params: unknown[] = []): T | undefined { + return this.db.prepare(sql).get(...params.map(sanitize)) as T | undefined + } + + all(sql: string, params: unknown[] = []): T[] { + return this.db.prepare(sql).all(...params.map(sanitize)) as T[] + } +} diff --git a/src/components/database/sqliteCompute.ts b/src/components/database/sqliteCompute.ts index 5a1006636..7511350b5 100644 --- a/src/components/database/sqliteCompute.ts +++ b/src/components/database/sqliteCompute.ts @@ -4,7 +4,12 @@ import { C2DStatusText, type DBComputeJob } from '../../@types/C2D/C2D.js' -import sqlite3, { RunResult } from 'sqlite3' +import { + ServiceStatusNumber, + SERVICE_START_PENDING_STATUSES, + type ServiceJob +} from '../../@types/C2D/ServiceOnDemand.js' +import { SqliteClient } from './sqliteClient.js' import { DATABASE_LOGGER } from '../../utils/logging/common.js' import { create256Hash } from '../../utils/crypt.js' @@ -60,7 +65,9 @@ export function generateBlobFromJSON(job: DBComputeJob): Buffer { } export function generateJSONFromBlob(blob: any): Promise { - return JSON.parse(blob.toString()) + // node:sqlite returns BLOB columns as Uint8Array (the old sqlite3 addon returned Buffer). + // Buffer.from() handles both, so decode through it before parsing. + return JSON.parse(Buffer.from(blob).toString()) } // we cannot store array of strings, so we use string separators instead @@ -83,28 +90,26 @@ export function convertStringToArray(str: string) { } export class SQLiteCompute implements ComputeDatabaseProvider { - private db: sqlite3.Database + private db: SqliteClient private schema: TypesenseSchema constructor(dbFilePath: string) { - this.db = new sqlite3.Database(dbFilePath) + this.db = new SqliteClient(dbFilePath) this.schema = typesenseSchemas.c2dSchemas } - deleteJob(jobId: string): Promise { + // eslint-disable-next-line require-await + async deleteJob(jobId: string): Promise { const deleteSQL = ` DELETE FROM ${this.schema.name} WHERE jobId = ? ` - return new Promise((resolve, reject) => { - this.db.run(deleteSQL, [jobId], function (this: RunResult, err) { - if (err) reject(err) - else resolve(this.changes === 1) - }) - }) + const { changes } = this.db.run(deleteSQL, [jobId]) + return changes === 1 } - createTable() { - /* although we have field called expiteTimestamp, we are actually storing maxJobDuration in it */ + // eslint-disable-next-line require-await + async createTable() { + /* although we have field called expireTimestamp, we are actually storing maxJobDuration in it */ const createTableSQL = ` CREATE TABLE IF NOT EXISTS ${this.schema.name} ( owner TEXT, @@ -123,140 +128,408 @@ export class SQLiteCompute implements ComputeDatabaseProvider { body BLOB ); ` - return new Promise((resolve, reject) => { - this.db.run(createTableSQL, (err) => { - if (err) reject(err) - else resolve() - }) - }) + try { + this.db.exec(createTableSQL) + } catch (err) { + DATABASE_LOGGER.error(`Could not create ${this.schema.name} table: ${err.message}`) + throw err + } } - createImageTable(): Promise { + // eslint-disable-next-line require-await + async createImageTable(): Promise { const createTableSQL = ` CREATE TABLE IF NOT EXISTS docker_images ( image TEXT PRIMARY KEY, lastUsedTimestamp INTEGER NOT NULL ); ` - return new Promise((resolve, reject) => { - this.db.run(createTableSQL, (err) => { - if (err) { - DATABASE_LOGGER.error('Could not create docker_images table: ' + err.message) - reject(err) - } else { - resolve() - } - }) - }) + try { + this.db.exec(createTableSQL) + } catch (err) { + DATABASE_LOGGER.error('Could not create docker_images table: ' + err.message) + throw err + } + } + + // ── Service-on-Demand jobs ────────────────────────────────────────── + + // Cross-process lifecycle locks for service jobs. A row = an exclusive start/stop/ + // restart operation in flight on serviceId, held by `holder` (a per-process id). Rows + // are heartbeated (acquiredAt refreshed) while the operation runs; a row whose + // acquiredAt is older than the staleness window is a crashed holder and may be stolen. + // This extends the engine's in-memory serviceOpsInFlight guarantee to setups where + // several node processes share the same DB file + Docker daemon (e.g. a stale + // container still running during a redeploy) — in-memory sets cannot see each other. + // eslint-disable-next-line require-await + async createServiceLocksTable(): Promise { + const createTableSQL = ` + CREATE TABLE IF NOT EXISTS service_locks ( + serviceId TEXT PRIMARY KEY, + holder TEXT NOT NULL, + acquiredAt INTEGER NOT NULL + ); + ` + try { + this.db.exec(createTableSQL) + } catch (err) { + DATABASE_LOGGER.error('Could not create service_locks table: ' + err.message) + throw err + } + } + + // Atomically takes the lock for serviceId: inserts a fresh row, or steals one whose + // acquiredAt is older than staleMs (crashed holder). The single upsert statement is + // the atomicity guarantee — two processes racing it can never both see success. + // eslint-disable-next-line require-await + async acquireServiceLock( + serviceId: string, + holder: string, + staleMs: number + ): Promise { + const now = Date.now() + const upsertSQL = ` + INSERT INTO service_locks (serviceId, holder, acquiredAt) VALUES (?, ?, ?) + ON CONFLICT(serviceId) DO UPDATE + SET holder = excluded.holder, acquiredAt = excluded.acquiredAt + WHERE service_locks.acquiredAt <= ?; + ` + try { + const { changes } = this.db.run(upsertSQL, [serviceId, holder, now, now - staleMs]) + return changes === 1 + } catch (err) { + DATABASE_LOGGER.error(`Could not acquire service lock: ${err.message}`) + throw err + } + } + + // Releases only a lock we still hold — a stale lock stolen by another process must + // not be deleted out from under its new holder. + // eslint-disable-next-line require-await + async releaseServiceLock(serviceId: string, holder: string): Promise { + const deleteSQL = `DELETE FROM service_locks WHERE serviceId = ? AND holder = ?;` + try { + this.db.run(deleteSQL, [serviceId, holder]) + } catch (err) { + DATABASE_LOGGER.error(`Could not release service lock: ${err.message}`) + throw err + } + } + + // Heartbeat: re-stamps every lock this holder owns so long operations (multi-minute + // image pulls/builds) are not stolen as stale. + // eslint-disable-next-line require-await + async refreshServiceLocks(holder: string): Promise { + const updateSQL = `UPDATE service_locks SET acquiredAt = ? WHERE holder = ?;` + try { + this.db.run(updateSQL, [Date.now(), holder]) + } catch (err) { + DATABASE_LOGGER.error(`Could not refresh service locks: ${err.message}`) + throw err + } + } + + // True while ANY process holds a fresh lock on serviceId — used by read-only + // observers (e.g. the container health check) to avoid judging a service that + // another process is mid-way through restarting. + // eslint-disable-next-line require-await + async isServiceLocked(serviceId: string, staleMs: number): Promise { + const selectSQL = `SELECT acquiredAt FROM service_locks WHERE serviceId = ?;` + try { + const row = this.db.get<{ acquiredAt: number }>(selectSQL, [serviceId]) + return !!row && row.acquiredAt > Date.now() - staleMs + } catch (err) { + DATABASE_LOGGER.error(`Could not read service lock: ${err.message}`) + throw err + } } - updateImage(image: string): Promise { + // eslint-disable-next-line require-await + async createServiceTable(): Promise { + const createTableSQL = ` + CREATE TABLE IF NOT EXISTS service_jobs ( + serviceId TEXT PRIMARY KEY, + owner TEXT, + clusterHash TEXT, + status INTEGER, + expiresAt INTEGER, + dateCreated TEXT, + body BLOB + ); + ` + try { + this.db.exec(createTableSQL) + } catch (err) { + DATABASE_LOGGER.error('Could not create service_jobs table: ' + err.message) + throw err + } + } + + // eslint-disable-next-line require-await + async newServiceJob(job: ServiceJob): Promise { + const insertSQL = ` + INSERT INTO service_jobs + (serviceId, owner, clusterHash, status, expiresAt, dateCreated, body) + VALUES (?, ?, ?, ?, ?, ?, ?); + ` + try { + this.db.run(insertSQL, [ + job.serviceId, + job.owner, + job.clusterHash, + job.status, + job.expiresAt, + job.dateCreated, + Buffer.from(JSON.stringify({ ...job, updatedAt: Date.now() })) + ]) + } catch (err) { + DATABASE_LOGGER.error('Could not insert service job on DB: ' + err.message) + throw err + } + } + + // eslint-disable-next-line require-await + async updateServiceJob(job: ServiceJob): Promise { + const updateSQL = ` + UPDATE service_jobs + SET owner = ?, clusterHash = ?, status = ?, expiresAt = ?, body = ? + WHERE serviceId = ?; + ` + try { + const { changes } = this.db.run(updateSQL, [ + job.owner, + job.clusterHash, + job.status, + job.expiresAt, + Buffer.from(JSON.stringify({ ...job, updatedAt: Date.now() })), + job.serviceId + ]) + return changes + } catch (err) { + DATABASE_LOGGER.error(`Error while updating service job: ${err.message}`) + throw err + } + } + + private mapServiceRows(rows: any[] | undefined): ServiceJob[] { + if (!rows || rows.length === 0) return [] + // BLOB comes back as Uint8Array from node:sqlite; decode through Buffer before parsing. + return rows.map((row) => JSON.parse(Buffer.from(row.body).toString()) as ServiceJob) + } + + // eslint-disable-next-line require-await + async getServiceJob(serviceId?: string, owner?: string): Promise { + const params: any[] = [] + let selectSQL = `SELECT * FROM service_jobs WHERE 1=1` + if (serviceId) { + selectSQL += ` AND serviceId = ?` + params.push(serviceId) + } + if (owner) { + selectSQL += ` AND owner = ?` + params.push(owner) + } + try { + const rows = this.db.all(selectSQL, params) + return this.mapServiceRows(rows) + } catch (err) { + DATABASE_LOGGER.error(err.message) + throw err + } + } + + // eslint-disable-next-line require-await + async getRunningServiceJobs(clusterHash?: string): Promise { + // Every status before Expired is "active": the consumer paid for the resources for a + // TIME WINDOW, so the reservation holds from the moment the record is created + // (Starting) through the whole start pipeline (Locking, image, Claiming), while + // Running/Restarting/Stopping, through Error (container died on its own) AND through + // an explicit Stopped — a stopped service can be restarted anytime on the same + // resources until expiresAt. Only the expiry sweep (→ Expired) releases the + // reservation; nothing else may free it inside the paid window. + const activeStatuses = [ + ServiceStatusNumber.Starting, + ServiceStatusNumber.Locking, + ServiceStatusNumber.PullImage, + ServiceStatusNumber.BuildImage, + ServiceStatusNumber.Claiming, + ServiceStatusNumber.Running, + ServiceStatusNumber.Restarting, + ServiceStatusNumber.Stopping, + ServiceStatusNumber.Stopped, + ServiceStatusNumber.Error + ] + const placeholders = activeStatuses.map(() => '?').join(',') + const params: Array = [...activeStatuses] + let selectSQL = `SELECT * FROM service_jobs WHERE status IN (${placeholders})` + if (clusterHash) { + selectSQL += ` AND clusterHash = ?` + params.push(clusterHash) + } + try { + const rows = this.db.all(selectSQL, params) + // The reservation is tied to PAYMENT: an Error/Stopped job whose payment was + // never claimed (escrow lock failed — e.g. insufficient funds — or refunded) + // must not hold resources, or anyone could squat a node's GPU for free by + // starting services against an empty escrow account. Mid-pipeline statuses + // keep reserving even without claimTx — they are en route to payment. + // JS-side filter because payment lives in the JSON body, not a SQL column. + return this.mapServiceRows(rows).filter( + (j) => + (j.status !== ServiceStatusNumber.Error && + j.status !== ServiceStatusNumber.Stopped) || + !!j.payment?.claimTx + ) + } catch (err) { + DATABASE_LOGGER.error(err.message) + throw err + } + } + + // eslint-disable-next-line require-await + async getExpiredServiceJobs(clusterHash?: string): Promise { + // Running, Error AND Stopped all still hold their paid reservation (see activeStatuses + // above), so all three must be swept once past expiresAt: the sweep is the ONLY place + // the reservation is released. Without it an abandoned Error or Stopped service would + // keep its resources/ports forever and still read as restartable. Stopping is swept + // too: a process crash mid-stop persists that status and nothing else recovers it + // (it is not a pending-start status), so it would otherwise be stuck reserving + // resources forever — the sweep's doStopService handles a Stopping row like any + // other teardown (benign 404s for whatever the crashed stop already removed). + const expirableStatuses = [ + ServiceStatusNumber.Running, + ServiceStatusNumber.Error, + ServiceStatusNumber.Stopped, + ServiceStatusNumber.Stopping + ] + const placeholders = expirableStatuses.map(() => '?').join(',') + const params: Array = [...expirableStatuses, Date.now()] + let selectSQL = `SELECT * FROM service_jobs WHERE status IN (${placeholders}) AND expiresAt <= ?` + if (clusterHash) { + selectSQL += ` AND clusterHash = ?` + params.push(clusterHash) + } + try { + const rows = this.db.all(selectSQL, params) + return this.mapServiceRows(rows) + } catch (err) { + DATABASE_LOGGER.error(err.message) + throw err + } + } + + // Service jobs that are mid-start and need the background loop to advance them. + // Starting = fresh (handler just created it); the intermediate states are picked up too so + // the loop can resume / orphan-recover them after a node restart. + // eslint-disable-next-line require-await + async getPendingServiceStarts(clusterHash?: string): Promise { + const startStatuses = SERVICE_START_PENDING_STATUSES + const placeholders = startStatuses.map(() => '?').join(',') + const params: Array = [...startStatuses] + let selectSQL = `SELECT * FROM service_jobs WHERE status IN (${placeholders})` + if (clusterHash) { + selectSQL += ` AND clusterHash = ?` + params.push(clusterHash) + } + try { + const rows = this.db.all(selectSQL, params) + return this.mapServiceRows(rows) + } catch (err) { + DATABASE_LOGGER.error(err.message) + throw err + } + } + + // eslint-disable-next-line require-await + async updateImage(image: string): Promise { const timestamp = Math.floor(Date.now() / 1000) // Unix timestamp in seconds const insertSQL = ` INSERT OR REPLACE INTO docker_images (image, lastUsedTimestamp) VALUES (?, ?); ` - return new Promise((resolve, reject) => { - this.db.run(insertSQL, [image, timestamp], (err) => { - if (err) { - DATABASE_LOGGER.error( - `Could not update image usage for ${image}: ${err.message}` - ) - reject(err) - } else { - DATABASE_LOGGER.debug(`Updated image usage timestamp for ${image}`) - resolve() - } - }) - }) + try { + this.db.run(insertSQL, [image, timestamp]) + DATABASE_LOGGER.debug(`Updated image usage timestamp for ${image}`) + } catch (err) { + DATABASE_LOGGER.error(`Could not update image usage for ${image}: ${err.message}`) + throw err + } } - deleteImage(image: string): Promise { + // eslint-disable-next-line require-await + async deleteImage(image: string): Promise { const deleteSQL = ` DELETE FROM docker_images WHERE image = ?; ` - return new Promise((resolve, reject) => { - this.db.run(deleteSQL, [image], (err) => { - if (err) { - DATABASE_LOGGER.error(`Could not delete image ${image}: ${err.message}`) - reject(err) - } else { - DATABASE_LOGGER.debug(`Deleted image ${image}`) - resolve() - } - }) - }) + try { + this.db.run(deleteSQL, [image]) + DATABASE_LOGGER.debug(`Deleted image ${image}`) + } catch (err) { + DATABASE_LOGGER.error(`Could not delete image ${image}: ${err.message}`) + throw err + } } - getOldImages(retentionDays: number = 7): Promise { + // eslint-disable-next-line require-await + async getOldImages(retentionDays: number = 7): Promise { const cutoffTimestamp = Math.floor(Date.now() / 1000) - retentionDays * 24 * 60 * 60 const selectSQL = ` SELECT image FROM docker_images WHERE lastUsedTimestamp < ? ORDER BY lastUsedTimestamp ASC; ` - return new Promise((resolve, reject) => { - this.db.all(selectSQL, [cutoffTimestamp], (err, rows: any[] | undefined) => { - if (err) { - DATABASE_LOGGER.error(`Could not get old images: ${err.message}`) - reject(err) - } else { - const images = rows ? rows.map((row) => row.image) : [] - resolve(images) - } - }) - }) + try { + const rows = this.db.all<{ image: string }>(selectSQL, [cutoffTimestamp]) + return rows.map((row) => row.image) + } catch (err) { + DATABASE_LOGGER.error(`Could not get old images: ${err.message}`) + throw err + } } - newJob(job: DBComputeJob): Promise { + // eslint-disable-next-line require-await + async newJob(job: DBComputeJob): Promise { // TO DO C2D const insertSQL = ` - INSERT INTO ${this.schema.name} + INSERT INTO ${this.schema.name} ( - owner, - did, - jobId, - dateCreated, - status, - statusText, - inputDID, - algoDID, - agreementId, - expireTimestamp, - environment, + owner, + did, + jobId, + dateCreated, + status, + statusText, + inputDID, + algoDID, + agreementId, + expireTimestamp, + environment, body ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); ` - return new Promise((resolve, reject) => { - this.db.run( - insertSQL, - [ - job.owner, - job.did, - job.jobId, - job.dateCreated || String(Date.now() / 1000), // seconds from epoch, - job.status || C2DStatusNumber.JobStarted, - job.statusText || C2DStatusText.JobStarted, - job.inputDID ? convertArrayToString(job.inputDID) : job.inputDID, - job.algoDID, - job.agreementId, - job.maxJobDuration, - job.environment, - generateBlobFromJSON(job) - ], - (err) => { - if (err) { - DATABASE_LOGGER.error('Could not insert C2D job on DB: ' + err.message) - reject(err) - } else { - DATABASE_LOGGER.info('Successfully inserted job with id:' + job.jobId) - resolve(job.jobId) - } - } - ) - }) + try { + this.db.run(insertSQL, [ + job.owner, + job.did, + job.jobId, + job.dateCreated || String(Date.now() / 1000), // seconds from epoch, + job.status || C2DStatusNumber.JobStarted, + job.statusText || C2DStatusText.JobStarted, + job.inputDID ? convertArrayToString(job.inputDID) : job.inputDID, + job.algoDID, + job.agreementId, + job.maxJobDuration, + job.environment, + generateBlobFromJSON(job) + ]) + DATABASE_LOGGER.info('Successfully inserted job with id:' + job.jobId) + return job.jobId + } catch (err) { + DATABASE_LOGGER.error('Could not insert C2D job on DB: ' + err.message) + throw err + } } /** @@ -269,7 +542,12 @@ export class SQLiteCompute implements ComputeDatabaseProvider { * @param owner the consumer address / job owner * @returns job(s) */ - getJob(jobId?: string, agreementId?: string, owner?: string): Promise { + // eslint-disable-next-line require-await + async getJob( + jobId?: string, + agreementId?: string, + owner?: string + ): Promise { const params: any = [] let selectSQL = `SELECT * FROM ${this.schema.name} WHERE 1=1` if (jobId) { @@ -288,35 +566,33 @@ export class SQLiteCompute implements ComputeDatabaseProvider { params.push(owner) } - return new Promise((resolve, reject) => { - this.db.all(selectSQL, params, (err, rows: any[] | undefined) => { - if (err) { - DATABASE_LOGGER.error(err.message) - reject(err) - } else { - // also decode the internal data into job data - if (rows && rows.length > 0) { - const all: DBComputeJob[] = rows.map((row) => { - const body = generateJSONFromBlob(row.body) - delete row.body - const maxJobDuration = row.expireTimestamp - delete row.expireTimestamp - const job: DBComputeJob = { ...row, ...body, maxJobDuration } - return job - }) - resolve(all) - } else { - DATABASE_LOGGER.error( - `Could not find any job with jobId: ${jobId}, agreementId: ${agreementId}, or owner: ${owner} in database!` - ) - resolve([]) - } - } + let rows: any[] + try { + rows = this.db.all(selectSQL, params) + } catch (err) { + DATABASE_LOGGER.error(err.message) + throw err + } + // also decode the internal data into job data + if (rows && rows.length > 0) { + const all: DBComputeJob[] = rows.map((row) => { + const body = generateJSONFromBlob(row.body) + delete row.body + const maxJobDuration = row.expireTimestamp + delete row.expireTimestamp + const job: DBComputeJob = { ...row, ...body, maxJobDuration } + return job }) - }) + return all + } + DATABASE_LOGGER.error( + `Could not find any job with jobId: ${jobId}, agreementId: ${agreementId}, or owner: ${owner} in database!` + ) + return [] } - updateJob(job: DBComputeJob): Promise { + // eslint-disable-next-line require-await + async updateJob(job: DBComputeJob): Promise { // if (job.dateFinished && job.isRunning) { // job.isRunning = false // } @@ -331,76 +607,72 @@ export class SQLiteCompute implements ComputeDatabaseProvider { job.jobId ] const updateSQL = ` - UPDATE ${this.schema.name} - SET + UPDATE ${this.schema.name} + SET owner = ?, status = ?, statusText = ?, - expireTimestamp = ?, + expireTimestamp = ?, body = ?, dateFinished = ? WHERE jobId = ?; ` - return new Promise((resolve, reject) => { - this.db.run(updateSQL, data, function (this: RunResult, err: Error | null) { - if (err) { - DATABASE_LOGGER.error(`Error while updating job: ${err.message}`) - reject(err) - } else { - // number of rows updated successfully - resolve(this.changes) - } - }) - }) + try { + // number of rows updated successfully + const { changes } = this.db.run(updateSQL, data) + return changes + } catch (err) { + DATABASE_LOGGER.error(`Error while updating job: ${err.message}`) + throw err + } } - getRunningJobs(engine?: string, environment?: string): Promise { + // eslint-disable-next-line require-await + async getRunningJobs(engine?: string, environment?: string): Promise { const selectSQL = ` SELECT * FROM ${this.schema.name} WHERE dateFinished IS NULL ORDER by dateCreated ` - return new Promise((resolve, reject) => { - this.db.all(selectSQL, (err, rows: any[] | undefined) => { - if (err) { - DATABASE_LOGGER.error(err.message) - reject(err) - } else { - // also decode the internal data into job data - // get them all running - if (rows && rows.length > 0) { - const all: DBComputeJob[] = rows.map((row) => { - const body = generateJSONFromBlob(row.body) - delete row.body - const maxJobDuration = row.expireTimestamp - delete row.expireTimestamp - const job: DBComputeJob = { ...row, ...body, maxJobDuration } - return job - }) - // filter them out - const filtered = all.filter((job) => { - let include = true - if (engine && engine !== job.clusterHash) { - include = false - } - if (environment && environment !== job.environment) { - include = false - } - if (job.dateFinished) { - include = false - } - return include - }) - resolve(filtered) - } else { - // DATABASE_LOGGER.info('Could not find any running C2D jobs!') - resolve([]) - } + let rows: any[] + try { + rows = this.db.all(selectSQL) + } catch (err) { + DATABASE_LOGGER.error(err.message) + throw err + } + // also decode the internal data into job data + // get them all running + if (rows && rows.length > 0) { + const all: DBComputeJob[] = rows.map((row) => { + const body = generateJSONFromBlob(row.body) + delete row.body + const maxJobDuration = row.expireTimestamp + delete row.expireTimestamp + const job: DBComputeJob = { ...row, ...body, maxJobDuration } + return job + }) + // filter them out + const filtered = all.filter((job) => { + let include = true + if (engine && engine !== job.clusterHash) { + include = false } + if (environment && environment !== job.environment) { + include = false + } + if (job.dateFinished) { + include = false + } + return include }) - }) + return filtered + } + // DATABASE_LOGGER.info('Could not find any running C2D jobs!') + return [] } - getFinishedJobs(environments?: string[]): Promise { + // eslint-disable-next-line require-await + async getFinishedJobs(environments?: string[]): Promise { let selectSQL = ` SELECT * FROM ${this.schema.name} WHERE (dateFinished IS NOT NULL OR results IS NOT NULL) ` @@ -413,35 +685,32 @@ export class SQLiteCompute implements ComputeDatabaseProvider { selectSQL += ` ORDER BY dateFinished DESC` - return new Promise((resolve, reject) => { - this.db.all(selectSQL, params, (err, rows: any[] | undefined) => { - if (err) { - DATABASE_LOGGER.error(err.message) - reject(err) - } else { - // also decode the internal data into job data - // get them all running - if (rows && rows.length > 0) { - const all: DBComputeJob[] = rows.map((row) => { - const body = generateJSONFromBlob(row.body) - delete row.body - const maxJobDuration = row.expireTimestamp - delete row.expireTimestamp - const job: DBComputeJob = { ...row, ...body, maxJobDuration } - return job - }) - resolve(all) - } else { - environments - ? DATABASE_LOGGER.info( - 'No jobs found for the specified enviroments: ' + environments.join(',') - ) - : DATABASE_LOGGER.info('No jobs found') - resolve([]) - } - } + let rows: any[] + try { + rows = this.db.all(selectSQL, params) + } catch (err) { + DATABASE_LOGGER.error(err.message) + throw err + } + // also decode the internal data into job data + // get them all running + if (rows && rows.length > 0) { + const all: DBComputeJob[] = rows.map((row) => { + const body = generateJSONFromBlob(row.body) + delete row.body + const maxJobDuration = row.expireTimestamp + delete row.expireTimestamp + const job: DBComputeJob = { ...row, ...body, maxJobDuration } + return job }) - }) + return all + } + environments + ? DATABASE_LOGGER.info( + 'No jobs found for the specified enviroments: ' + environments.join(',') + ) + : DATABASE_LOGGER.info('No jobs found') + return [] } async getJobs( @@ -499,7 +768,7 @@ export class SQLiteCompute implements ComputeDatabaseProvider { ): Promise { let selectSQL = `SELECT * FROM ${this.schema.name}` - // sqlite3 bindings accept both strings and numbers; `status` is a numeric enum. + // node:sqlite bindings accept both strings and numbers; `status` is a numeric enum. const params: Array = [] const conditions: string[] = [] @@ -523,42 +792,40 @@ export class SQLiteCompute implements ComputeDatabaseProvider { return await this.doQuery(selectSQL, params, environments) } - private doQuery( + // eslint-disable-next-line require-await + private async doQuery( selectSQL: string, params: Array, environments: string[] - ) { - return new Promise((resolve, reject) => { - this.db.all(selectSQL, params, (err, rows: any[] | undefined) => { - if (err) { - DATABASE_LOGGER.error(err.message) - reject(err) - } else { - // also decode the internal data into job data - // get them all running - if (rows && rows.length > 0) { - const all: DBComputeJob[] = rows.map((row) => { - const body = generateJSONFromBlob(row.body) - delete row.body - const maxJobDuration = row.expireTimestamp - delete row.expireTimestamp - const job: DBComputeJob = { ...row, ...body, maxJobDuration } - if (!job.jobIdHash && job.jobId) { - job.jobIdHash = create256Hash(job.jobId) - } - return job - }) - resolve(all) - } else { - environments - ? DATABASE_LOGGER.info( - 'No jobs found for the specified enviroments: ' + environments.join(',') - ) - : DATABASE_LOGGER.info('No jobs found') - resolve([]) - } + ): Promise { + let rows: any[] + try { + rows = this.db.all(selectSQL, params) + } catch (err) { + DATABASE_LOGGER.error(err.message) + throw err + } + // also decode the internal data into job data + // get them all running + if (rows && rows.length > 0) { + const all: DBComputeJob[] = rows.map((row) => { + const body = generateJSONFromBlob(row.body) + delete row.body + const maxJobDuration = row.expireTimestamp + delete row.expireTimestamp + const job: DBComputeJob = { ...row, ...body, maxJobDuration } + if (!job.jobIdHash && job.jobId) { + job.jobIdHash = create256Hash(job.jobId) } + return job }) - }) + return all + } + environments + ? DATABASE_LOGGER.info( + 'No jobs found for the specified enviroments: ' + environments.join(',') + ) + : DATABASE_LOGGER.info('No jobs found') + return [] } } diff --git a/src/components/database/typesenseApi.ts b/src/components/database/typesenseApi.ts index 11e62f5f4..384cb0011 100644 --- a/src/components/database/typesenseApi.ts +++ b/src/components/database/typesenseApi.ts @@ -1,4 +1,3 @@ -import axios, { AxiosRequestConfig, AxiosResponse } from 'axios' import { setTimeout } from 'timers/promises' import { TypesenseConfig } from './typesenseConfig.js' import { TypesenseError } from './typesense.js' @@ -96,60 +95,61 @@ export class TypesenseApi { ) try { - const url = `${node.protocol}://${node.host}:${node.port}${endpoint}` - const requestOptions: AxiosRequestConfig = { - method: requestType, - url, - headers: { 'X-TYPESENSE-API-KEY': this.config.apiKey }, - maxContentLength: Infinity, - maxBodyLength: Infinity, - validateStatus: (status) => { - return status > 0 - }, - transformResponse: [ - (data, headers) => { - let transformedData = data - if ( - headers !== undefined && - typeof data === 'string' && - headers['content-type'] && - headers['content-type'].startsWith('application/json') - ) { - transformedData = JSON.parse(data) - } - return transformedData + const url = new URL(`${node.protocol}://${node.host}:${node.port}${endpoint}`) + if (queryParameters !== null) { + for (const [key, value] of Object.entries(queryParameters)) { + if (value !== undefined && value !== null) { + url.searchParams.set(key, String(value)) } - ] - } - - if (skipConnectionTimeout !== true) { - requestOptions.timeout = this.config.connectionTimeoutSeconds * 1000 + } } - if (queryParameters !== null) { - requestOptions.params = queryParameters + const init: RequestInit = { + method: requestType.toUpperCase(), + headers: { + 'X-TYPESENSE-API-KEY': this.config.apiKey, + ...(bodyParameters !== null && { 'content-type': 'application/json' }) + }, + // axios sent string bodies raw; only stringify non-string bodies so a + // pre-serialized payload (e.g. JSONL) isn't double-encoded + ...(bodyParameters !== null && { + body: + typeof bodyParameters === 'string' + ? bodyParameters + : JSON.stringify(bodyParameters) + }) } - if (bodyParameters !== null) { - requestOptions.data = bodyParameters + if (skipConnectionTimeout !== true) { + init.signal = AbortSignal.timeout(this.config.connectionTimeoutSeconds * 1000) } - const response = await axios(requestOptions) + const response = await fetch(url, init) this.config.logger.debug( `Request ${endpoint}: Request to Node ${node.host} was made. Response Code was ${response.status}.` ) + // fetch never throws on HTTP status and doesn't auto-parse — replicate + // axios's old transformResponse (JSON only when content-type says so). + const contentType = response.headers.get('content-type') ?? '' + const data: any = contentType.startsWith('application/json') + ? await response.json() + : await response.text() + if (response.status >= 200 && response.status < 300) { - return Promise.resolve(response.data) + return data } else if (response.status < 500) { - return Promise.reject(this.customError(response)) + return Promise.reject(this.customError(response.status, data)) } else { - throw new Error(response.data?.message) + // non-json error bodies (e.g. a proxy 5xx) arrive as a plain string + throw new Error(typeof data === 'string' ? data : data?.message) } } catch (error: any) { lastException = error this.config.logger.debug( - `Request ${endpoint}: Request to Node ${node.host} failed due to "${error.code} ${error.message}"` + `Request ${endpoint}: Request to Node ${node.host} failed due to "${ + error.cause?.code ?? error.code + } ${error.message}"` ) this.config.logger.debug( `Request ${endpoint}: Sleeping for ${this.config.retryIntervalSeconds}s and then retrying request...` @@ -161,9 +161,11 @@ export class TypesenseApi { return Promise.reject(lastException) } - customError(response: AxiosResponse): TypesenseError { - const error = new TypesenseError(response.data?.message) - error.httpStatus = response.status + customError(status: number, data: any): TypesenseError { + // non-json error bodies (e.g. a proxy 4xx) arrive as a plain string + const message = typeof data === 'string' ? data : data?.message + const error = new TypesenseError(message) + error.httpStatus = status return error } } diff --git a/src/components/httpRoutes/compute.ts b/src/components/httpRoutes/compute.ts index 1885678df..d8e68a780 100644 --- a/src/components/httpRoutes/compute.ts +++ b/src/components/httpRoutes/compute.ts @@ -21,8 +21,26 @@ import type { ComputeStopCommand, ComputeGetResultCommand, ComputeGetStatusCommand, - ComputeGetStreamableLogsCommand + ComputeGetStreamableLogsCommand, + ServiceGetTemplatesCommand, + ServiceStartCommand, + ServiceStopCommand, + ServiceExtendCommand, + ServiceRestartCommand, + ServiceGetStatusCommand, + GetServicesCommand, + ServiceGetStreamableLogsCommand } from '../../@types/commands.js' +import { + ServiceGetTemplatesHandler, + ServiceStartHandler, + ServiceStopHandler, + ServiceExtendHandler, + ServiceRestartHandler, + ServiceGetStatusHandler, + GetServicesHandler, + ServiceGetStreamableLogsHandler +} from '../core/service/index.js' import { streamToObject, streamToString } from '../../utils/util.js' import { PROTOCOL_COMMANDS, SERVICES_API_BASE_PATH } from '../../utils/constants.js' @@ -338,3 +356,182 @@ computeRoutes.post(`${SERVICES_API_BASE_PATH}/initializeCompute`, async (req, re computeRoutes.delete(`${SERVICES_API_BASE_PATH}/compute`, (req, res) => { res.status(404).send('Not yet implemented!') }) + +// ── Service on Demand ───────────────────────────────────────────────── + +async function runServiceCommand( + HandlerClass: any, + task: any, + res: express.Response +): Promise { + try { + const response = await new HandlerClass(res.req.oceanNode).handle(task) + if (response?.status?.httpStatus === 200) { + const result = await streamToObject(response.stream as Readable) + res.status(200).json(result) + } else { + HTTP_LOGGER.log(LOG_LEVELS_STR.LEVEL_INFO, `Error: ${response?.status?.error}`) + res.status(response?.status?.httpStatus || 500).json(response?.status?.error) + } + } catch (error) { + HTTP_LOGGER.log(LOG_LEVELS_STR.LEVEL_ERROR, `Error: ${error}`) + res.status(500).send('Internal Server Error') + } +} + +computeRoutes.get(`${SERVICES_API_BASE_PATH}/serviceTemplates`, async (req, res) => { + const task: ServiceGetTemplatesCommand = { + command: PROTOCOL_COMMANDS.SERVICE_GET_TEMPLATES, + chainId: parseInt(req.query.chainId as string) || undefined, + node: (req.query.node as string) || null, + caller: req.caller + } + await runServiceCommand(ServiceGetTemplatesHandler, task, res) +}) + +computeRoutes.post(`${SERVICES_API_BASE_PATH}/serviceStart`, async (req, res) => { + const task: ServiceStartCommand = { + command: PROTOCOL_COMMANDS.SERVICE_START, + node: (req.body.node as string) || null, + consumerAddress: (req.body.consumerAddress as string) || null, + nonce: (req.body.nonce as string) || null, + signature: (req.body.signature as string) || null, + environment: (req.body.environment as string) || null, + image: (req.body.image as string) || null, + tag: (req.body.tag as string) || undefined, + checksum: (req.body.checksum as string) || undefined, + dockerfile: (req.body.dockerfile as string) || undefined, + additionalDockerFiles: req.body.additionalDockerFiles || undefined, + dockerCmd: (req.body.dockerCmd as string[]) || undefined, + dockerEntrypoint: (req.body.dockerEntrypoint as string[]) || undefined, + exposedPorts: (req.body.exposedPorts as number[]) || undefined, + resources: (req.body.resources as ComputeResourceRequest[]) || undefined, + duration: req.body.duration as number, + userData: (req.body.userData as string) || undefined, + payment: req.body.payment, + authorization: req.headers?.authorization, + caller: req.caller + } + await runServiceCommand(ServiceStartHandler, task, res) +}) + +computeRoutes.post(`${SERVICES_API_BASE_PATH}/serviceStop`, async (req, res) => { + const task: ServiceStopCommand = { + command: PROTOCOL_COMMANDS.SERVICE_STOP, + node: (req.body.node as string) || null, + consumerAddress: (req.body.consumerAddress as string) || null, + nonce: (req.body.nonce as string) || null, + signature: (req.body.signature as string) || null, + serviceId: (req.body.serviceId as string) || null, + authorization: req.headers?.authorization, + caller: req.caller + } + await runServiceCommand(ServiceStopHandler, task, res) +}) + +computeRoutes.post(`${SERVICES_API_BASE_PATH}/serviceExtend`, async (req, res) => { + const task: ServiceExtendCommand = { + command: PROTOCOL_COMMANDS.SERVICE_EXTEND, + node: (req.body.node as string) || null, + consumerAddress: (req.body.consumerAddress as string) || null, + nonce: (req.body.nonce as string) || null, + signature: (req.body.signature as string) || null, + serviceId: (req.body.serviceId as string) || null, + additionalDuration: req.body.additionalDuration as number, + payment: req.body.payment, + authorization: req.headers?.authorization, + caller: req.caller + } + await runServiceCommand(ServiceExtendHandler, task, res) +}) + +computeRoutes.post(`${SERVICES_API_BASE_PATH}/serviceRestart`, async (req, res) => { + const task: ServiceRestartCommand = { + command: PROTOCOL_COMMANDS.SERVICE_RESTART, + node: (req.body.node as string) || null, + consumerAddress: (req.body.consumerAddress as string) || null, + nonce: (req.body.nonce as string) || null, + signature: (req.body.signature as string) || null, + serviceId: (req.body.serviceId as string) || null, + userData: (req.body.userData as string) || undefined, + dockerCmd: (req.body.dockerCmd as string[]) || undefined, + dockerEntrypoint: (req.body.dockerEntrypoint as string[]) || undefined, + authorization: req.headers?.authorization, + caller: req.caller + } + await runServiceCommand(ServiceRestartHandler, task, res) +}) + +computeRoutes.get(`${SERVICES_API_BASE_PATH}/serviceStatus`, async (req, res) => { + const task: ServiceGetStatusCommand = { + command: PROTOCOL_COMMANDS.SERVICE_GET_STATUS, + consumerAddress: req.query.consumerAddress as string, + nonce: req.query.nonce as string, + signature: req.query.signature as string, + serviceId: (req.query.serviceId as string) || undefined, + node: (req.query.node as string) || null, + authorization: req.headers?.authorization, + caller: req.caller + } + await runServiceCommand(ServiceGetStatusHandler, task, res) +}) + +// node-wide service listing (any owner). Default: only services currently holding a +// resource reservation; `status` filters to one specific status, `includeAllStatuses` +// returns everything, `fromTimestamp` keeps services created at/after that moment. +computeRoutes.get(`${SERVICES_API_BASE_PATH}/serviceList`, async (req, res) => { + const task: GetServicesCommand = { + command: PROTOCOL_COMMANDS.SERVICE_LIST, + consumerAddress: req.query.consumerAddress as string, + nonce: req.query.nonce as string, + signature: req.query.signature as string, + status: req.query.status !== undefined ? Number(req.query.status) : undefined, + includeAllStatuses: req.query.includeAllStatuses === 'true', + fromTimestamp: (req.query.fromTimestamp as string) || undefined, + node: (req.query.node as string) || null, + authorization: req.headers?.authorization, + caller: req.caller + } + await runServiceCommand(GetServicesHandler, task, res) +}) + +// streaming logs for services +computeRoutes.get(`${SERVICES_API_BASE_PATH}/serviceStreamableLogs`, async (req, res) => { + try { + HTTP_LOGGER.logMessage( + `ServiceGetStreamableLogsCommand request received with query: ${JSON.stringify( + req.query + )}`, + true + ) + + const task: ServiceGetStreamableLogsCommand = { + command: PROTOCOL_COMMANDS.SERVICE_GET_STREAMABLE_LOGS, + node: (req.query.node as string) || null, + consumerAddress: (req.query.consumerAddress as string) || null, + serviceId: (req.query.serviceId as string) || null, + signature: (req.query.signature as string) || null, + nonce: (req.query.nonce as string) || null, + since: (req.query.since as string) || undefined, + authorization: req.headers?.authorization, + caller: req.caller + } + + const response = await new ServiceGetStreamableLogsHandler(req.oceanNode).handle(task) + if (response.stream) { + res.status(response.status.httpStatus) + res.set(response.status.headers) + response.stream.pipe(res) + } else { + const body = + response.status.error ?? + (response.status.httpStatus === 404 + ? 'Service not found or not running' + : 'Error') + res.status(response.status.httpStatus).send(body) + } + } catch (error) { + HTTP_LOGGER.log(LOG_LEVELS_STR.LEVEL_ERROR, `Error: ${error}`) + res.status(500).send('Internal Server Error') + } +}) diff --git a/src/components/persistentStorage/PersistentStorageFactory.ts b/src/components/persistentStorage/PersistentStorageFactory.ts index c00baf151..380ecd92e 100644 --- a/src/components/persistentStorage/PersistentStorageFactory.ts +++ b/src/components/persistentStorage/PersistentStorageFactory.ts @@ -6,9 +6,7 @@ import type { PersistentStorageObject } from '../../@types/PersistentStorage.js' -import sqlite3, { RunResult } from 'sqlite3' -import path from 'path' -import fs from 'fs' +import { SqliteClient } from '../database/sqliteClient.js' import { getAddress } from 'ethers' import { OceanNode } from '../../OceanNode.js' import { checkAddressOnAccessList } from '../../utils/accessList.js' @@ -69,19 +67,16 @@ export type PersistentStorageBucketRecord = { } export abstract class PersistentStorageFactory { - private db: sqlite3.Database + private db: SqliteClient private node: OceanNode private dbReady = false private dbReadyPromise: Promise constructor(node: OceanNode) { this.node = node - const dbDir = path.dirname('databases/persistentStorage.sqlite') - if (!fs.existsSync(dbDir)) { - fs.mkdirSync(dbDir, { recursive: true }) - } - this.db = new sqlite3.Database('databases/persistentStorage.sqlite') - const createBucketsSQL = ` + // SqliteClient creates the parent directory on construction. + this.db = new SqliteClient('databases/persistentStorage.sqlite') + this.db.exec(` CREATE TABLE IF NOT EXISTS persistent_storage_buckets ( bucketId TEXT PRIMARY KEY, owner TEXT NOT NULL, @@ -89,29 +84,20 @@ export abstract class PersistentStorageFactory { createdAt INTEGER NOT NULL, label TEXT ); - ` - this.dbReadyPromise = new Promise((resolve, reject) => { - this.db.run(createBucketsSQL, (err) => { - if (err) { - reject(err) - return - } - // Migration: add the label column if it doesn't exist - this.db.run( - `ALTER TABLE persistent_storage_buckets ADD COLUMN label TEXT`, - (alterErr) => { - // Ignore "duplicate column name" (expected once the column exists); - // surface any other failure instead of starting with a broken schema. - if (alterErr && !/duplicate column name/i.test(alterErr.message)) { - reject(alterErr) - return - } - this.dbReady = true - resolve() - } - ) - }) - }) + `) + // Migration: add the label column if it doesn't exist. A fresh table already has it, + // so ALTER throws "duplicate column name" — swallow only that; surface any other + // failure instead of starting with a broken schema. Schema setup is synchronous now, + // so the DB is ready by the time the constructor returns. + try { + this.db.exec(`ALTER TABLE persistent_storage_buckets ADD COLUMN label TEXT`) + } catch (alterErr) { + if (!/duplicate column name/i.test(alterErr.message)) { + throw alterErr + } + } + this.dbReady = true + this.dbReadyPromise = Promise.resolve() } public isDbReady(): boolean { @@ -261,7 +247,7 @@ export abstract class PersistentStorageFactory { * with constructor-time schema creation. */ - dbUpsertBucket( + async dbUpsertBucket( bucketId: string, owner: string, accessListJson: string, @@ -274,71 +260,39 @@ export abstract class PersistentStorageFactory { VALUES (?, ?, ?, ?, ?) ON CONFLICT(bucketId) DO UPDATE SET accessListJson=excluded.accessListJson; ` - return this.ensureDbReady().then( - () => - new Promise((resolve, reject) => { - this.db.run(sql, [bucketId, owner, accessListJson, createdAt, label], (err) => { - if (err) reject(err) - else resolve() - }) - }) - ) + await this.ensureDbReady() + this.db.run(sql, [bucketId, owner, accessListJson, createdAt, label]) } - dbGetBucket(bucketId: string): Promise { + async dbGetBucket(bucketId: string): Promise { const sql = `SELECT bucketId, owner, accessListJson, createdAt, label FROM persistent_storage_buckets WHERE bucketId = ?` - return this.ensureDbReady().then( - () => - new Promise((resolve, reject) => { - this.db.get(sql, [bucketId], (err, row: BucketRow | undefined) => { - if (err) reject(err) - else resolve(row ?? null) - }) - }) - ) + await this.ensureDbReady() + const row = this.db.get(sql, [bucketId]) + return row ?? null } - dbListBucketsByOwner(owner: string): Promise { + async dbListBucketsByOwner(owner: string): Promise { const sql = `SELECT bucketId, owner, accessListJson, createdAt, label FROM persistent_storage_buckets WHERE owner = ? ORDER BY createdAt ASC` - return this.ensureDbReady().then( - () => - new Promise((resolve, reject) => { - this.db.all(sql, [owner], (err, rows: BucketRow[]) => { - if (err) reject(err) - else resolve(rows ?? []) - }) - }) - ) + await this.ensureDbReady() + return this.db.all(sql, [owner]) } - dbDeleteBucket(bucketId: string): Promise { + async dbDeleteBucket(bucketId: string): Promise { const sql = `DELETE FROM persistent_storage_buckets WHERE bucketId = ?` - return this.ensureDbReady().then( - () => - new Promise((resolve, reject) => { - this.db.run(sql, [bucketId], function (this: RunResult, err) { - if (err) reject(err) - else resolve(this.changes === 1) - }) - }) - ) + await this.ensureDbReady() + const { changes } = this.db.run(sql, [bucketId]) + return changes === 1 } - dbUpdateBucketLabel( + async dbUpdateBucketLabel( bucketId: string, owner: string, label: string | null ): Promise { const sql = `UPDATE persistent_storage_buckets SET label = ? WHERE bucketId = ? AND owner = ?` - return this.ensureDbReady().then( - () => - new Promise((resolve, reject) => { - this.db.run(sql, [label, bucketId, owner], function (this: RunResult, err) { - if (err) reject(err) - else resolve(this.changes === 1) - }) - }) - ) + await this.ensureDbReady() + const { changes } = this.db.run(sql, [label, bucketId, owner]) + return changes === 1 } isAllowed(consumerAddress: string, accessLists: AccessList[]): Promise { diff --git a/src/components/storage/ArweaveStorage.ts b/src/components/storage/ArweaveStorage.ts index b4333b6b8..2bf18a5ab 100644 --- a/src/components/storage/ArweaveStorage.ts +++ b/src/components/storage/ArweaveStorage.ts @@ -5,8 +5,8 @@ import { } from '../../@types/fileObject.js' import { OceanNodeConfig } from '../../@types/OceanNode.js' import { fetchFileMetadata } from '../../utils/asset.js' +import { fetchStream } from '../../utils/http.js' import urlJoin from 'url-join' -import axios from 'axios' import { Storage } from './Storage.js' export class ArweaveStorage extends Storage { @@ -55,18 +55,7 @@ export class ArweaveStorage extends Storage { override async getReadableStream(): Promise { const input = this.getDownloadUrl() - const response = await axios({ - method: 'get', - url: input, - responseType: 'stream', - timeout: 30000 - }) - - return { - httpStatus: response.status, - stream: response.data, - headers: response.headers as any - } + return await fetchStream(input, { method: 'GET' }, 30000) } async fetchSpecificFileMetadata( diff --git a/src/components/storage/IpfsStorage.ts b/src/components/storage/IpfsStorage.ts index 5a3070b31..ac62e41e6 100644 --- a/src/components/storage/IpfsStorage.ts +++ b/src/components/storage/IpfsStorage.ts @@ -5,8 +5,8 @@ import { } from '../../@types/fileObject.js' import { OceanNodeConfig } from '../../@types/OceanNode.js' import { fetchFileMetadata } from '../../utils/asset.js' +import { fetchStream } from '../../utils/http.js' import urlJoin from 'url-join' -import axios from 'axios' import { Storage } from './Storage.js' export class IpfsStorage extends Storage { @@ -49,18 +49,7 @@ export class IpfsStorage extends Storage { override async getReadableStream(): Promise { const input = this.getDownloadUrl() - const response = await axios({ - method: 'get', - url: input, - responseType: 'stream', - timeout: 30000 - }) - - return { - httpStatus: response.status, - stream: response.data, - headers: response.headers as any - } + return await fetchStream(input, { method: 'GET' }, 30000) } async fetchSpecificFileMetadata( diff --git a/src/components/storage/UrlStorage.ts b/src/components/storage/UrlStorage.ts index 509f24130..107f64c09 100644 --- a/src/components/storage/UrlStorage.ts +++ b/src/components/storage/UrlStorage.ts @@ -6,7 +6,7 @@ import { } from '../../@types/fileObject.js' import { OceanNodeConfig } from '../../@types/OceanNode.js' import { fetchFileMetadata } from '../../utils/asset.js' -import axios from 'axios' +import { fetchStream, fetchHeadersTimeout, headersToObject } from '../../utils/http.js' import { Storage } from './Storage.js' @@ -23,19 +23,8 @@ export class UrlStorage extends Storage { const input = this.getDownloadUrl() const file = this.getFile() const { headers } = file - const response = await axios({ - method: 'get', - url: input, - headers, - responseType: 'stream', - timeout: 30000 - }) - - return { - httpStatus: response.status, - stream: response.data, - headers: response.headers as any - } + // download always uses GET regardless of file.method (which applies to metadata fetch) + return await fetchStream(input, { method: 'GET', headers }, 30000) } /** @@ -57,18 +46,28 @@ export class UrlStorage extends Storage { ...(fileHeaders ?? {}), 'Content-Disposition': `attachment; filename="${filename.replace(/"/g, '\\"')}"` } - const response = await axios({ - method: 'put', + const response = await fetchHeadersTimeout( url, - data: stream, - headers, - timeout: 30000, - maxBodyLength: Infinity, - maxContentLength: Infinity - }) + { + method: 'PUT', + headers, + // streaming a Node Readable as the request body requires a web stream + // plus duplex: 'half' (mandatory in undici) — chunked, no content-length. + // duplex isn't in the DOM RequestInit type, so cast the whole init. + body: Readable.toWeb(stream) as any, + duplex: 'half' + } as RequestInit, + 30000 + ) + if (!response.ok) { + // axios threw on non-2xx PUT; the caller relies on that to fail the job. + // cancel the undrained body so undici releases the socket back to the pool. + await response.body?.cancel().catch(() => {}) + throw new Error(`Upload failed with status code ${response.status} (${url})`) + } return { httpStatus: response.status, - headers: response.headers as Record + headers: headersToObject(response.headers) } } diff --git a/src/test/integration/algorithmsAccess.test.ts b/src/test/integration/algorithmsAccess.test.ts index 2272f7a79..3b7e08e20 100644 --- a/src/test/integration/algorithmsAccess.test.ts +++ b/src/test/integration/algorithmsAccess.test.ts @@ -146,7 +146,11 @@ describe('********** Trusted algorithms Flow', () => { // let's publish assets & algos it('should publish compute datasets & algos', async function () { - this.timeout(DEFAULT_TEST_TIMEOUT * 2) + // This suite runs after the AccessList suites, which leave many AddressAdded/NewAccessList + // events on the shared dev chain. The indexer drains that backlog (~1s/event) before it + // reaches these DDOs, so the default 15s effective wait can expire just before the DDO is + // saved. Give waitToIndex a longer window (and a matching mocha timeout) to absorb it. + this.timeout(DEFAULT_TEST_TIMEOUT * 6) publishedComputeDataset = await publishAsset( computeAssetWithNoAccess, publisherAccount @@ -156,7 +160,7 @@ describe('********** Trusted algorithms Flow', () => { oceanNode, publishedComputeDataset.ddo.id, EVENTS.METADATA_CREATED, - DEFAULT_TEST_TIMEOUT + DEFAULT_TEST_TIMEOUT * 2 ) // Fail the test if compute dataset DDO was not indexed - subsequent tests depend on it assert( @@ -169,7 +173,7 @@ describe('********** Trusted algorithms Flow', () => { oceanNode, publishedAlgoDataset.ddo.id, EVENTS.METADATA_CREATED, - DEFAULT_TEST_TIMEOUT + DEFAULT_TEST_TIMEOUT * 2 ) // Fail the test if algorithm DDO was not indexed - subsequent tests depend on it assert( diff --git a/src/test/integration/auth.test.ts b/src/test/integration/auth.test.ts index 54dd5608e..905bbfe75 100644 --- a/src/test/integration/auth.test.ts +++ b/src/test/integration/auth.test.ts @@ -126,7 +126,8 @@ describe('********** Auth Token Integration Tests', () => { const messageHash = createHashForSignature( await consumerAccount.getAddress(), nonce, - PROTOCOL_COMMANDS.CREATE_AUTH_TOKEN + PROTOCOL_COMMANDS.CREATE_AUTH_TOKEN, + oceanNode.getKeyManager().getPeerIdString() ) const signature = await safeSign(consumerAccount, messageHash) @@ -150,7 +151,8 @@ describe('********** Auth Token Integration Tests', () => { const messageHash = createHashForSignature( consumerAddress, nonce, - PROTOCOL_COMMANDS.CREATE_AUTH_TOKEN + PROTOCOL_COMMANDS.CREATE_AUTH_TOKEN, + oceanNode.getKeyManager().getPeerIdString() ) const signature = await safeSign(consumerAccount, messageHash) @@ -179,7 +181,8 @@ describe('********** Auth Token Integration Tests', () => { const messageHash = createHashForSignature( consumerAddress, nonce, - PROTOCOL_COMMANDS.CREATE_AUTH_TOKEN + PROTOCOL_COMMANDS.CREATE_AUTH_TOKEN, + oceanNode.getKeyManager().getPeerIdString() ) const signature = await safeSign(consumerAccount, messageHash) @@ -192,11 +195,17 @@ describe('********** Auth Token Integration Tests', () => { const token = await streamToObject(handlerResponse.stream as Readable) const newNonce = getRandomNonce() + const invalidateHash = createHashForSignature( + consumerAddress, + newNonce, + PROTOCOL_COMMANDS.INVALIDATE_AUTH_TOKEN + ) + const invalidateSignature = await safeSign(consumerAccount, invalidateHash) await new InvalidateAuthTokenHandler(oceanNode).handle({ command: PROTOCOL_COMMANDS.INVALIDATE_AUTH_TOKEN, address: consumerAddress, - signature, + signature: invalidateSignature, nonce: newNonce, token: token.token }) @@ -244,7 +253,8 @@ describe('********** Auth Token Integration Tests', () => { const messageHash = createHashForSignature( await consumerAccount.getAddress(), nonce, - PROTOCOL_COMMANDS.CREATE_AUTH_TOKEN + PROTOCOL_COMMANDS.CREATE_AUTH_TOKEN, + oceanNode.getKeyManager().getPeerIdString() ) const signature = await safeSign(consumerAccount, messageHash) diff --git a/src/test/integration/escrow.test.ts b/src/test/integration/escrow.test.ts index b0a2c1e4d..82b87801b 100644 --- a/src/test/integration/escrow.test.ts +++ b/src/test/integration/escrow.test.ts @@ -196,6 +196,7 @@ describe('Indexer stores Escrow contract events', () => { const event = events[0] expect(event.payer).to.equal(payerAddress.toLowerCase()) expect(event.payee).to.equal(payeeAddress.toLowerCase()) + expect(event.token).to.equal(paymentToken.toLowerCase()) expect(event.maxLockedAmount).to.equal(depositAmount.toString()) expect(event.maxLockCounts).to.equal('10') }) @@ -223,6 +224,39 @@ describe('Indexer stores Escrow contract events', () => { expect(event.token).to.equal(paymentToken.toLowerCase()) }) + it('indexes a ReLock event', async function () { + if (!escrowAddress || !paymentToken) this.skip() + this.timeout(DEFAULT_TEST_TIMEOUT * 3) + + // reLock adjusts the amount/expiry of the lock created in the previous test + // for the same jobId, preserving the original creation timestamp. The new + // expiry is a duration-from-now capped by (creationTimestamp + maxLockSeconds), + // so use a value well below `expiry` (== maxLockSeconds) to leave headroom for + // the wall-clock time spent waiting on the Lock event to be indexed above. + const newLockAmount = parseUnits('20', 18) + const reLockExpiry = Math.floor(expiry / 2) + const tx = await escrowContract + .connect(publisherAccount) + .reLock(jobId, paymentToken, payerAddress, newLockAmount, reLockExpiry) + const receipt = await tx.wait() + const reLockTxHash = receipt.hash + + const events = await waitForEscrowEvents({ + txHash: reLockTxHash, + eventType: EVENTS.ESCROW_RELOCK + }) + assert(events && events.length > 0, 'ReLock event should be indexed') + const event = events[0] + expect(event.payer).to.equal(payerAddress.toLowerCase()) + expect(event.payee).to.equal(payeeAddress.toLowerCase()) + expect(event.jobId).to.equal(jobId.toString()) + expect(event.oldAmount).to.equal(lockAmount.toString()) + expect(event.newAmount).to.equal(newLockAmount.toString()) + expect(event.token).to.equal(paymentToken.toLowerCase()) + // expiry is emitted as an absolute timestamp; just assert it is populated + assert(event.newExpiry, 'newExpiry should be populated') + }) + it('returns indexed events through the EscrowEventsHandler (query command)', async function () { if (!escrowAddress || !paymentToken) this.skip() this.timeout(DEFAULT_TEST_TIMEOUT) @@ -246,7 +280,7 @@ describe('Indexer stores Escrow contract events', () => { it('respects offset and size pagination', async function () { if (!escrowAddress || !paymentToken) this.skip() - // Deposit, Auth and Lock are all indexed for this chain by now (>= 2 rows). + // Deposit, Auth, Lock and ReLock are all indexed for this chain by now (>= 2 rows). const page = await database.escrow.search({ chainId }, 0, 2) assert(page && page.length === 2, 'size should cap the page to 2 rows') diff --git a/src/test/integration/imageCleanup.test.ts b/src/test/integration/imageCleanup.test.ts index 625d9e2f2..acabc7482 100644 --- a/src/test/integration/imageCleanup.test.ts +++ b/src/test/integration/imageCleanup.test.ts @@ -90,19 +90,13 @@ describe('********** Docker Image Cleanup Integration Tests', () => { const testImage = 'test-image:latest' await sqliteProvider.updateImage(testImage) - // Verify image was recorded by querying directly - // getOldImages(0) looks for images older than now, so we query the DB directly - const imageRecord = await new Promise((resolve, reject) => { - const { db } = sqliteProvider as any - db.get( - 'SELECT image, lastUsedTimestamp FROM docker_images WHERE image = ?', - [testImage], - (err: Error | null, row: any) => { - if (err) reject(err) - else resolve(row) - } - ) - }) + // Verify image was recorded by querying directly via the SqliteClient handle. + // getOldImages(0) looks for images older than now, so we query the DB directly. + const sqliteClient = (sqliteProvider as any).db + const imageRecord = sqliteClient.get( + 'SELECT image, lastUsedTimestamp FROM docker_images WHERE image = ?', + [testImage] + ) assert(imageRecord, 'Image should be recorded in database') expect(imageRecord.image).to.equal(testImage) @@ -125,17 +119,11 @@ describe('********** Docker Image Cleanup Integration Tests', () => { await sqliteProvider.updateImage(testImage) // Verify image exists with updated timestamp - const imageRecord = await new Promise((resolve, reject) => { - const { db } = sqliteProvider as any - db.get( - 'SELECT image, lastUsedTimestamp FROM docker_images WHERE image = ?', - [testImage], - (err: Error | null, row: any) => { - if (err) reject(err) - else resolve(row) - } - ) - }) + const sqliteClient = (sqliteProvider as any).db + const imageRecord = sqliteClient.get( + 'SELECT image, lastUsedTimestamp FROM docker_images WHERE image = ?', + [testImage] + ) assert(imageRecord, 'Image should be recorded in database') expect(imageRecord.image).to.equal(testImage) @@ -151,17 +139,11 @@ describe('********** Docker Image Cleanup Integration Tests', () => { // Manually insert an old image by directly updating the database const oldTimestamp = Math.floor(Date.now() / 1000) - 8 * 24 * 60 * 60 // 8 days ago - await new Promise((resolve, reject) => { - const { db } = sqliteProvider as any - db.run( - 'INSERT OR REPLACE INTO docker_images (image, lastUsedTimestamp) VALUES (?, ?)', - [oldImage, oldTimestamp], - (err: Error | null) => { - if (err) reject(err) - else resolve(undefined) - } - ) - }) + const sqliteClient = (sqliteProvider as any).db + sqliteClient.run( + 'INSERT OR REPLACE INTO docker_images (image, lastUsedTimestamp) VALUES (?, ?)', + [oldImage, oldTimestamp] + ) // Get images older than 7 days const oldImages = await sqliteProvider.getOldImages(7) @@ -206,17 +188,11 @@ describe('********** Docker Image Cleanup Integration Tests', () => { await (dockerEngine as any).updateImageUsage(testImage) // Verify image was recorded in database - const imageRecord = await new Promise((resolve, reject) => { - const { db } = sqliteProvider as any - db.get( - 'SELECT image, lastUsedTimestamp FROM docker_images WHERE image = ?', - [testImage], - (err: Error | null, row: any) => { - if (err) reject(err) - else resolve(row) - } - ) - }) + const sqliteClient = (sqliteProvider as any).db + const imageRecord = sqliteClient.get( + 'SELECT image, lastUsedTimestamp FROM docker_images WHERE image = ?', + [testImage] + ) assert(imageRecord, 'Image should be recorded in database after updateImageUsage') expect(imageRecord.image).to.equal(testImage) @@ -284,17 +260,11 @@ describe('********** Docker Image Cleanup Integration Tests', () => { // Track the image with old timestamp (8 days ago) const oldTimestamp = Math.floor(Date.now() / 1000) - 8 * 24 * 60 * 60 - await new Promise((resolve, reject) => { - const { db } = sqliteProvider as any - db.run( - 'INSERT OR REPLACE INTO docker_images (image, lastUsedTimestamp) VALUES (?, ?)', - [testImageName, oldTimestamp], - (err: Error | null) => { - if (err) reject(err) - else resolve(undefined) - } - ) - }) + const sqliteClient = (sqliteProvider as any).db + sqliteClient.run( + 'INSERT OR REPLACE INTO docker_images (image, lastUsedTimestamp) VALUES (?, ?)', + [testImageName, oldTimestamp] + ) // Verify image exists before cleanup const imagesBefore = await docker.listImages() diff --git a/src/test/integration/services.test.ts b/src/test/integration/services.test.ts new file mode 100644 index 000000000..2157ad38b --- /dev/null +++ b/src/test/integration/services.test.ts @@ -0,0 +1,1089 @@ +import { expect, assert } from 'chai' +import { + ServiceGetTemplatesHandler, + ServiceStartHandler, + ServiceStopHandler, + ServiceExtendHandler, + ServiceRestartHandler, + ServiceGetStatusHandler, + GetServicesHandler, + ServiceGetStreamableLogsHandler +} from '../../components/core/service/index.js' +import { ComputeGetEnvironmentsHandler } from '../../components/core/compute/index.js' +import type { + ServiceGetTemplatesCommand, + ServiceStartCommand, + ServiceStopCommand, + ServiceExtendCommand, + ServiceRestartCommand, + ServiceGetStatusCommand, + GetServicesCommand, + ServiceGetStreamableLogsCommand +} from '../../@types/commands.js' +import { + ServiceStatusNumber, + type ServiceJob, + type ServiceTemplatePublic +} from '../../@types/C2D/ServiceOnDemand.js' +import type { ComputeEnvironment } from '../../@types/C2D/C2D.js' +import { + ENVIRONMENT_VARIABLES, + PROTOCOL_COMMANDS, + getConfiguration +} from '../../utils/index.js' +import { Database } from '../../components/database/index.js' +import { OceanNode } from '../../OceanNode.js' +import { OceanNodeConfig } from '../../@types/OceanNode.js' +import { Readable } from 'stream' +import { streamToObject } from '../../utils/util.js' +import { ethers, JsonRpcProvider, Signer } from 'ethers' +import { RPCS } from '../../@types/blockchain.js' +import { + DEFAULT_TEST_TIMEOUT, + OverrideEnvConfig, + TEST_ENV_CONFIG_FILE, + buildEnvOverrideConfig, + getMockSupportedNetworks, + setupEnvironment, + tearDownEnvironment, + sleep +} from '../utils/utils.js' +import { DEVELOPMENT_CHAIN_ID, getOceanArtifactsAdresses } from '../../utils/address.js' +import OceanToken from '@oceanprotocol/contracts/artifacts/contracts/utils/OceanToken.sol/OceanToken.json' with { type: 'json' } +import EscrowJson from '@oceanprotocol/contracts/artifacts/contracts/escrow/Escrow.sol/Escrow.json' with { type: 'json' } +import { EncryptMethod } from '../../@types/fileObject.js' +import { createHashForSignature, safeSign } from '../utils/signature.js' +import { C2DEngineDocker } from '../../components/c2d/compute_engine_docker.js' +import Dockerode from 'dockerode' +import fsp from 'fs/promises' +import path from 'path' +import { tmpdir } from 'os' + +const TEMPLATE_ID = 'nginx-demo' +const MAX_DURATION = 600 // serviceOnDemand.maxDurationSeconds +const SERVICE_DURATION = 300 // long-lived service used through tests (d)→(l) +const EXPIRY_DURATION = 60 // short service for the expiry-cron test +const PORT_RANGE_START = 39000 +const PORT_RANGE_END = 39500 + +const TEMPLATE_JSON = { + id: TEMPLATE_ID, + name: 'Nginx demo', + image: 'nginxinc/nginx-unprivileged', + tag: 'alpine', + exposedPorts: [8080], + requiredResources: [ + { id: 'cpu', min: 1 }, + { id: 'ram', min: 1 } + ], + userConfigurableEnvVars: [{ key: 'EXTRA', validation: '^[a-zA-Z0-9]{1,20}$' }] +} + +describe('********** Service on Demand', () => { + let previousConfiguration: OverrideEnvConfig[] + let config: OceanNodeConfig + let dbconn: Database + let oceanNode: OceanNode + let provider: JsonRpcProvider + let publisherAccount: Signer + let consumerAccount: Signer + let nonOwnerAccount: Signer + let consumerAddress: string + let paymentToken: any + let paymentTokenContract: any + let escrowContract: any + let artifactsAddresses: any + let serviceTemplatesPath: string + let servicesEnv: ComputeEnvironment + let noServicesEnv: ComputeEnvironment + + // state threaded through the lifecycle tests + let serviceId: string + let hostPort: number + let expiresAt: number + let endpointUrl: string + const startedServices: string[] = [] + + const mockSupportedNetworks: RPCS = getMockSupportedNetworks() + + // ── helpers ────────────────────────────────────────────────────────── + + async function signFor(signer: Signer, command: string) { + const addr = await signer.getAddress() + const nonce = Date.now().toString() + const hash = createHashForSignature(addr, nonce, command) + const signature = await safeSign(signer, hash) + return { consumerAddress: addr, nonce, signature } + } + + async function encryptUserData(obj: Record): Promise { + const enc = await oceanNode + .getKeyManager() + .encrypt(new Uint8Array(Buffer.from(JSON.stringify(obj))), EncryptMethod.ECIES) + return Buffer.from(enc).toString('hex') + } + + async function fundEscrow(beneficiaryNodeAddr: string, durationForLock: number) { + // Always mint a large top-up rather than only when the balance is 0. Integration suites + // share one dev chain and run in sequence; by the time this suite runs, earlier suites + // (e.g. compute) have left locked funds against the same (token, payer, node beneficiary) + // and drained the wallet. Setting maxLockedAmount from a small leftover balance would push + // it below the already-accumulated currentLockedAmount → "will go over limit". A large + // deposit + authorization ceiling clears that leftover and covers all locks in (d)→(l). + const mintTx = await paymentTokenContract.mint( + consumerAddress, + ethers.parseUnits('1000000', 18) + ) + await mintTx.wait() + const balance = await paymentTokenContract.balanceOf(consumerAddress) + await ( + await paymentTokenContract + .connect(consumerAccount) + .approve(artifactsAddresses.development.Escrow, balance) + ).wait() + await ( + await escrowContract.connect(consumerAccount).deposit(paymentToken, balance) + ).wait() + const minLockSeconds = oceanNode.escrow.getMinLockTime(durationForLock) + await ( + await escrowContract + .connect(consumerAccount) + .authorize(paymentToken, beneficiaryNodeAddr, balance, minLockSeconds, 100) + ).wait() + return await oceanNode.escrow.getUserAvailableFunds( + DEVELOPMENT_CHAIN_ID, + consumerAddress, + paymentToken + ) + } + + async function getServiceJob(id: string): Promise { + const { nonce, signature } = await signFor( + consumerAccount, + PROTOCOL_COMMANDS.SERVICE_GET_STATUS + ) + const r = await new ServiceGetStatusHandler(oceanNode).handle({ + command: PROTOCOL_COMMANDS.SERVICE_GET_STATUS, + serviceId: id, + consumerAddress, + nonce, + signature + } as ServiceGetStatusCommand) + const jobs = (await streamToObject(r.stream as Readable)) as ServiceJob[] + return jobs.find((j) => j.serviceId === id) + } + + async function pollServiceStatus( + id: string, + target: ServiceStatusNumber, + timeoutMs = DEFAULT_TEST_TIMEOUT * 3 + ): Promise { + const deadline = Date.now() + timeoutMs + while (Date.now() < deadline) { + const job = await getServiceJob(id) + if (job && job.status === target) return job + if ( + job && + (job.status === ServiceStatusNumber.Error || + job.status === ServiceStatusNumber.PullImageFailed || + job.status === ServiceStatusNumber.BuildImageFailed) + ) { + throw new Error( + `service ${id} entered failure state ${job.status}: ${job.statusText}` + ) + } + await sleep(3000) + } + throw new Error(`pollServiceStatus(${id}) timed out waiting for status ${target}`) + } + + async function httpGetWithRetry( + url: string, + tries = 4 + ): Promise<{ ok: boolean; status: number; body: string }> { + let lastErr: any + for (let i = 0; i < tries; i++) { + try { + const res = await fetch(url) + const body = await res.text() + return { ok: res.ok, status: res.status, body } + } catch (e) { + lastErr = e + await sleep(1500) + } + } + throw lastErr + } + + function getDockerEngine(): C2DEngineDocker { + const engines = (oceanNode.getC2DEngines() as any).engines as C2DEngineDocker[] + return engines.find((e) => e instanceof C2DEngineDocker) as C2DEngineDocker + } + + // container.logs({follow: true}) never ends on its own, so read for a bounded + // window and always destroy the stream afterwards to release the docker log socket. + // eslint-disable-next-line require-await + async function readLogsWithTimeout( + stream: Readable, + timeoutMs = 5000 + ): Promise { + return new Promise((resolve) => { + let data = '' + const finish = () => { + stream.removeAllListeners() + stream.destroy() + resolve(data) + } + const timer = setTimeout(finish, timeoutMs) + stream.on('data', (chunk) => { + data += chunk.toString() + }) + stream.on('end', () => { + clearTimeout(timer) + finish() + }) + stream.on('error', () => { + clearTimeout(timer) + finish() + }) + }) + } + + // ── setup / teardown ───────────────────────────────────────────────── + + before(async function () { + this.timeout(DEFAULT_TEST_TIMEOUT * 3) + artifactsAddresses = getOceanArtifactsAdresses() + paymentToken = artifactsAddresses.development.Ocean + + // Write the template file BEFORE building the configuration. + serviceTemplatesPath = await fsp.mkdtemp(path.join(tmpdir(), 'ocean-svc-tmpl-')) + await fsp.writeFile( + path.join(serviceTemplatesPath, 'nginx-demo.json'), + JSON.stringify(TEMPLATE_JSON), + 'utf8' + ) + + const dockerEnvs = + '[{"socketPath":"/var/run/docker.sock",' + + '"serviceOnDemand":{"enabled":true,"nodeHost":"localhost","hostPortRange":[' + + PORT_RANGE_START + + ',' + + PORT_RANGE_END + + '],"maxDurationSeconds":' + + MAX_DURATION + + ',"allowImageBuild":true},' + + '"environments":[' + + '{"storageExpiry":604800,"maxJobDuration":3600,"minJobDuration":60,"features":{"computeJobs":true,"services":true},' + + '"resources":[{"id":"cpu","total":4,"max":4,"min":1,"type":"cpu"},{"id":"ram","total":10,"max":10,"min":1,"type":"ram"},{"id":"disk","total":10,"max":10,"min":0,"type":"disk"}],' + + '"fees":{"' + + DEVELOPMENT_CHAIN_ID + + '":[{"feeToken":"' + + paymentToken + + '","prices":[{"id":"cpu","price":1},{"id":"ram","price":1}]}]}},' + + '{"storageExpiry":604800,"maxJobDuration":3600,"minJobDuration":60,"features":{"computeJobs":true,"services":false},' + + '"resources":[{"id":"cpu","total":2,"max":2,"min":1,"type":"cpu"},{"id":"ram","total":4,"max":4,"min":1,"type":"ram"},{"id":"disk","total":4,"max":4,"min":0,"type":"disk"}],' + + '"fees":{"' + + DEVELOPMENT_CHAIN_ID + + '":[{"feeToken":"' + + paymentToken + + '","prices":[{"id":"cpu","price":1},{"id":"ram","price":1}]}]}}' + + ']}]' + + previousConfiguration = await setupEnvironment( + TEST_ENV_CONFIG_FILE, + buildEnvOverrideConfig( + [ + ENVIRONMENT_VARIABLES.RPCS, + ENVIRONMENT_VARIABLES.INDEXER_NETWORKS, + ENVIRONMENT_VARIABLES.PRIVATE_KEY, + ENVIRONMENT_VARIABLES.AUTHORIZED_DECRYPTERS, + ENVIRONMENT_VARIABLES.ADDRESS_FILE, + ENVIRONMENT_VARIABLES.DOCKER_COMPUTE_ENVIRONMENTS, + ENVIRONMENT_VARIABLES.SERVICE_TEMPLATES_PATH + ], + [ + JSON.stringify(mockSupportedNetworks), + JSON.stringify([DEVELOPMENT_CHAIN_ID]), + '0xc594c6e5def4bab63ac29eed19a134c130388f74f019bc74b8f4389df2837a58', + JSON.stringify(['0xe2DD09d719Da89e5a3D0F2549c7E24566e947260']), + `${process.env.HOME}/.ocean/ocean-contracts/artifacts/address.json`, + dockerEnvs, + serviceTemplatesPath + ] + ) + ) + + config = await getConfiguration(true) + assert( + config.serviceTemplatesPath === serviceTemplatesPath, + 'serviceTemplatesPath not applied to config' + ) + dbconn = await Database.init(config.dbConfig) + + // Clean stale service jobs so prior runs don't consume shared resources. Expired is + // the only status that releases the reservation (Stopped still counts as active). + const staleServices = await dbconn.c2d.getRunningServiceJobs() + for (const svc of staleServices) { + svc.status = ServiceStatusNumber.Expired + svc.statusText = 'Expired' + await dbconn.c2d.updateServiceJob(svc) + } + const staleJobs = await dbconn.c2d.getRunningJobs() + for (const job of staleJobs) { + await dbconn.c2d.deleteJob(job.jobId) + } + + oceanNode = OceanNode.getInstance(config, dbconn, null, null, null, null, null, true) + await oceanNode.addC2DEngines() + + provider = new JsonRpcProvider('http://127.0.0.1:8545') + publisherAccount = (await provider.getSigner(0)) as Signer + consumerAccount = (await provider.getSigner(1)) as Signer + nonOwnerAccount = (await provider.getSigner(3)) as Signer + consumerAddress = await consumerAccount.getAddress() + + paymentTokenContract = new ethers.Contract( + paymentToken, + OceanToken.abi, + publisherAccount + ) + escrowContract = new ethers.Contract( + artifactsAddresses.development.Escrow, + EscrowJson.abi, + publisherAccount + ) + }) + + after(async function () { + this.timeout(DEFAULT_TEST_TIMEOUT * 2) + // Best-effort: stop every service this suite started so no container/network/port leaks. + try { + const engine = getDockerEngine() + if (engine) { + for (const id of startedServices) { + await engine.stopService(id, consumerAddress).catch(() => {}) + } + } + } catch { + /* ignore */ + } + if (oceanNode) await oceanNode.tearDownAll() + await tearDownEnvironment(previousConfiguration) + if (serviceTemplatesPath) { + await fsp.rm(serviceTemplatesPath, { recursive: true, force: true }) + } + }) + + // ── tests ──────────────────────────────────────────────────────────── + + it('(a) sets up the service environment', () => { + assert(oceanNode, 'Failed to instantiate OceanNode') + assert(config.c2dClusters, 'Failed to get c2dClusters') + assert(config.serviceTemplatesPath === serviceTemplatesPath, 'wrong templates path') + assert(getDockerEngine(), 'No docker engine configured') + }) + + it('(b) SERVICE_GET_TEMPLATES returns the sanitized template catalogue', async () => { + const resp = await new ServiceGetTemplatesHandler(oceanNode).handle({ + command: PROTOCOL_COMMANDS.SERVICE_GET_TEMPLATES + } as ServiceGetTemplatesCommand) + assert(resp.status.httpStatus === 200, 'expected 200') + const templates = (await streamToObject( + resp.stream as Readable + )) as ServiceTemplatePublic[] + const tmpl = templates.find((t) => t.id === TEMPLATE_ID) + assert(tmpl, 'nginx-demo template not returned') + // compatibleEnvironments was removed — picking an env is the client's job. + expect((tmpl as any).compatibleEnvironments).to.equal(undefined) + + // Classify the two environments by their own features.services flag. + const envResp = await new ComputeGetEnvironmentsHandler(oceanNode).handle({ + command: PROTOCOL_COMMANDS.COMPUTE_GET_ENVIRONMENTS + }) + const envs = (await streamToObject( + envResp.stream as Readable + )) as ComputeEnvironment[] + assert(envs.length >= 2, 'expected at least 2 environments') + servicesEnv = envs.find((e) => e.features?.services === true) + noServicesEnv = envs.find((e) => e.features?.services === false) + assert(servicesEnv, 'services-enabled env not found') + assert(noServicesEnv, 'services-disabled env not found') + }) + + it('(c) funds the escrow for the consumer', async () => { + const funds = await fundEscrow(servicesEnv.consumerAddress, MAX_DURATION) + assert(BigInt(funds.toString()) > BigInt(0), 'Should have funds in escrow') + }) + + it('(d) SERVICE_START (nginx) → Running, endpoint reachable over HTTP', async function () { + this.timeout(DEFAULT_TEST_TIMEOUT * 4) + const { + consumerAddress: addr, + nonce, + signature + } = await signFor(consumerAccount, PROTOCOL_COMMANDS.SERVICE_START) + const task: ServiceStartCommand = { + command: PROTOCOL_COMMANDS.SERVICE_START, + consumerAddress: addr, + nonce, + signature, + environment: servicesEnv.id, + // Rootless nginx: runs as UID 101 and listens on 8080. The standard nginx image + // cannot start under the service hardening (CapDrop: ['ALL']) — it needs + // NET_BIND_SERVICE to bind :80 and CAP_SETUID/SETGID to drop workers to the nginx + // user. Services must use a high port and not rely on dropped capabilities. + image: 'nginxinc/nginx-unprivileged', + tag: 'alpine', + exposedPorts: [8080], + duration: SERVICE_DURATION, + resources: [ + { id: 'cpu', amount: 1 }, + { id: 'ram', amount: 1 } + ], + userData: await encryptUserData({ EXTRA: 'hello123' }), + payment: { chainId: DEVELOPMENT_CHAIN_ID, token: paymentToken } + } + const resp = await new ServiceStartHandler(oceanNode).handle(task) + assert( + resp.status.httpStatus === 200, + `expected 200, got ${resp.status.httpStatus}: ${resp.status?.error ?? ''}` + ) + const [job] = (await streamToObject(resp.stream as Readable)) as ServiceJob[] + assert(job.serviceId, 'no serviceId returned') + serviceId = job.serviceId + startedServices.push(serviceId) + + // serviceStart is async: the immediate response is a Starting record with no endpoints. + // The background loop then drives it (locking → image → claiming → running). + expect(job.status).to.equal(ServiceStatusNumber.Starting) + expect(job.endpoints).to.have.length(0) + + const running = await pollServiceStatus(serviceId, ServiceStatusNumber.Running) + assert(running.endpoints.length === 1, 'expected one endpoint') + hostPort = running.endpoints[0].hostPort + endpointUrl = running.endpoints[0].url + expiresAt = running.expiresAt + expect(hostPort).to.be.within(PORT_RANGE_START, PORT_RANGE_END) + expect(endpointUrl).to.equal(`http://localhost:${hostPort}`) + + const res = await httpGetWithRetry(endpointUrl) + assert(res.status === 200, `expected nginx HTTP 200, got ${res.status}`) + assert(res.body.toLowerCase().includes('nginx'), 'body should be the nginx page') + }) + + it('(e) SERVICE_GET_STATUS returns the job with userData stripped', async () => { + const job = await getServiceJob(serviceId) + assert(job, 'job not found') + expect(job.serviceId).to.equal(serviceId) + expect((job as any).userData).to.equal(undefined) + assert(job.payment, 'payment should be present') + + // an unauthenticated status request (no nonce/signature) is rejected + const unauth = await new ServiceGetStatusHandler(oceanNode).handle({ + command: PROTOCOL_COMMANDS.SERVICE_GET_STATUS, + consumerAddress, + serviceId + } as ServiceGetStatusCommand) + expect(unauth.status.httpStatus).to.not.equal(200) + }) + + it('(e2) SERVICE_LIST returns the node-wide resource-holding set (not owner-scoped)', async () => { + // authenticated as the NON-owner: the running service must still be listed + const { + consumerAddress: addr, + nonce, + signature + } = await signFor(nonOwnerAccount, PROTOCOL_COMMANDS.SERVICE_LIST) + const resp = await new GetServicesHandler(oceanNode).handle({ + command: PROTOCOL_COMMANDS.SERVICE_LIST, + consumerAddress: addr, + nonce, + signature + } as GetServicesCommand) + assert( + resp.status.httpStatus === 200, + `expected 200, got ${resp.status.httpStatus}: ${resp.status?.error ?? ''}` + ) + const jobs = (await streamToObject(resp.stream as Readable)) as ServiceJob[] + const listed = jobs.find((j) => j.serviceId === serviceId) + assert(listed, 'the running service must appear in the node-wide list') + expect(listed.owner.toLowerCase()).to.equal(consumerAddress.toLowerCase()) + // listing-grade sanitization: no env blob, no CMD/ENTRYPOINT overrides + expect((listed as any).userData).to.equal(undefined) + expect((listed as any).dockerCmd).to.equal(undefined) + expect((listed as any).dockerEntrypoint).to.equal(undefined) + + // status filter: Running includes the service, Expired does not + const sig2 = await signFor(nonOwnerAccount, PROTOCOL_COMMANDS.SERVICE_LIST) + const runningOnly = await new GetServicesHandler(oceanNode).handle({ + command: PROTOCOL_COMMANDS.SERVICE_LIST, + consumerAddress: sig2.consumerAddress, + nonce: sig2.nonce, + signature: sig2.signature, + status: ServiceStatusNumber.Running + } as GetServicesCommand) + const runningJobs = (await streamToObject( + runningOnly.stream as Readable + )) as ServiceJob[] + assert( + runningJobs.find((j) => j.serviceId === serviceId), + 'status=Running must include the service' + ) + const sig3 = await signFor(nonOwnerAccount, PROTOCOL_COMMANDS.SERVICE_LIST) + const expiredOnly = await new GetServicesHandler(oceanNode).handle({ + command: PROTOCOL_COMMANDS.SERVICE_LIST, + consumerAddress: sig3.consumerAddress, + nonce: sig3.nonce, + signature: sig3.signature, + status: ServiceStatusNumber.Expired + } as GetServicesCommand) + const expiredJobs = (await streamToObject( + expiredOnly.stream as Readable + )) as ServiceJob[] + expect(expiredJobs.find((j) => j.serviceId === serviceId)).to.equal(undefined) + + // fromTimestamp in the future excludes everything + const sig4 = await signFor(nonOwnerAccount, PROTOCOL_COMMANDS.SERVICE_LIST) + const future = await new GetServicesHandler(oceanNode).handle({ + command: PROTOCOL_COMMANDS.SERVICE_LIST, + consumerAddress: sig4.consumerAddress, + nonce: sig4.nonce, + signature: sig4.signature, + includeAllStatuses: true, + fromTimestamp: String(Date.now() + 3600_000) + } as GetServicesCommand) + const futureJobs = (await streamToObject(future.stream as Readable)) as ServiceJob[] + expect(futureJobs.find((j) => j.serviceId === serviceId)).to.equal(undefined) + + // unauthenticated requests are rejected + const unauth = await new GetServicesHandler(oceanNode).handle({ + command: PROTOCOL_COMMANDS.SERVICE_LIST, + consumerAddress: addr + } as GetServicesCommand) + expect(unauth.status.httpStatus).to.not.equal(200) + }) + + it('(f) SERVICE_GET_STREAMABLE_LOGS streams the real container logs (owner-only)', async () => { + const { + consumerAddress: addr, + nonce, + signature + } = await signFor(consumerAccount, PROTOCOL_COMMANDS.SERVICE_GET_STREAMABLE_LOGS) + const task: ServiceGetStreamableLogsCommand = { + command: PROTOCOL_COMMANDS.SERVICE_GET_STREAMABLE_LOGS, + consumerAddress: addr, + nonce, + signature, + serviceId + } + const resp = await new ServiceGetStreamableLogsHandler(oceanNode).handle(task) + assert( + resp.status.httpStatus === 200, + `expected 200, got ${resp.status.httpStatus}: ${resp.status?.error ?? ''}` + ) + // follow:true keeps the stream open indefinitely, so read for a bounded window + // and destroy it afterwards — nginx logs its startup banner to stdout on boot. + const logs = await readLogsWithTimeout(resp.stream as Readable) + assert(typeof logs === 'string', 'expected log output to be a string') + + // `since` accepts a relative duration (converted to a Unix timestamp before + // reaching dockerode) — e.g. "1h" to skip straight to the last hour's output. + const sinceSigned = await signFor( + consumerAccount, + PROTOCOL_COMMANDS.SERVICE_GET_STREAMABLE_LOGS + ) + const sinceResp = await new ServiceGetStreamableLogsHandler(oceanNode).handle({ + command: PROTOCOL_COMMANDS.SERVICE_GET_STREAMABLE_LOGS, + consumerAddress: sinceSigned.consumerAddress, + nonce: sinceSigned.nonce, + signature: sinceSigned.signature, + serviceId, + since: '1h' + } as ServiceGetStreamableLogsCommand) + assert( + sinceResp.status.httpStatus === 200, + `expected 200 with since=1h, got ${sinceResp.status.httpStatus}: ${sinceResp.status?.error ?? ''}` + ) + await readLogsWithTimeout(sinceResp.stream as Readable, 1000) + + // an invalid `since` format is rejected before any docker call is made + const badSinceSigned = await signFor( + consumerAccount, + PROTOCOL_COMMANDS.SERVICE_GET_STREAMABLE_LOGS + ) + const badSinceResp = await new ServiceGetStreamableLogsHandler(oceanNode).handle({ + command: PROTOCOL_COMMANDS.SERVICE_GET_STREAMABLE_LOGS, + consumerAddress: badSinceSigned.consumerAddress, + nonce: badSinceSigned.nonce, + signature: badSinceSigned.signature, + serviceId, + since: 'not-a-valid-since' + } as ServiceGetStreamableLogsCommand) + expect(badSinceResp.status.httpStatus).to.equal(400) + + // an unauthenticated request (no nonce/signature) is rejected + const unauth = await new ServiceGetStreamableLogsHandler(oceanNode).handle({ + command: PROTOCOL_COMMANDS.SERVICE_GET_STREAMABLE_LOGS, + consumerAddress, + serviceId + } as ServiceGetStreamableLogsCommand) + expect(unauth.status.httpStatus).to.not.equal(200) + + // a non-owner request is rejected too + const { + consumerAddress: otherAddr, + nonce: otherNonce, + signature: otherSignature + } = await signFor(nonOwnerAccount, PROTOCOL_COMMANDS.SERVICE_GET_STREAMABLE_LOGS) + const nonOwnerResp = await new ServiceGetStreamableLogsHandler(oceanNode).handle({ + command: PROTOCOL_COMMANDS.SERVICE_GET_STREAMABLE_LOGS, + consumerAddress: otherAddr, + nonce: otherNonce, + signature: otherSignature, + serviceId + } as ServiceGetStreamableLogsCommand) + expect(nonOwnerResp.status.httpStatus).to.not.equal(200) + }) + + it('(g) SERVICE_START on a services-disabled environment → 403', async () => { + const { + consumerAddress: addr, + nonce, + signature + } = await signFor(consumerAccount, PROTOCOL_COMMANDS.SERVICE_START) + const task: ServiceStartCommand = { + command: PROTOCOL_COMMANDS.SERVICE_START, + consumerAddress: addr, + nonce, + signature, + environment: noServicesEnv.id, + // Rootless nginx: runs as UID 101 and listens on 8080. The standard nginx image + // cannot start under the service hardening (CapDrop: ['ALL']) — it needs + // NET_BIND_SERVICE to bind :80 and CAP_SETUID/SETGID to drop workers to the nginx + // user. Services must use a high port and not rely on dropped capabilities. + image: 'nginxinc/nginx-unprivileged', + tag: 'alpine', + exposedPorts: [8080], + duration: SERVICE_DURATION, + payment: { chainId: DEVELOPMENT_CHAIN_ID, token: paymentToken } + } + const resp = await new ServiceStartHandler(oceanNode).handle(task) + expect(resp.status.httpStatus).to.equal(403) + }) + + it('(h) SERVICE_START with duration > maxDurationSeconds → 400', async () => { + const { + consumerAddress: addr, + nonce, + signature + } = await signFor(consumerAccount, PROTOCOL_COMMANDS.SERVICE_START) + const task: ServiceStartCommand = { + command: PROTOCOL_COMMANDS.SERVICE_START, + consumerAddress: addr, + nonce, + signature, + environment: servicesEnv.id, + // Rootless nginx: runs as UID 101 and listens on 8080. The standard nginx image + // cannot start under the service hardening (CapDrop: ['ALL']) — it needs + // NET_BIND_SERVICE to bind :80 and CAP_SETUID/SETGID to drop workers to the nginx + // user. Services must use a high port and not rely on dropped capabilities. + image: 'nginxinc/nginx-unprivileged', + tag: 'alpine', + exposedPorts: [8080], + duration: MAX_DURATION + 1, + payment: { chainId: DEVELOPMENT_CHAIN_ID, token: paymentToken } + } + const resp = await new ServiceStartHandler(oceanNode).handle(task) + expect(resp.status.httpStatus).to.equal(400) + }) + + it('(i) SERVICE_START with undecryptable userData → 400', async () => { + const { + consumerAddress: addr, + nonce, + signature + } = await signFor(consumerAccount, PROTOCOL_COMMANDS.SERVICE_START) + const task: ServiceStartCommand = { + command: PROTOCOL_COMMANDS.SERVICE_START, + consumerAddress: addr, + nonce, + signature, + environment: servicesEnv.id, + // Rootless nginx: runs as UID 101 and listens on 8080. The standard nginx image + // cannot start under the service hardening (CapDrop: ['ALL']) — it needs + // NET_BIND_SERVICE to bind :80 and CAP_SETUID/SETGID to drop workers to the nginx + // user. Services must use a high port and not rely on dropped capabilities. + image: 'nginxinc/nginx-unprivileged', + tag: 'alpine', + exposedPorts: [8080], + duration: SERVICE_DURATION, + // not ECIES-encrypted to the node key → decrypt must fail + userData: Buffer.from('not-encrypted-userData').toString('hex'), + payment: { chainId: DEVELOPMENT_CHAIN_ID, token: paymentToken } + } + const resp = await new ServiceStartHandler(oceanNode).handle(task) + expect(resp.status.httpStatus).to.equal(400) + }) + + it('(j) SERVICE_EXTEND advances expiresAt and records an extendPayment', async () => { + const { + consumerAddress: addr, + nonce, + signature + } = await signFor(consumerAccount, PROTOCOL_COMMANDS.SERVICE_EXTEND) + const task: ServiceExtendCommand = { + command: PROTOCOL_COMMANDS.SERVICE_EXTEND, + consumerAddress: addr, + nonce, + signature, + serviceId, + additionalDuration: 30, + payment: { chainId: DEVELOPMENT_CHAIN_ID, token: paymentToken } + } + const resp = await new ServiceExtendHandler(oceanNode).handle(task) + assert( + resp.status.httpStatus === 200, + `expected 200, got ${resp.status.httpStatus}: ${resp.status?.error ?? ''}` + ) + const [job] = (await streamToObject(resp.stream as Readable)) as ServiceJob[] + expect(job.expiresAt).to.equal(expiresAt + 30 * 1000) + expect(job.extendPayments?.length).to.equal(1) + expiresAt = job.expiresAt + }) + + it('(k) SERVICE_EXTEND by a non-owner is rejected (non-200)', async () => { + // In a real DB, getServiceJob filters by owner, so a non-owner lookup returns + // "not found" (400) rather than reaching the 401 ownership branch. + const { + consumerAddress: addr, + nonce, + signature + } = await signFor(nonOwnerAccount, PROTOCOL_COMMANDS.SERVICE_EXTEND) + const task: ServiceExtendCommand = { + command: PROTOCOL_COMMANDS.SERVICE_EXTEND, + consumerAddress: addr, + nonce, + signature, + serviceId, + additionalDuration: 30, + payment: { chainId: DEVELOPMENT_CHAIN_ID, token: paymentToken } + } + const resp = await new ServiceExtendHandler(oceanNode).handle(task) + expect(resp.status.httpStatus).to.not.equal(200) + }) + + it('(l) SERVICE_RESTART → new container, same hostPort + expiresAt', async function () { + this.timeout(DEFAULT_TEST_TIMEOUT * 4) + const before = await getServiceJob(serviceId) + const oldContainerId = before.containerId + + const { + consumerAddress: addr, + nonce, + signature + } = await signFor(consumerAccount, PROTOCOL_COMMANDS.SERVICE_RESTART) + const task: ServiceRestartCommand = { + command: PROTOCOL_COMMANDS.SERVICE_RESTART, + consumerAddress: addr, + nonce, + signature, + serviceId + } + const resp = await new ServiceRestartHandler(oceanNode).handle(task) + assert( + resp.status.httpStatus === 200, + `expected 200, got ${resp.status.httpStatus}: ${resp.status?.error ?? ''}` + ) + const running = await pollServiceStatus(serviceId, ServiceStatusNumber.Running) + expect(running.containerId).to.not.equal(oldContainerId) + expect(running.endpoints[0].hostPort).to.equal(hostPort) + expect(running.expiresAt).to.equal(expiresAt) + + const res = await httpGetWithRetry(endpointUrl) + assert(res.status === 200, `expected nginx HTTP 200 after restart, got ${res.status}`) + }) + + it('(l2) SERVICE_RESTART self-heals a network leaked by a crash mid-start', async function () { + this.timeout(DEFAULT_TEST_TIMEOUT * 4) + // Simulate a node that died between createNetwork and persisting the docker refs: + // the DB record loses containerId/networkId while the real container + network keep + // running. Pre-fix, restart then failed forever with Docker 409 ("network with name + // ocean-svc- already exists") because teardown only used the empty networkId. + const [raw] = await dbconn.c2d.getServiceJob(serviceId) + const oldContainerId = raw.containerId + raw.containerId = '' + raw.networkId = '' + raw.status = ServiceStatusNumber.Error + raw.statusText = 'simulated crash mid-start' + await dbconn.c2d.updateServiceJob(raw) + + const { + consumerAddress: addr, + nonce, + signature + } = await signFor(consumerAccount, PROTOCOL_COMMANDS.SERVICE_RESTART) + const task: ServiceRestartCommand = { + command: PROTOCOL_COMMANDS.SERVICE_RESTART, + consumerAddress: addr, + nonce, + signature, + serviceId + } + const resp = await new ServiceRestartHandler(oceanNode).handle(task) + assert( + resp.status.httpStatus === 200, + `expected 200, got ${resp.status.httpStatus}: ${resp.status?.error ?? ''}` + ) + const running = await pollServiceStatus(serviceId, ServiceStatusNumber.Running) + expect(running.containerId).to.not.equal('') + expect(running.containerId).to.not.equal(oldContainerId) + expect(running.endpoints[0].hostPort).to.equal(hostPort) + + const docker = new Dockerode() + // the stale container attached to the leaked network must have been force-removed + let staleGone = false + try { + await docker.getContainer(oldContainerId).inspect() + } catch { + staleGone = true + } + assert(staleGone, 'stale container should have been force-removed') + // exactly one network with the deterministic name survives + const nets = await docker.listNetworks() + const matching = nets.filter((n: any) => n.Name === `ocean-svc-${serviceId}`) + expect(matching.length).to.equal(1) + + const res = await httpGetWithRetry(endpointUrl) + assert( + res.status === 200, + `expected nginx HTTP 200 after self-heal, got ${res.status}` + ) + }) + + it('(l3) SERVICE_RESTART survives InternalLoop ticks mid-restart (orphan-recovery race)', async function () { + this.timeout(DEFAULT_TEST_TIMEOUT * 4) + const engine: any = getDockerEngine() + const before = await getServiceJob(serviceId) + const oldContainerId = before.containerId + + // Delay the image pull so the job sits in PullImage across several InternalLoop + // ticks (cronTime = 2 s) — the exact window in which the loop's orphan-recovery + // used to tear down the network the restart had just created, failing the restart + // with "network ocean-svc- not found" (the live-node bug). + const originalPull = engine.pullImageRef + engine.pullImageRef = async (...args: any[]) => { + await sleep(5000) + return originalPull.apply(engine, args) + } + try { + const { + consumerAddress: addr, + nonce, + signature + } = await signFor(consumerAccount, PROTOCOL_COMMANDS.SERVICE_RESTART) + const task: ServiceRestartCommand = { + command: PROTOCOL_COMMANDS.SERVICE_RESTART, + consumerAddress: addr, + nonce, + signature, + serviceId + } + const restartPromise = new ServiceRestartHandler(oceanNode).handle(task) + + // wait until the restart is actually mid-pull (deterministic, not sleep-based) — + // and ASSERT it got there: silently timing out would run the concurrent-command + // checks against a restart in the wrong phase, proving nothing. + const deadline = Date.now() + 10_000 + let sawPullImage = false + while (Date.now() < deadline) { + const j = await getServiceJob(serviceId) + if (j?.status === ServiceStatusNumber.PullImage) { + sawPullImage = true + break + } + await sleep(250) + } + assert(sawPullImage, 'restart must reach PullImage within the wait window') + + // concurrent lifecycle operations must be rejected while the restart holds the lock + const stopSig = await signFor(consumerAccount, PROTOCOL_COMMANDS.SERVICE_STOP) + const stopResp = await new ServiceStopHandler(oceanNode).handle({ + command: PROTOCOL_COMMANDS.SERVICE_STOP, + consumerAddress: stopSig.consumerAddress, + nonce: stopSig.nonce, + signature: stopSig.signature, + serviceId + } as ServiceStopCommand) + expect(stopResp.status.httpStatus).to.not.equal(200) + expect(String(stopResp.status.error)).to.contain('operation in progress') + + const retrySig = await signFor(consumerAccount, PROTOCOL_COMMANDS.SERVICE_RESTART) + const retryResp = await new ServiceRestartHandler(oceanNode).handle({ + ...task, + nonce: retrySig.nonce, + signature: retrySig.signature + }) + expect(retryResp.status.httpStatus).to.not.equal(200) + expect(String(retryResp.status.error)).to.contain('operation in progress') + + // the original restart must complete unharmed by the loop ticks that fired mid-pull + const resp = await restartPromise + assert( + resp.status.httpStatus === 200, + `expected 200, got ${resp.status.httpStatus}: ${resp.status?.error ?? ''}` + ) + } finally { + engine.pullImageRef = originalPull + } + + // pollServiceStatus throws if the job lands in Error — which is exactly what the + // pre-fix orphan-recovery race produced ("Service start aborted (node restarted + // mid-start)" / "network ocean-svc- not found"). + const running = await pollServiceStatus(serviceId, ServiceStatusNumber.Running) + expect(running.containerId).to.not.equal(oldContainerId) + expect(running.endpoints[0].hostPort).to.equal(hostPort) + expect(running.expiresAt).to.equal(expiresAt) + + const res = await httpGetWithRetry(endpointUrl) + assert( + res.status === 200, + `expected nginx HTTP 200 after raced restart, got ${res.status}` + ) + }) + + it('(m) SERVICE_STOP → Stopped, container + network removed', async function () { + this.timeout(DEFAULT_TEST_TIMEOUT * 2) + const before = await getServiceJob(serviceId) + const { containerId } = before + + const { + consumerAddress: addr, + nonce, + signature + } = await signFor(consumerAccount, PROTOCOL_COMMANDS.SERVICE_STOP) + const task: ServiceStopCommand = { + command: PROTOCOL_COMMANDS.SERVICE_STOP, + consumerAddress: addr, + nonce, + signature, + serviceId + } + const resp = await new ServiceStopHandler(oceanNode).handle(task) + assert( + resp.status.httpStatus === 200, + `expected 200, got ${resp.status.httpStatus}: ${resp.status?.error ?? ''}` + ) + const [job] = (await streamToObject(resp.stream as Readable)) as ServiceJob[] + expect(job.status).to.equal(ServiceStatusNumber.Stopped) + + // container should be gone + const docker = new Dockerode() + let inspectFailed = false + try { + await docker.getContainer(containerId).inspect() + } catch { + inspectFailed = true + } + assert(inspectFailed, 'container should have been removed') + + // network should be gone too (removed by deterministic name, independent of networkId) + const nets = await docker.listNetworks() + const matching = nets.filter((n: any) => n.Name === `ocean-svc-${serviceId}`) + expect(matching.length).to.equal(0) + }) + + it('(n) [slow] expiry cron marks a short-lived service Expired', async function () { + this.timeout(150000) + const { + consumerAddress: addr, + nonce, + signature + } = await signFor(consumerAccount, PROTOCOL_COMMANDS.SERVICE_START) + const task: ServiceStartCommand = { + command: PROTOCOL_COMMANDS.SERVICE_START, + consumerAddress: addr, + nonce, + signature, + environment: servicesEnv.id, + // Rootless nginx: runs as UID 101 and listens on 8080. The standard nginx image + // cannot start under the service hardening (CapDrop: ['ALL']) — it needs + // NET_BIND_SERVICE to bind :80 and CAP_SETUID/SETGID to drop workers to the nginx + // user. Services must use a high port and not rely on dropped capabilities. + image: 'nginxinc/nginx-unprivileged', + tag: 'alpine', + exposedPorts: [8080], + duration: EXPIRY_DURATION, + resources: [ + { id: 'cpu', amount: 1 }, + { id: 'ram', amount: 1 } + ], + payment: { chainId: DEVELOPMENT_CHAIN_ID, token: paymentToken } + } + const resp = await new ServiceStartHandler(oceanNode).handle(task) + assert(resp.status.httpStatus === 200, `start failed: ${resp.status?.error ?? ''}`) + const [job] = (await streamToObject(resp.stream as Readable)) as ServiceJob[] + startedServices.push(job.serviceId) + await pollServiceStatus(job.serviceId, ServiceStatusNumber.Running) + // wait out the duration; the InternalLoop cron (~2s) stops+expires it + const expired = await pollServiceStatus( + job.serviceId, + ServiceStatusNumber.Expired, + (EXPIRY_DURATION + 40) * 1000 + ) + expect(expired.status).to.equal(ServiceStatusNumber.Expired) + }) + + it('(o) [build] Dockerfile-based custom service builds and serves', async function () { + this.timeout(DEFAULT_TEST_TIMEOUT * 8) + const { + consumerAddress: addr, + nonce, + signature + } = await signFor(consumerAccount, PROTOCOL_COMMANDS.SERVICE_START) + // Rootless nginx base (listens on 8080 as UID 101) so it runs under the service + // hardening (CapDrop: ['ALL']). USER root only to write the file into the root-owned + // docroot, then back to 101 so the runtime nginx matches the image's unprivileged user. + const dockerfile = + 'FROM nginxinc/nginx-unprivileged:alpine\n' + + 'USER root\n' + + 'RUN echo built > /usr/share/nginx/html/built.txt\n' + + 'USER 101\n' + const task: ServiceStartCommand = { + command: PROTOCOL_COMMANDS.SERVICE_START, + consumerAddress: addr, + nonce, + signature, + environment: servicesEnv.id, + image: 'custom-svc', + dockerfile, + dockerCmd: ['nginx', '-g', 'daemon off;'], + exposedPorts: [8080], + duration: SERVICE_DURATION, + resources: [ + { id: 'cpu', amount: 1 }, + { id: 'ram', amount: 1 } + ], + payment: { chainId: DEVELOPMENT_CHAIN_ID, token: paymentToken } + } + const resp = await new ServiceStartHandler(oceanNode).handle(task) + assert( + resp.status.httpStatus === 200, + `expected 200, got ${resp.status.httpStatus}: ${resp.status?.error ?? ''}` + ) + const [job] = (await streamToObject(resp.stream as Readable)) as ServiceJob[] + startedServices.push(job.serviceId) + const running = await pollServiceStatus( + job.serviceId, + ServiceStatusNumber.Running, + DEFAULT_TEST_TIMEOUT * 8 + ) + const { url } = running.endpoints[0] + const res = await httpGetWithRetry(`${url}/built.txt`) + assert(res.status === 200, `expected built.txt HTTP 200, got ${res.status}`) + assert(res.body.includes('built'), 'built.txt should contain "built"') + + // stop it + await getDockerEngine().stopService(job.serviceId, consumerAddress) + }) +}) diff --git a/src/test/unit/auth/token.test.ts b/src/test/unit/auth/token.test.ts index a2a2c674f..f6c44d172 100644 --- a/src/test/unit/auth/token.test.ts +++ b/src/test/unit/auth/token.test.ts @@ -1,9 +1,12 @@ import { OceanNodeConfig } from '../../../@types/OceanNode.js' import { getConfiguration } from '../../../utils/index.js' import { expect } from 'chai' -import { Wallet } from 'ethers' +import { HDNodeWallet, Wallet } from 'ethers' import { Auth } from '../../../components/Auth/index.js' import { AuthTokenDatabase } from '../../../components/database/AuthTokenDatabase.js' +import { OceanP2P } from '../../../components/P2P/index.js' +import { Readable } from 'node:stream' +import { createHashForSignature, safeSign } from '../../utils/signature.js' describe('Auth Token Tests', () => { let wallet: Wallet @@ -66,3 +69,170 @@ describe('Auth Token Tests', () => { expect(validationResult).to.be.equal(null) }) }) + +describe('Auth cross-node delegated validation', () => { + let config: OceanNodeConfig + let authTokenDatabase: AuthTokenDatabase + let consumer: HDNodeWallet + + const ISSUER_PEER_ID = '12D3KooWJxrSgqyrknCAdLwZ8ZjAGmS7vC38QaCT8sLRfLxtxxmM' + const SELF_PEER_ID = '12D3KooWLPYsaVTn8W1n1hKmhZt5zS3AMCN7hdaAZkKm94wKrDsa' + + function makeFakeP2P(opts: { + verdict?: unknown + httpStatus?: number + isSelf?: boolean + }) { + const tracker = { calls: 0 } + const p2p = { + getPeerId: () => SELF_PEER_ID, + isTargetPeerSelf: (id: string) => (opts.isSelf ? true : id === SELF_PEER_ID), + // eslint-disable-next-line require-await + sendTo: async (_peer: string, _msg: string) => { + tracker.calls++ + const httpStatus = opts.httpStatus ?? 200 + if (httpStatus !== 200) { + return { status: { httpStatus } } + } + return { + status: { httpStatus: 200 }, + stream: Readable.from(JSON.stringify(opts.verdict)) + } + } + } as unknown as OceanP2P + return { p2p, tracker } + } + + async function mintToken( + auth: Auth, + signWith: HDNodeWallet, + claimAddress: string, + opts: { nonce?: string; issuerPeerId?: string } = {} + ): Promise { + const nonce = opts.nonce ?? '12345' + const signature = await safeSign( + signWith, + createHashForSignature( + claimAddress, + nonce, + 'createAuthToken', + opts.issuerPeerId ?? ISSUER_PEER_ID + ) + ) + return auth.getJWTToken( + claimAddress, + nonce, + Date.now(), + signature, + opts.issuerPeerId ?? ISSUER_PEER_ID + ) + } + + before(async () => { + const baseConfig = await getConfiguration(true) + config = { ...baseConfig, supportedNetworks: {} } + authTokenDatabase = await AuthTokenDatabase.create(baseConfig.dbConfig) + consumer = Wallet.createRandom() + }) + + it('accepts when the signature matches AND the issuer says valid', async () => { + const { p2p, tracker } = makeFakeP2P({ + verdict: { valid: true, validUntil: Date.now() + 60_000 } + }) + const auth = new Auth(authTokenDatabase, config, () => p2p) + const token = await mintToken(auth, consumer, consumer.address) + + const result = await auth.validateToken(token) + expect(result).to.not.equal(null) + expect(result.address).to.equal(consumer.address) + expect(result.isValid).to.equal(true) + expect(tracker.calls).to.equal(1) + }) + + it('rejects a signature from a different address BEFORE asking the issuer', async () => { + const attacker = Wallet.createRandom() + const { p2p, tracker } = makeFakeP2P({ verdict: { valid: true } }) + const auth = new Auth(authTokenDatabase, config, () => p2p) + const token = await mintToken(auth, attacker, consumer.address) + + const result = await auth.validateToken(token) + expect(result).to.equal(null) + expect(tracker.calls).to.equal(0) + }) + + it('rejects when the issuer says invalid (expired/revoked)', async () => { + const { p2p, tracker } = makeFakeP2P({ verdict: { valid: false } }) + const auth = new Auth(authTokenDatabase, config, () => p2p) + const token = await mintToken(auth, consumer, consumer.address) + + const result = await auth.validateToken(token) + expect(result).to.equal(null) + expect(tracker.calls).to.equal(1) + }) + + it('rejects when the issuer is unreachable', async () => { + const { p2p, tracker } = makeFakeP2P({ httpStatus: 404 }) + const auth = new Auth(authTokenDatabase, config, () => p2p) + const token = await mintToken(auth, consumer, consumer.address) + + const result = await auth.validateToken(token) + expect(result).to.equal(null) + expect(tracker.calls).to.equal(1) + }) + + it('rejects a token with no embedded signature without asking the issuer', async () => { + const { p2p, tracker } = makeFakeP2P({ verdict: { valid: true } }) + const auth = new Auth(authTokenDatabase, config, () => p2p) + const token = await auth.getJWTToken( + consumer.address, + '2', + Date.now(), + undefined, + ISSUER_PEER_ID + ) + + const result = await auth.validateToken(token) + expect(result).to.equal(null) + expect(tracker.calls).to.equal(0) + }) + + it('rejects a token with no issuerPeerId without asking the issuer', async () => { + const { p2p, tracker } = makeFakeP2P({ verdict: { valid: true } }) + const auth = new Auth(authTokenDatabase, config, () => p2p) + const nonce = '3' + const signature = await safeSign( + consumer, + createHashForSignature(consumer.address, nonce, 'createAuthToken', ISSUER_PEER_ID) + ) + const token = await auth.getJWTToken(consumer.address, nonce, Date.now(), signature) + + const result = await auth.validateToken(token) + expect(result).to.equal(null) + expect(tracker.calls).to.equal(0) + }) + + it('rejects when the issuer is self (already missed locally)', async () => { + const { p2p, tracker } = makeFakeP2P({ verdict: { valid: true }, isSelf: true }) + const auth = new Auth(authTokenDatabase, config, () => p2p) + const token = await mintToken(auth, consumer, consumer.address, { + issuerPeerId: SELF_PEER_ID + }) + + const result = await auth.validateToken(token) + expect(result).to.equal(null) + expect(tracker.calls).to.equal(0) + }) + + it('getLocalToken does NOT delegate', async () => { + const { p2p, tracker } = makeFakeP2P({ + verdict: { valid: true, validUntil: Date.now() + 60_000 } + }) + const auth = new Auth(authTokenDatabase, config, () => p2p) + const token = await mintToken(auth, consumer, consumer.address) + + expect(await auth.validateToken(token)).to.not.equal(null) + const callsBefore = tracker.calls + expect(await auth.getLocalToken(token)).to.equal(null) + expect(tracker.calls).to.equal(callsBefore) + }) +}) diff --git a/src/test/unit/c2d/serviceResourceMatching.test.ts b/src/test/unit/c2d/serviceResourceMatching.test.ts new file mode 100644 index 000000000..955fea501 --- /dev/null +++ b/src/test/unit/c2d/serviceResourceMatching.test.ts @@ -0,0 +1,22 @@ +import { expect } from 'chai' +import { resolveServiceImage } from '../../../components/c2d/serviceResourceMatching.js' + +describe('resolveServiceImage', () => { + it('image + tag → image:tag', () => { + expect(resolveServiceImage('vllm/vllm-openai', 'latest')).to.equal( + 'vllm/vllm-openai:latest' + ) + }) + it('image + checksum → image@sha256', () => { + const c = 'sha256:' + 'a'.repeat(64) + expect(resolveServiceImage('img', undefined, c)).to.equal(`img@${c}`) + }) + it('image only → image:latest', () => { + expect(resolveServiceImage('img')).to.equal('img:latest') + }) + it('dockerfile → {serviceId}-svc-image:latest', () => { + expect( + resolveServiceImage('img', undefined, undefined, 'FROM x', 'SvcID123') + ).to.equal('svcid123-svc-image:latest') + }) +}) diff --git a/src/test/unit/compute.test.ts b/src/test/unit/compute.test.ts index 07d1bf605..021e2f135 100644 --- a/src/test/unit/compute.test.ts +++ b/src/test/unit/compute.test.ts @@ -9,6 +9,7 @@ import { ComputeAsset, ComputeEnvironment, ComputeJob, + ComputeResource, ComputeResourceRequest, DBComputeJob, RunningPlatform @@ -35,7 +36,13 @@ import { C2DEngine, omitDBComputeFieldsFromComputeJob } from '../../components/c2d/index.js' -import { checkManifestPlatform } from '../../components/c2d/compute_engine_docker.js' +import { + checkManifestPlatform, + C2DEngineDocker +} from '../../components/c2d/compute_engine_docker.js' +import { ServiceStatusNumber } from '../../@types/C2D/ServiceOnDemand.js' +import { allocateHostPort, releaseHostPort } from '../../components/core/service/utils.js' +import { C2DDockerConfigSchema } from '../../utils/config/schemas.js' import { ValidateParams } from '../../components/httpRoutes/validateCommands.js' import { Readable } from 'stream' import sinon from 'sinon' @@ -49,6 +56,10 @@ class TestC2DEngine extends C2DEngine { super(null, null, null, null, null) } + setPhysicalLimits(limits: Map) { + this.physicalLimits = limits + } + async getComputeEnvironments(): Promise { return [] } @@ -342,9 +353,9 @@ describe('Compute Jobs Database', () => { }) const baseResources = [ - { id: 'cpu', total: 8, min: 1, max: 8, inUse: 0 }, - { id: 'ram', total: 32, min: 1, max: 32, inUse: 0 }, - { id: 'disk', total: 500, min: 10, max: 500, inUse: 0 } + { id: 'cpu', kind: 'fungible', total: 8, min: 1, max: 8, inUse: 0 }, + { id: 'ram', kind: 'fungible', total: 32, min: 1, max: 32, inUse: 0 }, + { id: 'disk', kind: 'fungible', total: 500, min: 10, max: 500, inUse: 0 } ] it('satisfies constraints exactly → passes without modification', async function () { @@ -445,6 +456,644 @@ describe('Compute Jobs Database', () => { expect(cpuEntry.amount).to.equal(1) // bumped to min expect(diskEntry.amount).to.equal(10) // bumped to min }) + + it('per-env constraint override: premium env uses 8 GB RAM, standard env uses 4 GB RAM', async function () { + // Pool default: gpu0 requires ram min:4. + // premium env overrides to ram min:8. + // standard env inherits pool default (ram min:4). + const gpuWithOverrideConstraint = { + id: 'gpu0', + kind: 'discrete', + total: 1, + min: 0, + max: 1, + inUse: 0, + constraints: [{ id: 'ram', min: 8 }] // premium override + } + const gpuWithPoolConstraint = { + id: 'gpu0', + kind: 'discrete', + total: 1, + min: 0, + max: 1, + inUse: 0, + constraints: [{ id: 'ram', min: 4 }] // pool default inherited by standard + } + const premiumEnv = makeEnv([ + { id: 'ram', kind: 'fungible', total: 32, min: 1, max: 32, inUse: 0 }, + gpuWithOverrideConstraint + ]) + const standardEnv = makeEnv([ + { id: 'ram', kind: 'fungible', total: 32, min: 1, max: 32, inUse: 0 }, + gpuWithPoolConstraint + ]) + + const premiumReq: ComputeResourceRequest[] = [ + { id: 'ram', amount: 1 }, + { id: 'gpu0', amount: 1 } + ] + const standardReq: ComputeResourceRequest[] = [ + { id: 'ram', amount: 1 }, + { id: 'gpu0', amount: 1 } + ] + + const premiumResult = await engine.checkAndFillMissingResources( + premiumReq, + premiumEnv, + false + ) + const standardResult = await engine.checkAndFillMissingResources( + standardReq, + standardEnv, + false + ) + + expect(premiumResult.find((r) => r.id === 'ram').amount).to.equal(8) + expect(standardResult.find((r) => r.id === 'ram').amount).to.equal(4) + }) + + it('ref.constraints: [] removes all constraints for env — GPU job admitted with only resource min', async function () { + const gpuNoConstraints = { + id: 'gpu0', + kind: 'discrete', + total: 1, + min: 0, + max: 1, + inUse: 0, + constraints: [] as any[] // no constraints for this env + } + const env = makeEnv([ + { id: 'ram', kind: 'fungible', total: 32, min: 1, max: 32, inUse: 0 }, + gpuNoConstraints + ]) + const req: ComputeResourceRequest[] = [ + { id: 'ram', amount: 1 }, + { id: 'gpu0', amount: 1 } + ] + // No constraints → ram stays at 1 (not bumped to any min) + const result = await engine.checkAndFillMissingResources(req, env, false) + expect(result.find((r) => r.id === 'ram').amount).to.equal(1) + }) + + it('constraint-driven exhaustion: GPU becomes unrentable when RAM nearly depleted', async function () { + // gpu0 requires min:4 GB RAM. Env has 10 GB RAM total, 9 GB in use → only 1 GB remaining. + // Requesting gpu0 triggers checkAndFillMissingResources to bump RAM to 4 GB. + // checkIfResourcesAreAvailable should then reject at Gate 1 (only 1 GB remaining). + const resources = [ + { id: 'ram', kind: 'fungible', total: 10, min: 1, max: 10, inUse: 9 }, + { + id: 'gpu0', + kind: 'discrete', + total: 1, + min: 0, + max: 1, + inUse: 0, + constraints: [{ id: 'ram', min: 4 }] + } + ] + const env = makeEnv(resources) + const req: ComputeResourceRequest[] = [ + { id: 'ram', amount: 1 }, + { id: 'gpu0', amount: 1 } + ] + + // Step 1: auto-bump RAM from 1 to 4 (constraint min per gpu unit) + const filled = await engine.checkAndFillMissingResources(req, env, false) + expect(filled.find((r) => r.id === 'ram').amount).to.equal(4) + + // Step 2: 10 - 9 = 1 available < 4 requested → Gate 1 blocks + try { + await engine.checkIfResourcesAreAvailable(filled, env, false) + assert.fail('Expected error was not thrown') + } catch (err: any) { + expect(err.message).to.include('ram') + } + }) + + // ── Group (type) + FLOOR (perUnit:false) constraints ────────────────── + // "if any cpu is selected, the job needs at least 1 GPU — no matter which id". + // The constraint lives on the env's cpu ref and targets the `type:'gpu'` group. + const cpuFloorConstraint: any[] = [{ type: 'gpu', min: 1, perUnit: false }] + const twoGpuResources = (inUse1 = 0, inUse2 = 0): any[] => [ + { + id: 'cpu', + type: 'cpu', + kind: 'fungible', + total: 8, + min: 1, + max: 8, + inUse: 0, + constraints: cpuFloorConstraint + }, + { id: 'ram', type: 'ram', kind: 'fungible', total: 32, min: 1, max: 32, inUse: 0 }, + { + id: 'disk', + type: 'disk', + kind: 'fungible', + total: 500, + min: 0, + max: 500, + inUse: 0 + }, + { + id: 'GPU1', + type: 'gpu', + kind: 'discrete', + total: 1, + min: 0, + max: 1, + inUse: inUse1 + }, + { + id: 'GPU2', + type: 'gpu', + kind: 'discrete', + total: 1, + min: 0, + max: 1, + inUse: inUse2 + } + ] + + it('group floor (type:gpu): satisfied by any one GPU → no change', async function () { + const env = makeEnv(twoGpuResources()) + const req: ComputeResourceRequest[] = [ + { id: 'cpu', amount: 2 }, + { id: 'GPU2', amount: 1 } + ] + const result = await engine.checkAndFillMissingResources(req, env, false) + // aggregate gpu (0 + 1) already meets floor 1 → nothing bumped + expect(result.find((r) => r.id === 'GPU1').amount).to.equal(0) + expect(result.find((r) => r.id === 'GPU2').amount).to.equal(1) + }) + + it('group floor: cpu selected, no GPU requested → exactly one GPU auto-bumped to 1', async function () { + const env = makeEnv(twoGpuResources()) + // 8 cpu but floor is absolute (perUnit:false) → still only 1 GPU total needed + const req: ComputeResourceRequest[] = [{ id: 'cpu', amount: 8 }] + const result = await engine.checkAndFillMissingResources(req, env, false) + const g1 = result.find((r) => r.id === 'GPU1').amount + const g2 = result.find((r) => r.id === 'GPU2').amount + expect(g1 + g2).to.equal(1) + }) + + it('group floor: prefers the GPU with lower inUse when auto-bumping', async function () { + // GPU1 already globally in use, GPU2 free → GPU2 should be the one bumped + const env = makeEnv(twoGpuResources(1, 0)) + const req: ComputeResourceRequest[] = [{ id: 'cpu', amount: 1 }] + const result = await engine.checkAndFillMissingResources(req, env, false) + expect(result.find((r) => r.id === 'GPU1').amount).to.equal(0) + expect(result.find((r) => r.id === 'GPU2').amount).to.equal(1) + }) + + it('group floor: required floor exceeds aggregate group max → throws Cannot satisfy', async function () { + const resources = twoGpuResources() + resources[0].constraints = [{ type: 'gpu', min: 3, perUnit: false }] + const env = makeEnv(resources) + const req: ComputeResourceRequest[] = [{ id: 'cpu', amount: 1 }] + try { + await engine.checkAndFillMissingResources(req, env, false) + assert.fail('Expected error was not thrown') + } catch (err: any) { + expect(err.message).to.include('Cannot satisfy') + expect(err.message).to.include('gpu resources') + } + }) + + it('group floor: constraint skipped when parent (cpu) not requested', async function () { + const resources = twoGpuResources() + resources[0].min = 0 // allow cpu to stay 0 so the parent is truly absent + const env = makeEnv(resources) + const req: ComputeResourceRequest[] = [{ id: 'ram', amount: 2 }] + const result = await engine.checkAndFillMissingResources(req, env, false) + expect(result.find((r) => r.id === 'GPU1').amount).to.equal(0) + expect(result.find((r) => r.id === 'GPU2').amount).to.equal(0) + }) + + it('group ceiling (type:gpu, max, perUnit:false): total group above max → throws Too much', async function () { + const resources = twoGpuResources() + resources[0].constraints = [{ type: 'gpu', max: 1, perUnit: false }] + const env = makeEnv(resources) + const req: ComputeResourceRequest[] = [ + { id: 'cpu', amount: 1 }, + { id: 'GPU1', amount: 1 }, + { id: 'GPU2', amount: 1 } + ] + try { + await engine.checkAndFillMissingResources(req, env, false) + assert.fail('Expected error was not thrown') + } catch (err: any) { + expect(err.message).to.include('Too much gpu resources') + expect(err.message).to.include('Max allowed: 1') + } + }) + + it('back-compat: exact-id constraint with no perUnit still uses ratio semantics', async function () { + const resources = [ + { ...baseResources[0], constraints: [{ id: 'ram', min: 2, max: 8 }] }, + ...baseResources.slice(1) + ] + const env = makeEnv(resources) + const req: ComputeResourceRequest[] = [ + { id: 'cpu', amount: 4 }, + { id: 'ram', amount: 4 }, + { id: 'disk', amount: 50 } + ] + const result = await engine.checkAndFillMissingResources(req, env, false) + // 4 cpu * 2 (ratio) = 8 + expect(result.find((r) => r.id === 'ram').amount).to.equal(8) + }) + + it('exact-id constraint with perUnit:false → absolute floor, not multiplied by parent', async function () { + const resources = [ + { ...baseResources[0], constraints: [{ id: 'ram', min: 8, perUnit: false }] }, + ...baseResources.slice(1) + ] + const env = makeEnv(resources) + const req: ComputeResourceRequest[] = [ + { id: 'cpu', amount: 4 }, + { id: 'ram', amount: 4 }, + { id: 'disk', amount: 50 } + ] + const result = await engine.checkAndFillMissingResources(req, env, false) + // absolute floor 8, NOT 4*8=32 + expect(result.find((r) => r.id === 'ram').amount).to.equal(8) + }) + + // ── Aggregate constraints (sum across per-device siblings) ──────────── + // Two per-device GPUs, each { id:'cpu', min:1, max:4, aggregate:true }. Renting both + // sums to cpu in [2, 8]; renting one gives [1, 4]. + const aggCpuConstraint = [{ id: 'cpu', min: 1, max: 4, aggregate: true }] + const twoAggGpuResources = (): any[] => [ + { id: 'cpu', type: 'cpu', kind: 'fungible', total: 16, min: 1, max: 16, inUse: 0 }, + { id: 'ram', type: 'ram', kind: 'fungible', total: 64, min: 1, max: 64, inUse: 0 }, + { + id: 'GPU1', + type: 'gpu', + kind: 'discrete', + total: 1, + min: 0, + max: 1, + inUse: 0, + constraints: aggCpuConstraint + }, + { + id: 'GPU2', + type: 'gpu', + kind: 'discrete', + total: 1, + min: 0, + max: 1, + inUse: 0, + constraints: aggCpuConstraint + } + ] + + it('aggregate min sums across two GPUs → cpu bumped to 2 (1+1)', async function () { + const env = makeEnv(twoAggGpuResources()) + const req: ComputeResourceRequest[] = [ + { id: 'GPU1', amount: 1 }, + { id: 'GPU2', amount: 1 } + ] + const result = await engine.checkAndFillMissingResources(req, env, false) + expect(result.find((r) => r.id === 'cpu').amount).to.equal(2) + }) + + it('aggregate max sums across two GPUs → cpu 8 allowed', async function () { + const env = makeEnv(twoAggGpuResources()) + const req: ComputeResourceRequest[] = [ + { id: 'GPU1', amount: 1 }, + { id: 'GPU2', amount: 1 }, + { id: 'cpu', amount: 8 } + ] + const result = await engine.checkAndFillMissingResources(req, env, false) + expect(result.find((r) => r.id === 'cpu').amount).to.equal(8) + }) + + it('aggregate max sums across two GPUs → cpu above 8 throws', async function () { + const env = makeEnv(twoAggGpuResources()) + const req: ComputeResourceRequest[] = [ + { id: 'GPU1', amount: 1 }, + { id: 'GPU2', amount: 1 }, + { id: 'cpu', amount: 10 } + ] + try { + await engine.checkAndFillMissingResources(req, env, false) + assert.fail('Expected error was not thrown') + } catch (err: any) { + expect(err.message).to.include('Too much cpu') + expect(err.message).to.include('Max allowed: 8') + } + }) + + it('aggregate with a single GPU → cpu bumped to 1 (only one contributes)', async function () { + const env = makeEnv(twoAggGpuResources()) + const req: ComputeResourceRequest[] = [{ id: 'GPU1', amount: 1 }] + const result = await engine.checkAndFillMissingResources(req, env, false) + expect(result.find((r) => r.id === 'cpu').amount).to.equal(1) + expect(result.find((r) => r.id === 'GPU2').amount).to.equal(0) + }) + + it('without aggregate the same per-GPU constraint does NOT sum (contrast)', async function () { + const resources = twoAggGpuResources() + // drop the aggregate flag → independent per-GPU constraints + resources[2].constraints = [{ id: 'cpu', min: 1, max: 4 }] + resources[3].constraints = [{ id: 'cpu', min: 1, max: 4 }] + const env = makeEnv(resources) + const result = await engine.checkAndFillMissingResources( + [ + { id: 'GPU1', amount: 1 }, + { id: 'GPU2', amount: 1 } + ], + env, + false + ) + // per-GPU: cpu floor is 1 (not 2) + expect(result.find((r) => r.id === 'cpu').amount).to.equal(1) + // per-GPU max is 4 (not 8): cpu:5 already throws + try { + await engine.checkAndFillMissingResources( + [ + { id: 'GPU1', amount: 1 }, + { id: 'GPU2', amount: 1 }, + { id: 'cpu', amount: 5 } + ], + makeEnv(resources), + false + ) + assert.fail('Expected error was not thrown') + } catch (err: any) { + expect(err.message).to.include('Max allowed: 4') + } + }) + + // ── More constraint recipes (mirrors docs/compute.md "More constraint recipes") ── + const mkCpu = (constraints?: any[]) => ({ + id: 'cpu', + type: 'cpu', + kind: 'fungible', + total: 32, + min: 1, + max: 32, + inUse: 0, + ...(constraints ? { constraints } : {}) + }) + const mkRam = { + id: 'ram', + type: 'ram', + kind: 'fungible', + total: 128, + min: 1, + max: 128, + inUse: 0 + } + const mkDisk = { + id: 'disk', + type: 'disk', + kind: 'fungible', + total: 2000, + min: 0, + max: 2000, + inUse: 0 + } + const mkGpu = (id: string, constraints?: any[]) => ({ + id, + type: 'gpu', + kind: 'discrete', + total: 1, + min: 0, + max: 1, + inUse: 0, + ...(constraints ? { constraints } : {}) + }) + + it('recipe 1: companion range {id:ram,min:8,max:16} bounds RAM per GPU', async function () { + const resources = [ + mkCpu(), + mkRam, + mkDisk, + mkGpu('GPU1', [{ id: 'ram', min: 8, max: 16 }]) + ] + const bumped = await engine.checkAndFillMissingResources( + [{ id: 'GPU1', amount: 1 }], + makeEnv(resources), + false + ) + expect(bumped.find((r) => r.id === 'ram').amount).to.equal(8) + try { + await engine.checkAndFillMissingResources( + [ + { id: 'GPU1', amount: 1 }, + { id: 'ram', amount: 20 } + ], + makeEnv(resources), + false + ) + assert.fail('Expected error was not thrown') + } catch (err: any) { + expect(err.message).to.include('Too much ram') + expect(err.message).to.include('Max allowed: 16') + } + }) + + it('recipe 2: group ratio {type:gpu,min:1} → one GPU per CPU', async function () { + const resources = [ + mkCpu([{ type: 'gpu', min: 1 }]), + mkRam, + mkDisk, + mkGpu('GPU1'), + mkGpu('GPU2') + ] + const r = await engine.checkAndFillMissingResources( + [{ id: 'cpu', amount: 2 }], + makeEnv(resources), + false + ) + // 2 CPUs (ratio, perUnit default) → 2 GPUs auto-assigned + const total = + r.find((x) => x.id === 'GPU1').amount + r.find((x) => x.id === 'GPU2').amount + expect(total).to.equal(2) + }) + + it('recipe 2b: group ratio requiring more GPUs than exposed → Cannot satisfy', async function () { + const resources = [ + mkCpu([{ type: 'gpu', min: 1 }]), + mkRam, + mkDisk, + mkGpu('GPU1'), + mkGpu('GPU2') + ] + try { + await engine.checkAndFillMissingResources( + [{ id: 'cpu', amount: 3 }], + makeEnv(resources), + false + ) + assert.fail('Expected error was not thrown') + } catch (err: any) { + expect(err.message).to.include('Cannot satisfy') + expect(err.message).to.include('at least 3 gpu resources') + expect(err.message).to.include('max is 2') + } + }) + + it('recipe 3: group ceiling {type:gpu,max:2,perUnit:false} caps total GPUs', async function () { + const resources = [ + mkCpu([{ type: 'gpu', max: 2, perUnit: false }]), + mkRam, + mkDisk, + mkGpu('GPU1'), + mkGpu('GPU2'), + mkGpu('GPU3') + ] + // 2 GPUs is fine + const ok = await engine.checkAndFillMissingResources( + [ + { id: 'cpu', amount: 1 }, + { id: 'GPU1', amount: 1 }, + { id: 'GPU2', amount: 1 } + ], + makeEnv(resources), + false + ) + expect(ok.find((r) => r.id === 'GPU3').amount).to.equal(0) + // 3 GPUs exceeds the cap + try { + await engine.checkAndFillMissingResources( + [ + { id: 'cpu', amount: 1 }, + { id: 'GPU1', amount: 1 }, + { id: 'GPU2', amount: 1 }, + { id: 'GPU3', amount: 1 } + ], + makeEnv(resources), + false + ) + assert.fail('Expected error was not thrown') + } catch (err: any) { + expect(err.message).to.include('Too much gpu resources') + expect(err.message).to.include('Max allowed: 2') + } + }) + + it('recipe 4: aggregate {id:ram,min:8,aggregate:true} sums RAM across GPUs', async function () { + const agg = [{ id: 'ram', min: 8, aggregate: true }] + const resources = [mkCpu(), mkRam, mkDisk, mkGpu('GPU1', agg), mkGpu('GPU2', agg)] + const two = await engine.checkAndFillMissingResources( + [ + { id: 'GPU1', amount: 1 }, + { id: 'GPU2', amount: 1 } + ], + makeEnv(resources), + false + ) + expect(two.find((r) => r.id === 'ram').amount).to.equal(16) + const one = await engine.checkAndFillMissingResources( + [{ id: 'GPU1', amount: 1 }], + makeEnv(resources), + false + ) + expect(one.find((r) => r.id === 'ram').amount).to.equal(8) + }) + + it('recipe 5: multiple companion constraints on one GPU bump each target', async function () { + const resources = [ + mkCpu(), + mkRam, + mkDisk, + mkGpu('GPU1', [ + { id: 'ram', min: 8 }, + { id: 'cpu', min: 2 }, + { id: 'disk', min: 20 } + ]) + ] + const r = await engine.checkAndFillMissingResources( + [{ id: 'GPU1', amount: 1 }], + makeEnv(resources), + false + ) + expect(r.find((x) => x.id === 'ram').amount).to.equal(8) + expect(r.find((x) => x.id === 'cpu').amount).to.equal(2) + expect(r.find((x) => x.id === 'disk').amount).to.equal(20) + }) + + it('recipe 6: absolute RAM floor {id:ram,min:32,perUnit:false} on a GPU', async function () { + const resources = [ + mkCpu(), + mkRam, + mkDisk, + mkGpu('GPU1', [{ id: 'ram', min: 32, perUnit: false }]) + ] + const r = await engine.checkAndFillMissingResources( + [{ id: 'GPU1', amount: 1 }], + makeEnv(resources), + false + ) + expect(r.find((x) => x.id === 'ram').amount).to.equal(32) + }) + + // ── Contradictory per-GPU constraints must be rejected regardless of order ── + // gpu0 caps cpu at 6 while gpu1 requires at least 8: renting both is unsatisfiable. + // A single-pass check let this through when gpu0 appeared first (its max was validated + // before gpu1's min bumped cpu to 8); the max phase now re-validates ceilings against + // the settled amounts, so both orderings reject. + it('contradictory constraints (gpu0 cpu max:6 vs gpu1 cpu min:8) → rejected in both env orders', async function () { + const gpu0 = () => mkGpu('gpu0', [{ id: 'cpu', max: 6 }]) + const gpu1 = () => mkGpu('gpu1', [{ id: 'cpu', min: 8 }]) + const bothGpus: ComputeResourceRequest[] = [ + { id: 'cpu', amount: 4 }, + { id: 'gpu0', amount: 1 }, + { id: 'gpu1', amount: 1 } + ] + for (const resources of [ + [mkCpu(), mkRam, mkDisk, gpu0(), gpu1()], + [mkCpu(), mkRam, mkDisk, gpu1(), gpu0()] + ]) { + try { + await engine.checkAndFillMissingResources( + bothGpus.map((r) => ({ ...r })), + makeEnv(resources), + false + ) + assert.fail('Expected error was not thrown') + } catch (err: any) { + expect(err.message).to.include('Too much cpu') + expect(err.message).to.include('1 gpu0') + expect(err.message).to.include('Max allowed: 6') + expect(err.message).to.include('requested: 8') + } + } + }) + + it('contradictory constraints: renting a single GPU still works', async function () { + const resources = [ + mkCpu(), + mkRam, + mkDisk, + mkGpu('gpu0', [{ id: 'cpu', max: 6 }]), + mkGpu('gpu1', [{ id: 'cpu', min: 8 }]) + ] + // gpu0 alone: cpu 4 ≤ 6 → untouched + const r0 = await engine.checkAndFillMissingResources( + [ + { id: 'cpu', amount: 4 }, + { id: 'gpu0', amount: 1 } + ], + makeEnv(resources), + false + ) + expect(r0.find((x) => x.id === 'cpu').amount).to.equal(4) + // gpu1 alone: cpu bumped to its floor of 8 + const r1 = await engine.checkAndFillMissingResources( + [ + { id: 'cpu', amount: 4 }, + { id: 'gpu1', amount: 1 } + ], + makeEnv(resources), + false + ) + expect(r1.find((x) => x.id === 'cpu').amount).to.equal(8) + }) }) describe('testing checkIfResourcesAreAvailable', function () { @@ -452,13 +1101,22 @@ describe('Compute Jobs Database', () => { before(function () { engine = new TestC2DEngine() + engine.setPhysicalLimits( + new Map([ + ['cpu', 10], + ['ram', 32], + ['disk', 100], + ['gpu0', 1], + ['nic0', 1] + ]) + ) }) it('resources within env limits → passes', async function () { const env = makeEnv([ - { id: 'cpu', total: 8, min: 1, max: 8, inUse: 2 }, - { id: 'ram', total: 32, min: 1, max: 32, inUse: 4 }, - { id: 'disk', total: 500, min: 10, max: 500, inUse: 50 } + { id: 'cpu', kind: 'fungible', total: 8, min: 1, max: 8, inUse: 2 }, + { id: 'ram', kind: 'fungible', total: 32, min: 1, max: 32, inUse: 4 }, + { id: 'disk', kind: 'fungible', total: 500, min: 10, max: 500, inUse: 50 } ]) const req: ComputeResourceRequest[] = [ { id: 'cpu', amount: 4 }, @@ -471,9 +1129,9 @@ describe('Compute Jobs Database', () => { it('resources exceed env availability → throws', async function () { const env = makeEnv([ - { id: 'cpu', total: 4, min: 1, max: 4, inUse: 3 }, - { id: 'ram', total: 32, min: 1, max: 32, inUse: 0 }, - { id: 'disk', total: 500, min: 10, max: 500, inUse: 0 } + { id: 'cpu', kind: 'fungible', total: 4, min: 1, max: 4, inUse: 3 }, + { id: 'ram', kind: 'fungible', total: 32, min: 1, max: 32, inUse: 0 }, + { id: 'disk', kind: 'fungible', total: 500, min: 10, max: 500, inUse: 0 } ]) const req: ComputeResourceRequest[] = [ { id: 'cpu', amount: 4 }, // only 1 available (4-3) @@ -491,15 +1149,15 @@ describe('Compute Jobs Database', () => { it('free resource limit exceeded → throws', async function () { const env = makeEnv( [ - { id: 'cpu', total: 8, min: 1, max: 8, inUse: 0 }, - { id: 'ram', total: 32, min: 1, max: 32, inUse: 0 }, - { id: 'disk', total: 500, min: 10, max: 500, inUse: 0 } + { id: 'cpu', kind: 'fungible', total: 8, min: 1, max: 8, inUse: 0 }, + { id: 'ram', kind: 'fungible', total: 32, min: 1, max: 32, inUse: 0 }, + { id: 'disk', kind: 'fungible', total: 500, min: 10, max: 500, inUse: 0 } ], { freeResources: [ - { id: 'cpu', total: 2, min: 1, max: 2, inUse: 2 }, // fully used - { id: 'ram', total: 4, min: 1, max: 4, inUse: 0 }, - { id: 'disk', total: 20, min: 10, max: 20, inUse: 0 } + { id: 'cpu', kind: 'fungible', total: 2, min: 1, max: 2, inUse: 2 }, // fully used + { id: 'ram', kind: 'fungible', total: 4, min: 1, max: 4, inUse: 0 }, + { id: 'disk', kind: 'fungible', total: 20, min: 10, max: 20, inUse: 0 } ] } ) @@ -515,32 +1173,1159 @@ describe('Compute Jobs Database', () => { expect(err.message).to.include('cpu') } }) - }) - after(async () => { - await tearDownEnvironment(envOverrides) - }) -}) - -describe('getAlgoChecksums', () => { - let findDdoStub: sinon.SinonStub - let loggerErrorSpy: sinon.SinonSpy + it('Gate 1 (per-env ceiling, fungible) blocks when env capacity exhausted', async function () { + const env = makeEnv([ + { id: 'cpu', kind: 'fungible', total: 6, min: 1, max: 6, inUse: 6 } + ]) + const req: ComputeResourceRequest[] = [{ id: 'cpu', amount: 1 }] + try { + await engine.checkIfResourcesAreAvailable(req, env, false) + assert.fail('Expected error was not thrown') + } catch (err: any) { + expect(err.message).to.include('Not enough available cpu') + expect(err.message).to.include('environment') + } + }) - beforeEach(() => { - findDdoStub = sinon.stub(FindDdoHandler.prototype, 'findAndFormatDdo') - loggerErrorSpy = sinon.spy(CORE_LOGGER, 'error') - }) + it('Gate 2 (engine-wide pool, fungible) blocks when global capacity exhausted across two envs', async function () { + // Pool: 10 physical CPUs. env1 uses 6, env2 uses 4 → 10 total in-use. + const env1 = makeEnv([ + { id: 'cpu', kind: 'fungible', total: 6, min: 1, max: 6, inUse: 6 } + ]) + env1.id = 'env1' + const env2 = makeEnv([ + { id: 'cpu', kind: 'fungible', total: 6, min: 1, max: 6, inUse: 4 } + ]) + env2.id = 'env2' + // env2 Gate 1: 6 - 4 = 2 >= 1 → passes. Gate 2: total 12 capped to 10, used 10, remaining 0 < 1 → blocks. + const req: ComputeResourceRequest[] = [{ id: 'cpu', amount: 1 }] + try { + await engine.checkIfResourcesAreAvailable(req, env2, false, [env1, env2]) + assert.fail('Expected error was not thrown') + } catch (err: any) { + expect(err.message).to.include('globally') + } + }) - afterEach(() => { - findDdoStub.restore() - loggerErrorSpy.restore() - }) + it('Gate 2 passes when global capacity is available (env1 partially used)', async function () { + const env1 = makeEnv([ + { id: 'cpu', kind: 'fungible', total: 6, min: 1, max: 6, inUse: 3 } + ]) + env1.id = 'env1' + const env2 = makeEnv([ + { id: 'cpu', kind: 'fungible', total: 6, min: 1, max: 6, inUse: 2 } + ]) + env2.id = 'env2' + // Gate 2: total 12 capped to 10, used 5, remaining 5 >= 1 → passes + const req: ComputeResourceRequest[] = [{ id: 'cpu', amount: 1 }] + await engine.checkIfResourcesAreAvailable(req, env2, false, [env1, env2]) + // no throw = pass + }) - it('returns empty checksums without a DDO lookup for raw-code algorithms (no documentId)', async () => { - const checksums = await getAlgoChecksums( - undefined, - undefined, - null as any, + it('discrete exclusive (GPU) globally tracked — second job blocked when total:1 in use', async function () { + // gpu0 is a discrete exclusive resource (total:1). env1 has it in-use. + const env1 = makeEnv([ + { + id: 'gpu0', + kind: 'discrete', + shareable: false, + total: 1, + min: 0, + max: 1, + inUse: 1 + } + ]) + env1.id = 'env1' + const env2 = makeEnv([ + { + id: 'gpu0', + kind: 'discrete', + shareable: false, + total: 1, + min: 0, + max: 1, + inUse: 0 + } + ]) + env2.id = 'env2' + // Gate 2: globalTotal = 2 capped to physicalLimits['gpu0']=1, globalUsed=1, remaining=0 < 1 → blocks + const req: ComputeResourceRequest[] = [{ id: 'gpu0', amount: 1 }] + try { + await engine.checkIfResourcesAreAvailable(req, env2, false, [env1, env2]) + assert.fail('Expected error was not thrown') + } catch (err: any) { + expect(err.message).to.include('gpu0') + expect(err.message).to.include('globally') + } + }) + + it('discrete shareable (NIC) never blocks allocation — both jobs admitted', async function () { + // nic0 is shareable discrete. env1 already has it in-use. + const env1 = makeEnv([ + { + id: 'nic0', + kind: 'discrete', + shareable: true, + total: 1, + min: 0, + max: 1, + inUse: 1 + } + ]) + env1.id = 'env1' + const env2 = makeEnv([ + { + id: 'nic0', + kind: 'discrete', + shareable: true, + total: 1, + min: 0, + max: 1, + inUse: 0 + } + ]) + env2.id = 'env2' + // isShareableDiscrete = true → Gate 2 skipped → no throw + const req: ComputeResourceRequest[] = [{ id: 'nic0', amount: 1 }] + await engine.checkIfResourcesAreAvailable(req, env2, false, [env1, env2]) + // no throw = pass + }) + + it('non-GPU discrete resource is globally tracked (kind drives tracking, not type)', async function () { + // An FPGA with kind:'discrete' but type:'fpga' must be globally tracked just like a GPU. + const env1 = makeEnv([ + { + id: 'fpga0', + kind: 'discrete', + type: 'fpga', + total: 1, + min: 0, + max: 1, + inUse: 1 + } + ]) + env1.id = 'env1' + const env2 = makeEnv([ + { + id: 'fpga0', + kind: 'discrete', + type: 'fpga', + total: 1, + min: 0, + max: 1, + inUse: 0 + } + ]) + env2.id = 'env2' + ;(engine as any).physicalLimits.set('fpga0', 1) + const req: ComputeResourceRequest[] = [{ id: 'fpga0', amount: 1 }] + try { + await engine.checkIfResourcesAreAvailable(req, env2, false, [env1, env2]) + assert.fail('Expected error was not thrown') + } catch (err: any) { + expect(err.message).to.include('fpga0') + expect(err.message).to.include('globally') + } + }) + + it('discrete GPU — double-counting across envs does not block when capacity remains', async function () { + // Setup: 2 physical GPUs (physicalLimits gpu0=2), two environments each advertising + // total:2. A single job consumes 1 GPU on env1. getUsedResources aggregates discrete + // usage globally, so both env1 and env2 receive inUse:1. Without the max-vs-sum fix, + // checkGlobalResourceAvailability would compute globalUsed = 1+1 = 2, exhausting + // the physical pool and incorrectly blocking the next allocation. + engine.setPhysicalLimits( + new Map([ + ['cpu', 10], + ['ram', 32], + ['disk', 100], + ['gpu0', 2], + ['nic0', 1] + ]) + ) + const env1 = makeEnv([ + { + id: 'gpu0', + kind: 'discrete', + shareable: false, + total: 2, + min: 0, + max: 2, + inUse: 1 + } + ]) + env1.id = 'env1' + // env2 carries the same global inUse value because getUsedResources tracks discrete globally + const env2 = makeEnv([ + { + id: 'gpu0', + kind: 'discrete', + shareable: false, + total: 2, + min: 0, + max: 2, + inUse: 1 + } + ]) + env2.id = 'env2' + // 1 GPU in use, 1 remaining — this request must succeed, not be double-blocked + const req: ComputeResourceRequest[] = [{ id: 'gpu0', amount: 1 }] + await engine.checkIfResourcesAreAvailable(req, env2, false, [env1, env2]) + // no throw = pass (double-counting would have thrown "Not enough gpu0 globally") + }) + }) + + after(async () => { + await tearDownEnvironment(envOverrides) + }) +}) + +describe('Schema validation (C2DDockerConfigSchema)', () => { + const validBase = { + socketPath: '/var/run/docker.sock', + environments: [ + { + storageExpiry: 604800, + maxJobDuration: 3600, + minJobDuration: 60, + fees: { '1': [{ feeToken: '0x123', prices: [{ id: 'cpu', price: 1 }] }] } + } + ] + } + + it('old format (env resources with init) is rejected — clean break enforced', function () { + const config = [ + { + ...validBase, + environments: [ + { + ...validBase.environments[0], + resources: [ + { + id: 'gpu0', + total: 1, + init: { + deviceRequests: { + Driver: 'nvidia', + DeviceIDs: ['uuid-a'], + Capabilities: [['gpu']] + } + } + } + ] + } + ] + } + ] + const result = C2DDockerConfigSchema.safeParse(config) + expect(result.success).to.equal(false) + const msgs = result.error?.issues.map((i) => i.message).join(' ') + expect(msgs).to.include('migration guide') + }) + + it('env ref pointing to unknown pool id is rejected', function () { + const config = [ + { + ...validBase, + environments: [ + { + ...validBase.environments[0], + resources: [{ id: 'unknown-gpu' }] + } + ] + } + ] + const result = C2DDockerConfigSchema.safeParse(config) + expect(result.success).to.equal(false) + const msgs = result.error?.issues.map((i) => i.message).join(' ') + expect(msgs).to.include('not found in connection-level resources') + }) + + it('shareable:true on type:gpu resource is rejected', function () { + const config = [ + { + ...validBase, + resources: [ + { + id: 'gpu0', + type: 'gpu', + kind: 'discrete', + total: 1, + shareable: true, + init: { + deviceRequests: { + Driver: 'nvidia', + DeviceIDs: ['uuid-a'], + Capabilities: [['gpu']] + } + } + } + ], + environments: [ + { + ...validBase.environments[0], + resources: [{ id: 'gpu0' }] + } + ] + } + ] + const result = C2DDockerConfigSchema.safeParse(config) + expect(result.success).to.equal(false) + const msgs = result.error?.issues.map((i) => i.message).join(' ') + expect(msgs).to.include('shareable:true is not allowed') + }) + + it('valid two-level config with GPU pool and env refs parses successfully', function () { + const config = [ + { + socketPath: '/var/run/docker.sock', + resources: [ + { + id: 'gpu0', + kind: 'discrete', + type: 'gpu', + total: 1, + init: { + deviceRequests: { + Driver: 'nvidia', + DeviceIDs: ['uuid-a'], + Capabilities: [['gpu']] + } + } + } + ], + environments: [ + { + storageExpiry: 604800, + maxJobDuration: 3600, + minJobDuration: 60, + resources: [{ id: 'cpu' }, { id: 'ram' }, { id: 'disk' }, { id: 'gpu0' }], + fees: { '1': [{ feeToken: '0x123', prices: [{ id: 'gpu0', price: 5 }] }] } + } + ] + } + ] + const result = C2DDockerConfigSchema.safeParse(config) + expect(result.success).to.equal(true) + }) + + const gpuPool = [ + { + id: 'gpu0', + kind: 'discrete', + type: 'gpu', + total: 1, + init: { + deviceRequests: { + Driver: 'nvidia', + DeviceIDs: ['uuid-a'], + Capabilities: [['gpu']] + } + } + } + ] + + it('resource constraint with both id and type is rejected', function () { + const config = [ + { + ...validBase, + resources: gpuPool, + environments: [ + { + ...validBase.environments[0], + resources: [ + { id: 'cpu', constraints: [{ id: 'ram', type: 'gpu', min: 1 }] }, + { id: 'ram' }, + { id: 'disk' }, + { id: 'gpu0' } + ] + } + ] + } + ] + const result = C2DDockerConfigSchema.safeParse(config) + expect(result.success).to.equal(false) + const msgs = result.error?.issues.map((i) => i.message).join(' ') + expect(msgs).to.include('mutually exclusive') + }) + + it('resource constraint with neither id nor type is rejected', function () { + const config = [ + { + ...validBase, + resources: gpuPool, + environments: [ + { + ...validBase.environments[0], + resources: [ + { id: 'cpu', constraints: [{ min: 1 }] }, + { id: 'ram' }, + { id: 'disk' }, + { id: 'gpu0' } + ] + } + ] + } + ] + const result = C2DDockerConfigSchema.safeParse(config) + expect(result.success).to.equal(false) + const msgs = result.error?.issues.map((i) => i.message).join(' ') + expect(msgs).to.include('either "id" or "type"') + }) + + it('valid group+floor constraint (type:gpu, perUnit:false) parses successfully', function () { + const config = [ + { + ...validBase, + resources: gpuPool, + environments: [ + { + ...validBase.environments[0], + resources: [ + { id: 'cpu', constraints: [{ type: 'gpu', min: 1, perUnit: false }] }, + { id: 'ram' }, + { id: 'disk' }, + { id: 'gpu0' } + ] + } + ] + } + ] + const result = C2DDockerConfigSchema.safeParse(config) + expect(result.success).to.equal(true) + }) + + it('aggregate constraint targeting a type group is rejected', function () { + const config = [ + { + ...validBase, + resources: gpuPool, + environments: [ + { + ...validBase.environments[0], + resources: [ + { id: 'cpu' }, + { id: 'ram' }, + { id: 'disk' }, + { id: 'gpu0', constraints: [{ type: 'gpu', min: 1, aggregate: true }] } + ] + } + ] + } + ] + const result = C2DDockerConfigSchema.safeParse(config) + expect(result.success).to.equal(false) + const msgs = result.error?.issues.map((i) => i.message).join(' ') + expect(msgs).to.include('aggregate constraints must target a single "id"') + }) + + it('valid aggregate constraint (single id) parses successfully', function () { + const config = [ + { + ...validBase, + resources: gpuPool, + environments: [ + { + ...validBase.environments[0], + resources: [ + { id: 'cpu' }, + { id: 'ram' }, + { id: 'disk' }, + { + id: 'gpu0', + constraints: [{ id: 'cpu', min: 1, max: 4, aggregate: true }] + } + ] + } + ] + } + ] + const result = C2DDockerConfigSchema.safeParse(config) + expect(result.success).to.equal(true) + }) + + // Keeps the "Full example" config in docs/compute.md honest — if the schema changes in a way + // that would break the documented example, this test fails. + it('the docs/compute.md full example config parses successfully', function () { + const config: any = [ + { + socketPath: '/var/run/docker.sock', + resources: [ + { id: 'cpu', type: 'cpu', total: 16 }, + { id: 'ram', type: 'ram', total: 64 }, + { id: 'disk', type: 'disk', total: 2000 }, + { + id: 'GPU1', + kind: 'discrete', + type: 'gpu', + total: 1, + description: 'NVIDIA A100 40GB (slot 0)', + platform: 'nvidia', + driverVersion: '570.195.03', + memoryTotal: '40960 MiB', + init: { + deviceRequests: { + Driver: 'nvidia', + DeviceIDs: ['GPU-uuid-1'], + Capabilities: [['gpu']] + } + }, + constraints: [ + { id: 'ram', min: 8 }, + { id: 'cpu', min: 2 } + ] + }, + { + id: 'GPU2', + kind: 'discrete', + type: 'gpu', + total: 1, + description: 'NVIDIA A100 40GB (slot 1)', + platform: 'nvidia', + driverVersion: '570.195.03', + memoryTotal: '40960 MiB', + init: { + deviceRequests: { + Driver: 'nvidia', + DeviceIDs: ['GPU-uuid-2'], + Capabilities: [['gpu']] + } + }, + constraints: [ + { id: 'ram', min: 8 }, + { id: 'cpu', min: 2 } + ] + } + ], + environments: [ + { + id: 'gpu-premium', + description: 'Paid GPU environment — any CPU job is guaranteed a GPU', + storageExpiry: 604800, + maxJobDuration: 3600, + minJobDuration: 60, + maxJobs: 2, + enableNetwork: true, + resources: [ + { + id: 'cpu', + min: 1, + max: 8, + constraints: [{ type: 'gpu', min: 1, perUnit: false }] + }, + { id: 'ram', min: 1, max: 32 }, + { id: 'disk', min: 1, max: 500 }, + { id: 'GPU1' }, + { id: 'GPU2' } + ], + access: { addresses: [], accessLists: null }, + fees: { + '1': [ + { + feeToken: '0x967da4048cD07aB37855c090aAF366e4ce1b9F48', + prices: [ + { id: 'cpu', price: 1 }, + { id: 'ram', price: 0.5 }, + { id: 'disk', price: 0.1 }, + { id: 'GPU1', price: 10 }, + { id: 'GPU2', price: 10 } + ] + } + ], + '137': [ + { + feeToken: '0x2791Bca1f2de4661ED88A30C99A7a9449Aa84174', + prices: [ + { id: 'cpu', price: 0.5 }, + { id: 'ram', price: 0.2 }, + { id: 'GPU1', price: 5 }, + { id: 'GPU2', price: 5 } + ] + } + ] + }, + free: { + maxJobDuration: 300, + minJobDuration: 10, + maxJobs: 1, + access: { addresses: [], accessLists: null }, + resources: [ + { + id: 'cpu', + max: 2, + constraints: [{ type: 'gpu', min: 1, perUnit: false }] + }, + { id: 'ram', max: 8 }, + { id: 'disk', max: 10 }, + { + id: 'GPU1', + constraints: [ + { id: 'ram', min: 4 }, + { id: 'cpu', min: 1 } + ] + } + ] + } + }, + { + id: 'cpu-standard', + description: 'CPU-only environment — cheaper, no GPU', + storageExpiry: 604800, + maxJobDuration: 1800, + minJobDuration: 60, + maxJobs: 10, + enableNetwork: false, + resources: [ + { id: 'cpu', total: 8, min: 1, max: 4 }, + { id: 'ram', total: 16, min: 1, max: 8 }, + { id: 'disk', max: 100 } + ], + access: { addresses: [], accessLists: null }, + fees: { + '1': [ + { + feeToken: '0x967da4048cD07aB37855c090aAF366e4ce1b9F48', + prices: [ + { id: 'cpu', price: 0.3 }, + { id: 'ram', price: 0.1 }, + { id: 'disk', price: 0.05 } + ] + } + ] + }, + free: { + maxJobDuration: 120, + minJobDuration: 10, + maxJobs: 3, + access: { addresses: [], accessLists: null }, + resources: [ + { id: 'cpu', max: 1 }, + { id: 'ram', max: 2 }, + { id: 'disk', max: 5 } + ] + } + } + ] + } + ] + const result = C2DDockerConfigSchema.safeParse(config) + expect(result.success).to.equal(true) + }) + + describe('cpuList validation', function () { + const withCpuResource = (cpuEntry: any) => [{ ...validBase, resources: [cpuEntry] }] + + it('cpu resource with a single valid range parses successfully', function () { + const result = C2DDockerConfigSchema.safeParse( + withCpuResource({ id: 'cpu', cpuList: '32-63' }) + ) + expect(result.success).to.equal(true) + }) + + it('cpu resource with multiple ascending ranges parses successfully', function () { + const result = C2DDockerConfigSchema.safeParse( + withCpuResource({ id: 'cpu', cpuList: '0-15,32-47' }) + ) + expect(result.success).to.equal(true) + }) + + it('cpu resource with a single bare core ID parses successfully', function () { + const result = C2DDockerConfigSchema.safeParse( + withCpuResource({ id: 'cpu', cpuList: '0' }) + ) + expect(result.success).to.equal(true) + }) + + it('cpu resource mixing ranges and bare core IDs parses successfully', function () { + const result = C2DDockerConfigSchema.safeParse( + withCpuResource({ id: 'cpu', cpuList: '0-1,3' }) + ) + expect(result.success).to.equal(true) + }) + + it('cpu resource with a list of bare core IDs parses successfully', function () { + const result = C2DDockerConfigSchema.safeParse( + withCpuResource({ id: 'cpu', cpuList: '3,5,7' }) + ) + expect(result.success).to.equal(true) + }) + + it('cpu resource with only total still parses (existing behavior)', function () { + const result = C2DDockerConfigSchema.safeParse( + withCpuResource({ id: 'cpu', total: 6 }) + ) + expect(result.success).to.equal(true) + }) + + it('cpu resource with both total and cpuList is rejected', function () { + const result = C2DDockerConfigSchema.safeParse( + withCpuResource({ id: 'cpu', total: 32, cpuList: '32-63' }) + ) + expect(result.success).to.equal(false) + const msgs = result.error?.issues.map((i) => i.message).join(' ') + expect(msgs).to.include('not both') + }) + + it('cpu resource with neither total nor cpuList is rejected', function () { + const result = C2DDockerConfigSchema.safeParse(withCpuResource({ id: 'cpu' })) + expect(result.success).to.equal(false) + const msgs = result.error?.issues.map((i) => i.message).join(' ') + expect(msgs).to.include('must specify either "total" or "cpuList"') + }) + + it('cpuList on a non-cpu resource is rejected', function () { + const result = C2DDockerConfigSchema.safeParse( + withCpuResource({ id: 'ram', cpuList: '0-3' }) + ) + expect(result.success).to.equal(false) + const msgs = result.error?.issues.map((i) => i.message).join(' ') + expect(msgs).to.include('only valid on the cpu resource') + }) + + it('cpuList inside an env-level resource ref is rejected', function () { + const config = [ + { + ...validBase, + environments: [ + { + ...validBase.environments[0], + resources: [{ id: 'cpu', cpuList: '0-3' }] + } + ] + } + ] + const result = C2DDockerConfigSchema.safeParse(config) + expect(result.success).to.equal(false) + const msgs = result.error?.issues.map((i) => i.message).join(' ') + expect(msgs).to.include('must be defined at connection level') + }) + + // Format is strict: comma-separated core IDs and/or integer ranges, each range's + // right side strictly greater than the left, all parts ascending and non-overlapping. + const invalidLists: [string, string][] = [ + ['15-15', 'strictly greater'], // right side equal — write "15" instead + ['20-10', 'strictly greater'], // right side lower + ['0-8,4-12', 'ascending and non-overlapping'], // overlapping ranges + ['8-11,0-3', 'ascending and non-overlapping'], // out of order + ['0-3,2', 'ascending and non-overlapping'], // bare ID inside a previous range + ['3,3', 'ascending and non-overlapping'], // duplicate bare ID + ['1.5-4', 'is invalid'], // floats + ['0-9000', 'maximum supported limit'], // core ID above the 8192 cap + ['-1-4', 'is invalid'], // negative + ['0-3, 8-11', 'is invalid'], // space + ['0-3,,8-11', 'is invalid'], // empty part + ['0-3,', 'is invalid'], // trailing comma + ['', 'is invalid'] // empty string + ] + for (const [value, expectedFragment] of invalidLists) { + it(`cpuList "${value}" is rejected (${expectedFragment})`, function () { + const result = C2DDockerConfigSchema.safeParse( + withCpuResource({ id: 'cpu', cpuList: value }) + ) + expect(result.success).to.equal(false) + const msgs = result.error?.issues.map((i) => i.message).join(' ') + expect(msgs).to.include(expectedFragment) + }) + } + }) +}) + +describe('resolveResourceKind / resolveConnectionResourcePool / resolveEnvironmentResources', () => { + let engine: any + + beforeEach(function () { + // Use Object.create to bypass the Docker-specific constructor while retaining the prototype chain. + engine = Object.create(C2DEngineDocker.prototype) + engine.physicalLimits = new Map() + }) + + describe('resolveResourceKind()', function () { + it('explicit kind:"discrete" wins over init presence', function () { + const res: Partial = { + id: 'cpu', + kind: 'discrete', + init: undefined + } + expect(engine.resolveResourceKind(res)).to.equal('discrete') + }) + + it('explicit kind:"fungible" wins even when init is present', function () { + const res: Partial = { + id: 'cpu', + kind: 'fungible', + init: { + deviceRequests: { Driver: 'nvidia', DeviceIDs: ['x'], Capabilities: [['gpu']] } + } + } + expect(engine.resolveResourceKind(res)).to.equal('fungible') + }) + + it('no kind + init present → inferred as discrete', function () { + const res: Partial = { + id: 'gpu0', + init: { + deviceRequests: { Driver: 'nvidia', DeviceIDs: ['x'], Capabilities: [['gpu']] } + } + } + expect(engine.resolveResourceKind(res)).to.equal('discrete') + }) + + it('no kind, no init → inferred as fungible', function () { + const res: Partial = { id: 'cpu' } + expect(engine.resolveResourceKind(res)).to.equal('fungible') + }) + }) + + describe('resolveConnectionResourcePool()', function () { + it('auto-detects cpu and ram from sysinfo; disk from physicalLimits', function () { + engine.physicalLimits.set('disk', 200) + const sysinfo = { NCPU: 8, MemTotal: 32 * 1024 * 1024 * 1024 } // 32 GB + const pool = engine.resolveConnectionResourcePool(sysinfo, []) + expect(pool.get('cpu').total).to.equal(8) + expect(pool.get('ram').total).to.equal(32) + expect(pool.get('disk').total).to.equal(200) + expect(pool.get('cpu').kind).to.equal('fungible') + expect(pool.get('ram').kind).to.equal('fungible') + }) + + it('configured total caps cpu at physical limit', function () { + engine.physicalLimits.set('cpu', 8) + engine.physicalLimits.set('disk', 100) + const sysinfo = { NCPU: 8, MemTotal: 32 * 1024 * 1024 * 1024 } + // Config requests 6 cores (cap below physical) → should use 6. + const pool = engine.resolveConnectionResourcePool(sysinfo, [ + { id: 'cpu', total: 6, min: 1 } + ]) + expect(pool.get('cpu').total).to.equal(6) + }) + + it('configured total exceeding physical is capped at physical', function () { + engine.physicalLimits.set('cpu', 8) + engine.physicalLimits.set('disk', 100) + const sysinfo = { NCPU: 8, MemTotal: 16 * 1024 * 1024 * 1024 } + // Config requests 20 cores on an 8-core host → capped to 8. + const pool = engine.resolveConnectionResourcePool(sysinfo, [ + { id: 'cpu', total: 20 } + ]) + expect(pool.get('cpu').total).to.equal(8) + }) + + it('cpuList sets effective cpu total to the expanded core count', function () { + engine.physicalLimits.set('cpu', 64) + engine.physicalLimits.set('disk', 100) + const sysinfo = { NCPU: 64, MemTotal: 32 * 1024 * 1024 * 1024 } + const pool = engine.resolveConnectionResourcePool(sysinfo, [ + { id: 'cpu', cpuList: '32-63' } + ]) + expect(pool.get('cpu').total).to.equal(32) + expect(pool.get('cpu').max).to.equal(32) + }) + + it('custom GPU resource is added to pool and registered in physicalLimits', function () { + engine.physicalLimits.set('disk', 100) + const sysinfo = { NCPU: 4, MemTotal: 8 * 1024 * 1024 * 1024 } + const gpu = { + id: 'gpu0', + type: 'gpu', + total: 1, + init: { + deviceRequests: { + Driver: 'nvidia', + DeviceIDs: ['uuid-a'], + Capabilities: [['gpu']] + } + } + } + const pool = engine.resolveConnectionResourcePool(sysinfo, [gpu]) + expect(pool.has('gpu0')).to.equal(true) + expect(pool.get('gpu0').kind).to.equal('discrete') // inferred from init + expect(pool.get('gpu0').total).to.equal(1) + expect(engine.physicalLimits.get('gpu0')).to.equal(1) + }) + }) + + describe('resolveEnvironmentResources()', function () { + let pool: Map + + beforeEach(function () { + pool = new Map([ + ['cpu', { id: 'cpu', kind: 'fungible', type: 'cpu', total: 10, min: 1, max: 10 }], + ['ram', { id: 'ram', kind: 'fungible', type: 'ram', total: 32, min: 1, max: 32 }], + [ + 'disk', + { id: 'disk', kind: 'fungible', type: 'disk', total: 100, min: 1, max: 100 } + ], + [ + 'gpu0', + { + id: 'gpu0', + kind: 'discrete', + type: 'gpu', + total: 1, + min: 0, + max: 1, + constraints: [{ id: 'ram', min: 4 }], + init: { + deviceRequests: { + Driver: 'nvidia', + DeviceIDs: ['uuid-a'], + Capabilities: [['gpu']] + } + } + } + ] + ]) + }) + + it('ref.total becomes env aggregate ceiling for fungible (capped at pool.total)', function () { + const envDef = { resources: [{ id: 'cpu', total: 6 }] } + const result = engine.resolveEnvironmentResources(envDef, pool) + expect(result[0].total).to.equal(6) + }) + + it('ref.total exceeding pool.total is capped at pool.total', function () { + const envDef = { resources: [{ id: 'cpu', total: 999 }] } + const result = engine.resolveEnvironmentResources(envDef, pool) + expect(result[0].total).to.equal(10) // pool.total = 10 + }) + + it('omitting ref.total inherits pool total for fungible', function () { + const envDef = { resources: [{ id: 'cpu' }] } + const result = engine.resolveEnvironmentResources(envDef, pool) + expect(result[0].total).to.equal(10) + }) + + it('ref.max is capped to resolved.total', function () { + const envDef = { resources: [{ id: 'cpu', total: 6, max: 99 }] } + const result = engine.resolveEnvironmentResources(envDef, pool) + expect(result[0].max).to.equal(6) // capped to total + }) + + it('warns when ref.max exceeds resolved.total, so the operator can fix their config', function () { + const warnSpy = sinon.spy(CORE_LOGGER, 'warn') + try { + const envDef = { + description: 'gpu-restricted', + resources: [{ id: 'cpu', total: 1, max: 10, min: 1 }] + } + const result = engine.resolveEnvironmentResources(envDef, pool) + expect(result[0].max).to.equal(1) // still clamped to total + expect( + warnSpy.calledWithMatch(sinon.match(/cpu.*max \(10\) greater than total \(1\)/)) + ).to.equal(true) + expect(warnSpy.calledWithMatch(sinon.match(/gpu-restricted/))).to.equal(true) + } finally { + warnSpy.restore() + } + }) + + it('does not warn when ref.max is within resolved.total', function () { + const warnSpy = sinon.spy(CORE_LOGGER, 'warn') + try { + const envDef = { resources: [{ id: 'cpu', total: 6, max: 6 }] } + engine.resolveEnvironmentResources(envDef, pool) + expect(warnSpy.called).to.equal(false) + } finally { + warnSpy.restore() + } + }) + + it('ref.min overrides pool min', function () { + const envDef = { resources: [{ id: 'cpu', min: 2 }] } + const result = engine.resolveEnvironmentResources(envDef, pool) + expect(result[0].min).to.equal(2) + }) + + it('ref.constraints replaces pool constraints entirely', function () { + const envDef = { + resources: [ + { + id: 'gpu0', + constraints: [ + { id: 'ram', min: 8 }, + { id: 'cpu', min: 4 } + ] + } + ] + } + const result = engine.resolveEnvironmentResources(envDef, pool) + const gpuRes = result.find((r: ComputeResource) => r.id === 'gpu0') + expect(gpuRes.constraints).to.have.length(2) + expect(gpuRes.constraints[0]).to.deep.equal({ id: 'ram', min: 8 }) + }) + + it('per-env group+floor constraint (type/perUnit) is carried through and deep-cloned', function () { + const envDef: any = { + resources: [ + { id: 'cpu', constraints: [{ type: 'gpu', min: 1, perUnit: false }] }, + { id: 'gpu0' } + ] + } + const result = engine.resolveEnvironmentResources(envDef, pool) + const cpuRes = result.find((r: ComputeResource) => r.id === 'cpu') + expect(cpuRes.constraints).to.deep.equal([{ type: 'gpu', min: 1, perUnit: false }]) + // deep-cloned: mutating the resolved constraint must not affect the envDef source + cpuRes.constraints[0].min = 99 + expect(envDef.resources[0].constraints[0].min).to.equal(1) + }) + + it('ref.constraints: [] removes all constraints for this env', function () { + const envDef = { resources: [{ id: 'gpu0', constraints: [] as any[] }] } + const result = engine.resolveEnvironmentResources(envDef, pool) + const gpuRes = result.find((r: ComputeResource) => r.id === 'gpu0') + expect(gpuRes.constraints).to.deep.equal([]) + }) + + it('omitting ref.constraints inherits pool constraints (deep-cloned)', function () { + const envDef = { resources: [{ id: 'gpu0' }] } + const result = engine.resolveEnvironmentResources(envDef, pool) + const gpuRes = result.find((r: ComputeResource) => r.id === 'gpu0') + expect(gpuRes.constraints).to.deep.equal([{ id: 'ram', min: 4 }]) + // Mutating the resolved constraints must not affect the pool + gpuRes.constraints[0].min = 99 + expect(pool.get('gpu0').constraints[0].min).to.equal(4) + }) + + it('init is deep-cloned: mutating resolved.init does not corrupt pool', function () { + const envDef = { resources: [{ id: 'gpu0' }] } + const result = engine.resolveEnvironmentResources(envDef, pool) + const gpuRes = result.find((r: ComputeResource) => r.id === 'gpu0') + gpuRes.init.deviceRequests.DeviceIDs[0] = 'mutated' + expect(pool.get('gpu0').init.deviceRequests.DeviceIDs[0]).to.equal('uuid-a') + }) + + it('unknown ref.id is skipped silently, baseline cpu/ram/disk still resolved', function () { + const envDef = { resources: [{ id: 'nonexistent' }] } + const result = engine.resolveEnvironmentResources(envDef, pool) + expect(result.map((r: ComputeResource) => r.id).sort()).to.deep.equal([ + 'cpu', + 'disk', + 'ram' + ]) + }) + + it('cpu/ram/disk are always resolved even when the config references none of them', function () { + const envDef = { resources: [] as any[] } + const result = engine.resolveEnvironmentResources(envDef, pool) + expect(result.map((r: ComputeResource) => r.id).sort()).to.deep.equal([ + 'cpu', + 'disk', + 'ram' + ]) + }) + + it('an explicit baseline override is preserved, not clobbered by the auto-filled default', function () { + const envDef = { resources: [{ id: 'ram', max: 4 }] } + const result = engine.resolveEnvironmentResources(envDef, pool) + const ramRes = result.find((r: ComputeResource) => r.id === 'ram') + expect(ramRes.max).to.equal(4) + expect(result.map((r: ComputeResource) => r.id).sort()).to.deep.equal([ + 'cpu', + 'disk', + 'ram' + ]) + }) + }) +}) + +describe('CPU affinity restricted by cpuList', () => { + let engine: any + + beforeEach(function () { + // Bypass the Docker-specific constructor but keep the prototype methods; seed the + // fields that constructor initializers would otherwise set. + engine = Object.create(C2DEngineDocker.prototype) + engine.physicalLimits = new Map() + engine.cpuAllocations = new Map() + engine.envCpuCoresMap = new Map() + engine.configuredCpuList = null + engine.getC2DConfig = sinon.stub().returns({ hash: 'cluster-hash' }) + }) + + describe('resolveConfiguredCpuList()', function () { + it('no cpu entry / cpu entry without cpuList → unrestricted (null pool)', function () { + engine.resolveConfiguredCpuList({ resources: [] }, 8) + expect(engine.configuredCpuList).to.equal(null) + engine.resolveConfiguredCpuList({ resources: [{ id: 'cpu', total: 4 }] }, 8) + expect(engine.configuredCpuList).to.equal(null) + }) + + it('valid cpuList expands to the listed core IDs', function () { + engine.resolveConfiguredCpuList({ resources: [{ id: 'cpu', cpuList: '2-5' }] }, 8) + expect(engine.configuredCpuList).to.deep.equal([2, 3, 4, 5]) + }) + + it('multi-range cpuList expands to all listed core IDs', function () { + engine.resolveConfiguredCpuList( + { resources: [{ id: 'cpu', cpuList: '0-1,4-6' }] }, + 8 + ) + expect(engine.configuredCpuList).to.deep.equal([0, 1, 4, 5, 6]) + }) + + it('mixed ranges and bare core IDs expand to all listed core IDs', function () { + engine.resolveConfiguredCpuList({ resources: [{ id: 'cpu', cpuList: '0-1,3' }] }, 8) + expect(engine.configuredCpuList).to.deep.equal([0, 1, 3]) + }) + + it('single bare core ID works on a single-cpu host', function () { + engine.resolveConfiguredCpuList({ resources: [{ id: 'cpu', cpuList: '0' }] }, 1) + expect(engine.configuredCpuList).to.deep.equal([0]) + }) + + it('core IDs the host does not have throw (fail fast at startup / config push)', function () { + expect(() => + engine.resolveConfiguredCpuList( + { resources: [{ id: 'cpu', cpuList: '0-15' }] }, + 8 + ) + ).to.throw("don't exist on this host") + expect(engine.configuredCpuList).to.equal(null) + }) + }) + + describe('allocateCpus() with a restricted pool', function () { + beforeEach(function () { + engine.envCpuCoresMap.set('env1', [32, 33, 34, 35, 36, 37, 38, 39]) + }) + + it('allocations only hand out cores from the restricted pool', function () { + expect(engine.allocateCpus('job1', 4, 'env1')).to.equal('32,33,34,35') + expect(engine.allocateCpus('job2', 2, 'env1')).to.equal('36,37') + }) + + it('released cores are reused from the restricted pool', function () { + engine.allocateCpus('job1', 4, 'env1') + engine.releaseCpus('job1') + expect(engine.allocateCpus('job2', 2, 'env1')).to.equal('32,33') + }) + + it('requests beyond the restricted pool return null instead of spilling to other cores', function () { + expect(engine.allocateCpus('job1', 9, 'env1')).to.equal(null) + }) + }) +}) + +describe('getAlgoChecksums', () => { + let findDdoStub: sinon.SinonStub + let loggerErrorSpy: sinon.SinonSpy + + beforeEach(() => { + findDdoStub = sinon.stub(FindDdoHandler.prototype, 'findAndFormatDdo') + loggerErrorSpy = sinon.spy(CORE_LOGGER, 'error') + }) + + afterEach(() => { + findDdoStub.restore() + loggerErrorSpy.restore() + }) + + it('returns empty checksums without a DDO lookup for raw-code algorithms (no documentId)', async () => { + const checksums = await getAlgoChecksums( + undefined, + undefined, + null as any, null as any ) @@ -551,3 +2336,400 @@ describe('getAlgoChecksums', () => { expect(loggerErrorSpy.called).to.equal(false) }) }) + +describe('service start/restart Docker cleanup on failure', function () { + let engine: any + let network: { id: string; inspect: sinon.SinonStub; remove: sinon.SinonStub } + + function makeContainer(startRejects: boolean) { + return { + id: 'container-1', + start: startRejects + ? sinon.stub().rejects(new Error('start failed')) + : sinon.stub().resolves(), + stop: sinon.stub().resolves(), + remove: sinon.stub().resolves() + } + } + + beforeEach(function () { + // Bypass the Docker-specific constructor but keep the prototype methods. + engine = Object.create(C2DEngineDocker.prototype) + // Constructor field initializers don't run via Object.create, so seed the CPU + // pinning maps that releaseCpus/allocateCpus (called by cleanupServiceDocker) touch, + // and the per-service lifecycle lock taken by stopService/restartService. + engine.cpuAllocations = new Map() + engine.envCpuCoresMap = new Map() + engine.serviceOpsInFlight = new Set() + engine.serviceOpPromises = new Set() + // inspect is needed by removeServiceNetwork (cleanup resolves the network by + // deterministic name/id and checks for attached containers before removing). + network = { + id: 'net-1', + inspect: sinon.stub().resolves({ Containers: {} }), + remove: sinon.stub().resolves() + } + + engine.db = { + newServiceJob: sinon.stub().resolves(), + updateServiceJob: sinon.stub().resolves(), + // lifecycle lease (fail-closed: restart/stop reject without it) + acquireServiceLock: sinon.stub().resolves(true), + releaseServiceLock: sinon.stub().resolves(undefined) + } + engine.getC2DConfig = sinon.stub().returns({ + hash: 'cluster-hash', + connection: { + serviceOnDemand: { hostPortRange: [30000, 32767], nodeHost: 'localhost' } + } + }) + // Image pull succeeds; the failure we exercise is later, at container create/start. + engine.pullImageRef = sinon.stub().resolves() + engine.buildServiceResourceConstraints = sinon + .stub() + .returns({ Memory: 0, NanoCpus: 0, DeviceRequests: [] }) + // Escrow succeeds (lock + claim) so the pipeline reaches the container phase. + engine.escrow = { + createLock: sinon.stub().resolves('0xlock'), + waitForTransaction: sinon.stub().resolves(), + claimLock: sinon.stub().resolves('0xclaim'), + cancelExpiredLock: sinon.stub().resolves('0xcancel'), + getMinLockTime: sinon.stub().returns(3600) + } + engine.keyManager = { decrypt: sinon.stub().resolves(Buffer.from('{}')) } + }) + + afterEach(() => sinon.restore()) + + async function expectRejects(promise: Promise, messagePart: string) { + let thrown: Error | null = null + try { + await promise + } catch (err: any) { + thrown = err + } + expect(thrown, 'expected the call to reject').to.not.equal(null) + expect(thrown!.message).to.contain(messagePart) + } + + // A fresh Starting job, as createServiceJob would have persisted it. + function makeStartingJob(overrides: any = {}) { + return { + serviceId: 'svc-1', + clusterHash: 'cluster-hash', + environment: 'env-1', + owner: '0xowner', + image: 'nginx', + tag: 'latest', + containerImage: 'nginx:latest', + containerId: '', + networkId: '', + status: 10, // Starting + statusText: 'Starting', + dateCreated: new Date().toISOString(), + expiresAt: Date.now() + 60000, + duration: 60, + exposedPorts: [80], + endpoints: [], + resources: [{ id: 'cpu', amount: 1 }], + payment: { + chainId: 1, + token: '0xtoken', + cost: 10, + lockTx: '', + claimTx: '', + cancelTx: '' + }, + ...overrides + } + } + + it('processServiceStart removes the network and marks Error when createContainer fails', async function () { + engine.docker = { + createNetwork: sinon.stub().resolves(network), + createContainer: sinon.stub().rejects(new Error('createContainer failed')), + getNetwork: sinon.stub().returns(network) + } + const job = makeStartingJob({ serviceId: 'svc-1' }) + // the pipeline re-reads the job under the lifecycle lock before acting + engine.db.getServiceJob = sinon.stub().resolves([job]) + + await engine.processServiceStart(job) // never throws — failures are persisted as status + + expect(network.remove.called, 'network.remove should be called').to.equal(true) + expect(job.status).to.equal(ServiceStatusNumber.Error) + // Funds were already claimed before the container step, so no refund here. + expect(engine.escrow.claimLock.calledOnce).to.equal(true) + expect(engine.escrow.cancelExpiredLock.called).to.equal(false) + }) + + it('processServiceStart removes container and network when container.start fails', async function () { + const container = makeContainer(true) + engine.docker = { + createNetwork: sinon.stub().resolves(network), + createContainer: sinon.stub().resolves(container), + getNetwork: sinon.stub().returns(network) + } + const job = makeStartingJob({ serviceId: 'svc-2' }) + engine.db.getServiceJob = sinon.stub().resolves([job]) + + await engine.processServiceStart(job) + + expect(container.remove.calledOnce, 'container.remove should be called').to.equal( + true + ) + expect(network.remove.called, 'network.remove should be called').to.equal(true) + expect(job.status).to.equal(ServiceStatusNumber.Error) + // The payment was claimed before the container step, so the Error job keeps its + // host ports reserved (restartable on the same endpoints until expiresAt): the + // allocator must refuse to hand the port out again. + expect(job.endpoints.length).to.be.greaterThan(0) + const reservedPort = job.endpoints[0].hostPort + try { + await expectRejects( + allocateHostPort(reservedPort, reservedPort), + 'No free host port' + ) + } finally { + releaseHostPort(reservedPort) // don't leak the reservation into other tests + } + }) + + it('processServiceStart refunds (cancelLock) and marks PullImageFailed when the image pull fails', async function () { + engine.pullImageRef = sinon.stub().rejects(new Error('pull failed')) + engine.docker = { + createNetwork: sinon.stub().resolves(network), + createContainer: sinon.stub().resolves(makeContainer(false)), + getNetwork: sinon.stub().returns(network) + } + const job = makeStartingJob({ serviceId: 'svc-img' }) + engine.db.getServiceJob = sinon.stub().resolves([job]) + + await engine.processServiceStart(job) + + expect(engine.escrow.claimLock.called, 'must not claim when image failed').to.equal( + false + ) + expect(engine.escrow.cancelExpiredLock.calledOnce, 'must refund the lock').to.equal( + true + ) + expect(job.status).to.equal(ServiceStatusNumber.PullImageFailed) + expect(engine.docker.createContainer.called, 'must not create a container').to.equal( + false + ) + }) + + it('processServiceStart marks Error and skips the image when createLock fails', async function () { + engine.escrow.createLock = sinon.stub().resolves(null) + engine.docker = { + createNetwork: sinon.stub(), + createContainer: sinon.stub(), + getNetwork: sinon.stub().returns(network) + } + const job = makeStartingJob({ serviceId: 'svc-lock' }) + engine.db.getServiceJob = sinon.stub().resolves([job]) + + await engine.processServiceStart(job) + + expect(engine.pullImageRef.called, 'must not pull when lock failed').to.equal(false) + expect(engine.escrow.claimLock.called).to.equal(false) + expect(job.status).to.equal(ServiceStatusNumber.Error) + }) + + it('processServiceStart orphan recovery: cancels an unclaimed lock and marks Error', async function () { + engine.docker = { + createNetwork: sinon.stub(), + createContainer: sinon.stub(), + getNetwork: sinon.stub().returns(network) + } + // Resuming a job left in Locking from a previous process, with a lock but no claim. + const job = makeStartingJob({ + serviceId: 'svc-orphan', + status: ServiceStatusNumber.Locking, + statusText: 'Locking', + payment: { + chainId: 1, + token: '0xtoken', + cost: 10, + lockTx: '0xlock', + claimTx: '', + cancelTx: '' + } + }) + engine.db.getServiceJob = sinon.stub().resolves([job]) + + await engine.processServiceStart(job) + + expect( + engine.escrow.cancelExpiredLock.calledOnce, + 'orphan lock must be refunded' + ).to.equal(true) + expect(engine.escrow.createLock.called, 'must not re-lock an orphan').to.equal(false) + expect(job.status).to.equal(ServiceStatusNumber.Error) + }) + + it('restartService removes the newly created network when createContainer fails', async function () { + const existingJob = { + serviceId: 'svc-3', + clusterHash: 'cluster-hash', + environment: 'env-1', + owner: '0xowner', + image: 'nginx', + tag: 'latest', + containerImage: 'nginx:latest', + containerId: '', // empty → skip pre-teardown + networkId: '', + status: 40, // Running + statusText: 'Running', + dateCreated: new Date().toISOString(), + expiresAt: Date.now() + 60000, + duration: 60, + exposedPorts: [80], + endpoints: [{ containerPort: 80, hostPort: 30001, url: 'http://localhost:30001' }], + resources: [{ id: 'cpu', amount: 1 }], + payment: { chainId: 1, token: '0xtoken', claimTx: '0xclaim' } + } + engine.db.getServiceJob = sinon.stub().resolves([existingJob]) + engine.docker = { + createNetwork: sinon.stub().resolves(network), + createContainer: sinon.stub().rejects(new Error('createContainer failed')), + getNetwork: sinon.stub().returns(network) + } + + // restart is async: the call returns the Restarting snapshot; the failure surfaces + // on the persisted job status once the background op settles. + const snapshot = await engine.restartService('svc-3', '0xowner', undefined) + expect(snapshot.status).to.equal(ServiceStatusNumber.Restarting) + await Promise.allSettled([...engine.serviceOpPromises]) + + expect(network.remove.called, 'network.remove should be called').to.equal(true) + expect(existingJob.status).to.equal(ServiceStatusNumber.Error) + expect(existingJob.statusText).to.contain('createContainer failed') + // the Error outcome must be PERSISTED — status polls read the DB, not memory + expect( + engine.db.updateServiceJob.calledWith( + sinon.match({ serviceId: 'svc-3', status: ServiceStatusNumber.Error }) + ) + ).to.equal(true) + }) + + function makeRunningJobWithCmd(overrides: any = {}) { + return { + serviceId: 'svc-cmd', + clusterHash: 'cluster-hash', + environment: 'env-1', + owner: '0xowner', + image: 'nginx', + tag: 'latest', + containerImage: 'nginx:latest', + containerId: '', + networkId: '', + status: 40, // Running + statusText: 'Running', + dateCreated: new Date().toISOString(), + expiresAt: Date.now() + 60000, + duration: 60, + exposedPorts: [80], + endpoints: [{ containerPort: 80, hostPort: 30001, url: 'http://localhost:30001' }], + resources: [{ id: 'cpu', amount: 1 }], + payment: { chainId: 1, token: '0xtoken', claimTx: '0xclaim' }, + dockerCmd: ['old', 'cmd'], + dockerEntrypoint: ['/old-entrypoint'], + ...overrides + } + } + + it('restartService overrides dockerCmd/dockerEntrypoint when new ones are supplied', async function () { + const existingJob = makeRunningJobWithCmd() + engine.db.getServiceJob = sinon.stub().resolves([existingJob]) + const container = makeContainer(false) + engine.docker = { + createNetwork: sinon.stub().resolves(network), + createContainer: sinon.stub().resolves(container), + getNetwork: sinon.stub().returns(network) + } + + await engine.restartService( + 'svc-cmd', + '0xowner', + undefined, + ['new', 'cmd'], + ['/new-entrypoint'] + ) + await Promise.allSettled([...engine.serviceOpPromises]) + + const createArgs = engine.docker.createContainer.firstCall.args[0] + expect(createArgs.Cmd).to.deep.equal(['new', 'cmd']) + expect(createArgs.Entrypoint).to.deep.equal(['/new-entrypoint']) + expect(existingJob.dockerCmd).to.deep.equal(['new', 'cmd']) + expect(existingJob.dockerEntrypoint).to.deep.equal(['/new-entrypoint']) + // the override must be PERSISTED so future restarts reuse it + expect( + engine.db.updateServiceJob.calledWith( + sinon.match({ + status: ServiceStatusNumber.Running, + dockerCmd: ['new', 'cmd'], + dockerEntrypoint: ['/new-entrypoint'] + }) + ) + ).to.equal(true) + }) + + it('restartService reuses the stored dockerCmd/dockerEntrypoint when none are supplied', async function () { + const existingJob = makeRunningJobWithCmd() + engine.db.getServiceJob = sinon.stub().resolves([existingJob]) + const container = makeContainer(false) + engine.docker = { + createNetwork: sinon.stub().resolves(network), + createContainer: sinon.stub().resolves(container), + getNetwork: sinon.stub().returns(network) + } + + await engine.restartService('svc-cmd', '0xowner', undefined) + await Promise.allSettled([...engine.serviceOpPromises]) + + const createArgs = engine.docker.createContainer.firstCall.args[0] + expect(createArgs.Cmd).to.deep.equal(['old', 'cmd']) + expect(createArgs.Entrypoint).to.deep.equal(['/old-entrypoint']) + expect(existingJob.dockerCmd).to.deep.equal(['old', 'cmd']) + expect(existingJob.dockerEntrypoint).to.deep.equal(['/old-entrypoint']) + expect( + engine.db.updateServiceJob.calledWith( + sinon.match({ + status: ServiceStatusNumber.Running, + dockerCmd: ['old', 'cmd'], + dockerEntrypoint: ['/old-entrypoint'] + }) + ) + ).to.equal(true) + }) + + it('restartService clears dockerCmd/dockerEntrypoint when explicitly given an empty array', async function () { + const existingJob = makeRunningJobWithCmd() + engine.db.getServiceJob = sinon.stub().resolves([existingJob]) + const container = makeContainer(false) + engine.docker = { + createNetwork: sinon.stub().resolves(network), + createContainer: sinon.stub().resolves(container), + getNetwork: sinon.stub().returns(network) + } + + await engine.restartService('svc-cmd', '0xowner', undefined, [], []) + await Promise.allSettled([...engine.serviceOpPromises]) + + const createArgs = engine.docker.createContainer.firstCall.args[0] + expect(createArgs.Cmd).to.equal(undefined) + expect(createArgs.Entrypoint).to.equal(undefined) + expect(existingJob.dockerCmd).to.deep.equal([]) + expect(existingJob.dockerEntrypoint).to.deep.equal([]) + expect( + engine.db.updateServiceJob.calledWith( + sinon.match({ + status: ServiceStatusNumber.Running, + dockerCmd: [], + dockerEntrypoint: [] + }) + ) + ).to.equal(true) + }) +}) diff --git a/src/test/unit/http.test.ts b/src/test/unit/http.test.ts new file mode 100644 index 000000000..739660172 --- /dev/null +++ b/src/test/unit/http.test.ts @@ -0,0 +1,156 @@ +import { expect } from 'chai' +import http, { Server } from 'http' +import { AddressInfo } from 'net' +import { Readable } from 'stream' +import { gzipSync } from 'zlib' +import { fetchStream, fetchHeadersTimeout, headersToObject } from '../../utils/http.js' + +// spin up a throwaway local http server on an ephemeral port +function startServer(handler: http.RequestListener): Promise { + return new Promise((resolve) => { + const server = http.createServer(handler) + server.listen(0, () => resolve(server)) + }) +} + +function baseUrl(server: Server): string { + const { port } = server.address() as AddressInfo + return `http://127.0.0.1:${port}` +} + +function stopServer(server: Server): Promise { + return new Promise((resolve) => { + // drop any hung/keep-alive sockets so close() actually resolves + server.closeAllConnections() + server.close(() => resolve()) + }) +} + +async function collect(stream: Readable): Promise { + const chunks: Buffer[] = [] + for await (const chunk of stream) { + chunks.push(Buffer.from(chunk)) + } + return Buffer.concat(chunks).toString('utf8') +} + +describe('utils/http', () => { + describe('headersToObject', () => { + it('produces a plain object with lowercase keys (Object.entries-friendly)', () => { + const headers = new Headers({ + 'X-Foo': 'bar', + 'Content-Type': 'application/json' + }) + const obj = headersToObject(headers) + expect(obj).to.deep.equal({ + 'x-foo': 'bar', + 'content-type': 'application/json' + }) + // consumers (downloadHandler) iterate with Object.entries — must work + expect(Object.entries(obj)).to.have.length(2) + }) + }) + + describe('fetchStream', () => { + let server: Server + afterEach(async () => { + if (server) await stopServer(server) + }) + + it('returns a working Node Readable and a plain headers object on 2xx', async () => { + server = await startServer((req, res) => { + res.writeHead(200, { 'content-type': 'text/plain', 'x-custom': 'yes' }) + res.end('hello world') + }) + const { httpStatus, stream, headers } = await fetchStream(baseUrl(server)) + expect(httpStatus).to.equal(200) + expect(stream).to.be.instanceOf(Readable) + expect(headers['content-type']).to.equal('text/plain') + expect(headers['x-custom']).to.equal('yes') + const body = await collect(stream) + expect(body).to.equal('hello world') + }) + + it('decodes a gzip body and strips the stale content-encoding/length headers', async () => { + const payload = 'the quick brown fox '.repeat(50) + const gz = gzipSync(Buffer.from(payload)) + server = await startServer((req, res) => { + res.writeHead(200, { + 'content-type': 'text/plain', + 'content-encoding': 'gzip', + 'content-length': String(gz.length) + }) + res.end(gz) + }) + const { stream, headers } = await fetchStream(baseUrl(server)) + // undici already decoded the body — the re-served headers must not still + // advertise gzip, or a downstream client double-decompresses and fails + expect(headers).to.not.have.property('content-encoding') + expect(headers).to.not.have.property('content-length') + const body = await collect(stream) + expect(body).to.equal(payload) + }) + + it('throws on a non-2xx status (axios throw-on-non-2xx contract)', async () => { + server = await startServer((req, res) => { + res.writeHead(404, { 'content-type': 'text/plain' }) + res.end('nope') + }) + let threw = false + try { + await fetchStream(baseUrl(server)) + } catch (err: any) { + threw = true + expect(err.message).to.contain('404') + } + expect(threw).to.equal(true) + }) + }) + + describe('fetchHeadersTimeout', () => { + let server: Server + afterEach(async () => { + if (server) await stopServer(server) + }) + + it('aborts when headers never arrive within the timeout', async () => { + // handler intentionally never responds + server = await startServer(() => {}) + let threw = false + try { + await fetchHeadersTimeout(baseUrl(server), { method: 'GET' }, 150) + } catch (err: any) { + threw = true + // undici surfaces the abort as an AbortError / "aborted" + expect(String(err.name + err.message).toLowerCase()).to.match( + /abort|headers timeout/ + ) + } + expect(threw).to.equal(true) + }) + + it('does NOT abort a slow body once headers have arrived', async () => { + // headers flush immediately, then the body drips out over ~250ms, + // which is well past the 100ms headers timeout — the full body must + // still arrive because the timer is cleared once headers resolve. + server = await startServer((req, res) => { + res.writeHead(200, { 'content-type': 'text/plain' }) + res.flushHeaders() + let i = 0 + const iv = setInterval(() => { + if (i >= 5) { + clearInterval(iv) + res.end() + return + } + res.write(`chunk${i}`) + i++ + }, 50) + }) + const response = await fetchHeadersTimeout(baseUrl(server), { method: 'GET' }, 100) + expect(response.status).to.equal(200) + const body = await collect(Readable.fromWeb(response.body as any)) + expect(body).to.equal('chunk0chunk1chunk2chunk3chunk4') + }) + }) +}) diff --git a/src/test/unit/service/serviceHandlers.test.ts b/src/test/unit/service/serviceHandlers.test.ts new file mode 100644 index 000000000..0866c9b51 --- /dev/null +++ b/src/test/unit/service/serviceHandlers.test.ts @@ -0,0 +1,926 @@ +import { assert, expect } from 'chai' +import { Readable } from 'stream' +import sinon from 'sinon' +import { streamToObject } from '../../../utils/util.js' +import { PROTOCOL_COMMANDS } from '../../../utils/constants.js' +import { ServiceStatusNumber, ServiceJob } from '../../../@types/C2D/ServiceOnDemand.js' +import { ServiceGetTemplatesHandler } from '../../../components/core/service/getTemplates.js' +import { ServiceGetStatusHandler } from '../../../components/core/service/getStatus.js' +import { GetServicesHandler } from '../../../components/core/service/getServices.js' +import { ServiceStartHandler } from '../../../components/core/service/startService.js' +import { ServiceStopHandler } from '../../../components/core/service/stopService.js' +import { ServiceExtendHandler } from '../../../components/core/service/extendService.js' +import { ServiceRestartHandler } from '../../../components/core/service/restartService.js' +import { ServiceGetStreamableLogsHandler } from '../../../components/core/service/getStreamableLogs.js' + +const OWNER = '0x0000000000000000000000000000000000000abc' + +function makeJob(overrides: Partial = {}): ServiceJob { + return { + serviceId: 'svc-1', + clusterHash: 'hash-1', + environment: 'env-1', + owner: OWNER, + image: 'img', + tag: 'latest', + containerImage: 'img:latest', + containerId: 'c1', + networkId: 'n1', + status: ServiceStatusNumber.Running, + statusText: 'Running', + dateCreated: new Date(0).toISOString(), + expiresAt: Date.now() + 3600_000, + duration: 3600, + exposedPorts: [8888], + endpoints: [{ containerPort: 8888, hostPort: 31000, url: 'http://localhost:31000' }], + userData: 'ENCRYPTED', + resources: [{ id: 'cpu', amount: 2 }], + payment: { + chainId: 8996, + token: '0xtoken', + lockTx: '0xl', + claimTx: '0xc', + cancelTx: '', + cost: 5 + }, + ...overrides + } +} + +const TEMPLATE = { + id: 'jupyter-cpu', + image: 'quay.io/jupyter/datascience-notebook', + tag: 'latest', + exposedPorts: [8888] +} + +interface FakeOpts { + serviceEnabled?: boolean + serviceJobInDb?: ServiceJob | null + cost?: number | null + envId?: string + streamableLogs?: Readable | null +} + +function buildFakes(opts: FakeOpts = {}) { + const env: any = { + id: opts.envId ?? 'env-1', + features: { + computeJobs: true, + services: opts.serviceEnabled !== false + }, + resources: [{ id: 'cpu', kind: 'fungible', total: 8, min: 1, max: 8 }] + } + + const escrow = { + createLock: sinon.stub().resolves('0xlock'), + claimLock: sinon.stub().resolves('0xclaim'), + cancelExpiredLock: sinon.stub().resolves('0xcancel'), + waitForTransaction: sinon.stub().resolves(undefined), + getMinLockTime: sinon.stub().returns(3600), + // the SERVICE_START handler's fail-fast funds pre-check — plentiful by default + getUserAvailableFunds: sinon.stub().resolves(1_000_000n), + getPaymentAmountInWei: sinon.stub().resolves(10n) + } + + const engine: any = { + db: { + getServiceJob: sinon + .stub() + .resolves( + opts.serviceJobInDb === undefined + ? [] + : opts.serviceJobInDb + ? [opts.serviceJobInDb] + : [] + ), + updateServiceJob: sinon.stub().resolves(1), + // SERVICE_LIST: the engine's cluster-scoped resource-holding set + getRunningServiceJobs: sinon + .stub() + .resolves(opts.serviceJobInDb ? [opts.serviceJobInDb] : []) + }, + escrow, + getComputeEnvironments: sinon.stub().resolves([env]), + // hash must match makeJob's clusterHash: handlers resolve the owning engine by it + getC2DConfig: sinon.stub().returns({ + hash: 'hash-1', + connection: { serviceOnDemand: { maxDurationSeconds: 86400 } } + }), + calculateResourcesCost: sinon + .stub() + .returns(opts.cost === undefined ? 10 : opts.cost), + checkAndFillMissingResources: sinon + .stub() + .callsFake((r: any) => Promise.resolve(r ?? [])), + checkIfResourcesAreAvailable: sinon.stub().resolves(undefined), + getEnvPricesForToken: sinon.stub().returns([{ id: 'cpu', price: 1 }]), + // Async start: the handler only persists a Starting record and returns; the escrow + + // image + container work is done later by processServiceStart (driven by the cron). + createServiceJob: sinon.stub().callsFake(() => + Promise.resolve( + makeJob({ + status: ServiceStatusNumber.Starting, + statusText: 'Starting', + containerId: '', + networkId: '', + endpoints: [] + }) + ) + ), + stopService: sinon + .stub() + .callsFake(() => Promise.resolve(makeJob({ status: ServiceStatusNumber.Stopped }))), + restartService: sinon + .stub() + .callsFake(() => + Promise.resolve( + makeJob({ containerId: 'c2', status: ServiceStatusNumber.Running }) + ) + ), + getServiceStatus: sinon + .stub() + .resolves(opts.serviceJobInDb ? [opts.serviceJobInDb] : []), + getServiceStreamableLogs: sinon + .stub() + .resolves( + opts.streamableLogs === undefined + ? Readable.from(['hello logs']) + : opts.streamableLogs + ), + // handlers (SERVICE_EXTEND) serialize read-mutate-write flows through this; the + // fake just runs the callback (the real engine wraps it in the lifecycle lock) + runExclusive: sinon.stub().callsFake((_id: string, fn: () => Promise) => fn()) + } + + const engines: any = { + getAllEngines: () => [engine], + getC2DByEnvId: sinon.stub().resolves(engine), + getC2DByHash: sinon.stub().resolves(engine), + fetchServiceTemplates: sinon.stub().resolves([{ ...TEMPLATE }]) + } + + const node: any = { + getRequestMap: () => new Map(), + getConfig: (): any => ({ + rateLimit: undefined as number | undefined, + serviceTemplatesPath: undefined as string | undefined + }), + getC2DEngines: () => engines, + getKeyManager: () => ({ + decrypt: (d: Uint8Array) => Promise.resolve(Buffer.from(d)) + }), + getAuth: () => ({ + validateAuthenticationOrToken: () => Promise.resolve({ valid: true }) + }) + } + + return { node, engine, engines, escrow, env } +} + +function body(response: any): Promise { + return streamToObject(response.stream as Readable) +} + +describe('Service handlers', () => { + afterEach(() => sinon.restore()) + + describe('ServiceGetTemplatesHandler', () => { + it('returns templates from fetchServiceTemplates (200)', async () => { + const { node, engines } = buildFakes() + const res = await new ServiceGetTemplatesHandler(node).handle({ + command: PROTOCOL_COMMANDS.SERVICE_GET_TEMPLATES + } as any) + expect(res.status.httpStatus).to.equal(200) + const templates = await body(res) + expect(templates).to.have.length(1) + expect(engines.fetchServiceTemplates.calledOnce).to.equal(true) + }) + }) + + describe('ServiceGetStatusHandler', () => { + it('400 when consumerAddress is missing', async () => { + const { node } = buildFakes() + const res = await new ServiceGetStatusHandler(node).handle({ + command: PROTOCOL_COMMANDS.SERVICE_GET_STATUS, + serviceId: 'svc-1' + } as any) + expect(res.status.httpStatus).to.equal(400) + }) + + it('401 when the signature/token is invalid', async () => { + const { node } = buildFakes({ serviceJobInDb: makeJob() }) + node.getAuth = () => ({ + validateAuthenticationOrToken: () => + Promise.resolve({ valid: false, error: 'bad signature' }) + }) + const res = await new ServiceGetStatusHandler(node).handle({ + command: PROTOCOL_COMMANDS.SERVICE_GET_STATUS, + consumerAddress: OWNER, + nonce: '1', + signature: '0xbad', + serviceId: 'svc-1' + } as any) + expect(res.status.httpStatus).to.equal(401) + }) + + it('returns jobs by serviceId with userData stripped (authenticated)', async () => { + const { node } = buildFakes({ serviceJobInDb: makeJob() }) + const res = await new ServiceGetStatusHandler(node).handle({ + command: PROTOCOL_COMMANDS.SERVICE_GET_STATUS, + consumerAddress: OWNER, + nonce: '1', + signature: '0xsig', + serviceId: 'svc-1' + } as any) + expect(res.status.httpStatus).to.equal(200) + const jobs = await body(res) + expect(jobs).to.have.length(1) + expect(jobs[0]).to.not.have.property('userData') + expect(jobs[0].serviceId).to.equal('svc-1') + }) + }) + + describe('ServiceStopHandler', () => { + const baseTask = { + command: PROTOCOL_COMMANDS.SERVICE_STOP, + consumerAddress: OWNER, + nonce: '1', + signature: '0xsig', + serviceId: 'svc-1' + } + + it('400 when service not found', async () => { + const { node } = buildFakes({ serviceJobInDb: null }) + const res = await new ServiceStopHandler(node).handle({ ...baseTask } as any) + expect(res.status.httpStatus).to.equal(400) + }) + + it('401 when caller is not the owner', async () => { + const { node } = buildFakes({ serviceJobInDb: makeJob({ owner: '0xsomeoneelse' }) }) + const res = await new ServiceStopHandler(node).handle({ ...baseTask } as any) + expect(res.status.httpStatus).to.equal(401) + }) + + it('200 and calls engine.stopService on success', async () => { + const { node, engine } = buildFakes({ serviceJobInDb: makeJob() }) + const res = await new ServiceStopHandler(node).handle({ ...baseTask } as any) + expect(res.status.httpStatus).to.equal(200) + expect(engine.stopService.calledOnce).to.equal(true) + const jobs = await body(res) + expect(jobs[0].status).to.equal(ServiceStatusNumber.Stopped) + expect(jobs[0]).to.not.have.property('userData') + }) + }) + + describe('ServiceRestartHandler', () => { + const baseTask = { + command: PROTOCOL_COMMANDS.SERVICE_RESTART, + consumerAddress: OWNER, + nonce: '1', + signature: '0xsig', + serviceId: 'svc-1' + } + + it('400 when not found', async () => { + const { node } = buildFakes({ serviceJobInDb: null }) + const res = await new ServiceRestartHandler(node).handle({ ...baseTask } as any) + expect(res.status.httpStatus).to.equal(400) + }) + + it('401 when not owner', async () => { + const { node } = buildFakes({ serviceJobInDb: makeJob({ owner: '0xother' }) }) + const res = await new ServiceRestartHandler(node).handle({ ...baseTask } as any) + expect(res.status.httpStatus).to.equal(401) + }) + + it('400 when restarting an expired service', async () => { + const { node } = buildFakes({ + serviceJobInDb: makeJob({ status: ServiceStatusNumber.Expired }) + }) + const res = await new ServiceRestartHandler(node).handle({ ...baseTask } as any) + expect(res.status.httpStatus).to.equal(400) + }) + + it('200 and calls engine.restartService on success', async () => { + const { node, engine } = buildFakes({ serviceJobInDb: makeJob() }) + const res = await new ServiceRestartHandler(node).handle({ ...baseTask } as any) + expect(res.status.httpStatus).to.equal(200) + expect(engine.restartService.calledOnce).to.equal(true) + const jobs = await body(res) + expect(jobs[0].containerId).to.equal('c2') + expect(jobs[0]).to.not.have.property('userData') + }) + + it('forwards dockerCmd and dockerEntrypoint overrides to engine.restartService', async () => { + const { node, engine } = buildFakes({ serviceJobInDb: makeJob() }) + await new ServiceRestartHandler(node).handle({ + ...baseTask, + dockerCmd: ['python', 'new_script.py'], + dockerEntrypoint: ['/bin/new-entrypoint'] + } as any) + expect(engine.restartService.calledOnce).to.equal(true) + const callArgs = engine.restartService.firstCall.args + expect(callArgs[0]).to.equal('svc-1') + expect(callArgs[1]).to.equal(OWNER) + expect(callArgs[3]).to.deep.equal(['python', 'new_script.py']) + expect(callArgs[4]).to.deep.equal(['/bin/new-entrypoint']) + }) + + it('forwards undefined dockerCmd/dockerEntrypoint when not supplied, so the engine reuses stored values', async () => { + const { node, engine } = buildFakes({ serviceJobInDb: makeJob() }) + await new ServiceRestartHandler(node).handle({ ...baseTask } as any) + const callArgs = engine.restartService.firstCall.args + expect(callArgs[3]).to.equal(undefined) + expect(callArgs[4]).to.equal(undefined) + }) + }) + + describe('ServiceExtendHandler', () => { + const baseTask = { + command: PROTOCOL_COMMANDS.SERVICE_EXTEND, + consumerAddress: OWNER, + nonce: '1', + signature: '0xsig', + serviceId: 'svc-1', + additionalDuration: 3600, + payment: { chainId: 8996, token: '0xtoken' } + } + + it('400 when not found', async () => { + const { node } = buildFakes({ serviceJobInDb: null }) + const res = await new ServiceExtendHandler(node).handle({ ...baseTask } as any) + expect(res.status.httpStatus).to.equal(400) + }) + + it('400 when additionalDuration is not strictly positive', async () => { + const { node } = buildFakes({ serviceJobInDb: makeJob() }) + for (const additionalDuration of [0, -1, -3600]) { + const res = await new ServiceExtendHandler(node).handle({ + ...baseTask, + additionalDuration + } as any) + expect( + res.status.httpStatus, + `additionalDuration=${additionalDuration}` + ).to.equal(400) + } + }) + + it('401 when not owner', async () => { + const { node } = buildFakes({ serviceJobInDb: makeJob({ owner: '0xother' }) }) + const res = await new ServiceExtendHandler(node).handle({ ...baseTask } as any) + expect(res.status.httpStatus).to.equal(401) + }) + + it('400 when service is Stopped (bad state)', async () => { + const { node } = buildFakes({ + serviceJobInDb: makeJob({ status: ServiceStatusNumber.Stopped }) + }) + const res = await new ServiceExtendHandler(node).handle({ ...baseTask } as any) + expect(res.status.httpStatus).to.equal(400) + }) + + it('400 when extension exceeds maxDurationSeconds', async () => { + const { node } = buildFakes({ + serviceJobInDb: makeJob({ expiresAt: Date.now() + 86000 * 1000 }) + }) + const res = await new ServiceExtendHandler(node).handle({ + ...baseTask, + additionalDuration: 10000 + } as any) + expect(res.status.httpStatus).to.equal(400) + }) + + it('402 when escrow lock fails', async () => { + const { node, escrow } = buildFakes({ serviceJobInDb: makeJob() }) + escrow.createLock.resolves(null) + const res = await new ServiceExtendHandler(node).handle({ ...baseTask } as any) + expect(res.status.httpStatus).to.equal(402) + }) + + it('402 and cancels lock when claim fails', async () => { + const { node, escrow } = buildFakes({ serviceJobInDb: makeJob() }) + escrow.claimLock.resolves(null) + const res = await new ServiceExtendHandler(node).handle({ ...baseTask } as any) + expect(res.status.httpStatus).to.equal(402) + expect(escrow.cancelExpiredLock.calledOnce).to.equal(true) + }) + + it('200, advances expiresAt and records an extendPayment', async () => { + const job = makeJob() + const before = job.expiresAt + const { node, engine, escrow } = buildFakes({ serviceJobInDb: job }) + const res = await new ServiceExtendHandler(node).handle({ ...baseTask } as any) + expect(res.status.httpStatus).to.equal(200) + expect(escrow.createLock.calledOnce).to.equal(true) + expect(escrow.claimLock.calledOnce).to.equal(true) + // two writes: the durable intent (before claim) + the finalized extension + expect(engine.db.updateServiceJob.calledTwice).to.equal(true) + const out = await body(res) + expect(out[0].expiresAt).to.equal(before + 3600 * 1000) + expect(out[0].extendPayments).to.have.length(1) + expect(out[0].extendPayments[0].claimTx).to.equal('0xclaim') + expect(out[0]).to.not.have.property('userData') + }) + + it('auto-refunds an unresolved extension intent from a previous crash, then proceeds', async () => { + const job = makeJob({ + extendPayments: [ + // lockTx set, neither claimTx nor cancelTx — a crash between claim and write + { + chainId: 8996, + token: '0xtoken', + lockTx: '0xoldlock', + claimTx: '', + cancelTx: '', + cost: 5 + } + ] + }) + const { node, escrow } = buildFakes({ serviceJobInDb: job }) + const res = await new ServiceExtendHandler(node).handle({ ...baseTask } as any) + expect(res.status.httpStatus).to.equal(200) + // the stale lock was refunded before charging again + expect(escrow.cancelExpiredLock.calledOnce).to.equal(true) + const out = await body(res) + expect(out[0].extendPayments).to.have.length(2) + expect(out[0].extendPayments[0].cancelTx).to.equal('0xcancel') + expect(out[0].extendPayments[1].claimTx).to.equal('0xclaim') + }) + + it('402 and refunds the lock when the intent cannot be persisted (no unrecorded charge)', async () => { + const { node, escrow, engine } = buildFakes({ serviceJobInDb: makeJob() }) + // first write in the flow is the durable intent — make it fail + engine.db.updateServiceJob.onFirstCall().rejects(new Error('db down')) + const res = await new ServiceExtendHandler(node).handle({ ...baseTask } as any) + expect(res.status.httpStatus).to.equal(402) + expect(String(res.status.error)).to.contain('refunded') + // the mined lock was compensated, and the claim was never attempted + expect(escrow.cancelExpiredLock.calledOnce).to.equal(true) + expect(escrow.claimLock.called).to.equal(false) + }) + + it('409 when the intent cannot be persisted AND the lock refund fails (stranded funds)', async () => { + const { node, escrow, engine } = buildFakes({ serviceJobInDb: makeJob() }) + engine.db.updateServiceJob.onFirstCall().rejects(new Error('db down')) + escrow.cancelExpiredLock.rejects(new Error('rpc down')) + const res = await new ServiceExtendHandler(node).handle({ ...baseTask } as any) + expect(res.status.httpStatus).to.equal(409) + expect(String(res.status.error)).to.contain('could not be refunded') + expect(escrow.claimLock.called).to.equal(false) + }) + + it('409 when an unresolved extension intent cannot be refunded (no double charge)', async () => { + const job = makeJob({ + extendPayments: [ + { + chainId: 8996, + token: '0xtoken', + lockTx: '0xoldlock', + claimTx: '', + cancelTx: '', + cost: 5 + } + ] + }) + const { node, escrow } = buildFakes({ serviceJobInDb: job }) + escrow.cancelExpiredLock.rejects(new Error('lock already claimed')) + const res = await new ServiceExtendHandler(node).handle({ ...baseTask } as any) + expect(res.status.httpStatus).to.equal(409) + // no new charge may be attempted while the old intent is unresolved + expect(escrow.createLock.called).to.equal(false) + expect(escrow.claimLock.called).to.equal(false) + }) + }) + + describe('GetServicesHandler (SERVICE_LIST)', () => { + const baseTask = { + command: PROTOCOL_COMMANDS.SERVICE_LIST, + consumerAddress: OWNER, + nonce: '1', + signature: '0xsig' + } + + it('400 when consumerAddress is missing', async () => { + const { node } = buildFakes() + const { consumerAddress, ...noAddress } = baseTask + const res = await new GetServicesHandler(node).handle({ ...noAddress } as any) + expect(res.status.httpStatus).to.equal(400) + }) + + it('400 on an unknown status number or an unparseable fromTimestamp', async () => { + const { node } = buildFakes() + const badStatus = await new GetServicesHandler(node).handle({ + ...baseTask, + status: 1234 + } as any) + expect(badStatus.status.httpStatus).to.equal(400) + const badTs = await new GetServicesHandler(node).handle({ + ...baseTask, + fromTimestamp: 'not-a-date' + } as any) + expect(badTs.status.httpStatus).to.equal(400) + }) + + it('200 default: the node-wide resource-holding set (any owner), listing-sanitized', async () => { + // a service owned by SOMEONE ELSE must be listed too — SERVICE_LIST is not + // owner-scoped like SERVICE_GET_STATUS + const job = makeJob({ + owner: '0xsomeoneelse', + status: ServiceStatusNumber.Stopped, + dockerCmd: ['secret', '--key=abc'], + dockerEntrypoint: ['/entry.sh'], + dockerfile: 'FROM x\nENV SECRET=1' + }) + const { node, engine } = buildFakes({ serviceJobInDb: job }) + const res = await new GetServicesHandler(node).handle({ ...baseTask } as any) + expect(res.status.httpStatus).to.equal(200) + // the engine is queried for ITS cluster's resource-holding set + expect(engine.db.getRunningServiceJobs.calledWith('hash-1')).to.equal(true) + const out = await body(res) + expect(out).to.have.length(1) + expect(out[0].serviceId).to.equal(job.serviceId) + expect(out[0].owner).to.equal('0xsomeoneelse') + expect(out[0].status).to.equal(ServiceStatusNumber.Stopped) + // listing-grade sanitization: no env blob, no CMD/ENTRYPOINT, no Dockerfile + expect(out[0]).to.not.have.property('userData') + expect(out[0]).to.not.have.property('dockerCmd') + expect(out[0]).to.not.have.property('dockerEntrypoint') + expect(out[0]).to.not.have.property('dockerfile') + // identity/resource fields survive + expect(out[0].resources).to.deep.equal(job.resources) + expect(out[0].endpoints).to.deep.equal(job.endpoints) + }) + + it('status filter: returns only that status (incl. Expired, outside the resource set)', async () => { + const expired = makeJob({ status: ServiceStatusNumber.Expired }) + const { node, engine } = buildFakes({ serviceJobInDb: expired }) + const res = await new GetServicesHandler(node).handle({ + ...baseTask, + status: ServiceStatusNumber.Expired + } as any) + expect(res.status.httpStatus).to.equal(200) + // status-explicit listing reads ALL jobs, not the resource-holding query + expect(engine.db.getRunningServiceJobs.called).to.equal(false) + const out = await body(res) + expect(out).to.have.length(1) + expect(out[0].status).to.equal(ServiceStatusNumber.Expired) + + // a status nothing matches → empty list + const none = await new GetServicesHandler(node).handle({ + ...baseTask, + status: ServiceStatusNumber.Running + } as any) + expect(await body(none)).to.deep.equal([]) + }) + + it('includeAllStatuses: returns every status; fromTimestamp narrows by creation date', async () => { + const job = makeJob({ + status: ServiceStatusNumber.Expired, + dateCreated: new Date('2026-01-15T00:00:00Z').toISOString() + }) + const { node } = buildFakes({ serviceJobInDb: job }) + const all = await new GetServicesHandler(node).handle({ + ...baseTask, + includeAllStatuses: true + } as any) + expect(await body(all)).to.have.length(1) + + // created before the cutoff → filtered out (ISO form) + const afterIso = await new GetServicesHandler(node).handle({ + ...baseTask, + includeAllStatuses: true, + fromTimestamp: '2026-02-01T00:00:00Z' + } as any) + expect(await body(afterIso)).to.deep.equal([]) + + // created after the cutoff → kept (Unix-seconds form) + const beforeUnix = await new GetServicesHandler(node).handle({ + ...baseTask, + includeAllStatuses: true, + fromTimestamp: String( + Math.floor(new Date('2026-01-01T00:00:00Z').getTime() / 1000) + ) + } as any) + expect(await body(beforeUnix)).to.have.length(1) + }) + + it('200 with an empty list when nothing holds resources', async () => { + const { node } = buildFakes({ serviceJobInDb: null }) + const res = await new GetServicesHandler(node).handle({ ...baseTask } as any) + expect(res.status.httpStatus).to.equal(200) + const out = await body(res) + expect(out).to.deep.equal([]) + }) + + it('400 on an updatedSince that is unparseable, empty, or overflows', async () => { + const { node } = buildFakes() + // an explicit empty string and an overflowing digit-only value (→ Infinity) must + // be rejected, not fall through to an unfiltered all-statuses dump + for (const updatedSince of ['not-a-date', '', '9'.repeat(400)]) { + const res = await new GetServicesHandler(node).handle({ + ...baseTask, + updatedSince + } as any) + expect( + res.status.httpStatus, + `updatedSince=${JSON.stringify(updatedSince)}` + ).to.equal(400) + } + }) + + it('updatedSince: returns every status changed at/after the cursor, from the all-jobs read', async () => { + // realistic epoch-ms so parseFromTimestamp treats the values as ms, not seconds + const base = 1_700_000_000_000 + const older = makeJob({ + serviceId: 'svc-old', + status: ServiceStatusNumber.Running, + updatedAt: base + }) + const failed = makeJob({ + serviceId: 'svc-fail', + status: ServiceStatusNumber.PullImageFailed, + updatedAt: base + 5000 + }) + const expired = makeJob({ + serviceId: 'svc-exp', + status: ServiceStatusNumber.Expired, + updatedAt: base + 6000 + }) + const { node, engine } = buildFakes() + engine.db.getServiceJob.resolves([older, failed, expired]) + + const res = await new GetServicesHandler(node).handle({ + ...baseTask, + updatedSince: String(base + 3000) + } as any) + expect(res.status.httpStatus).to.equal(200) + // incremental read uses the all-jobs path, not the resource-holding query + expect(engine.db.getServiceJob.called).to.equal(true) + expect(engine.db.getRunningServiceJobs.called).to.equal(false) + + const out = await body(res) + const ids = out.map((s: any) => s.serviceId) + // older is before the cursor → excluded; both changed-since are returned, + // proving all statuses (incl. the resource-set-hidden PullImageFailed/Expired) pass + expect(ids).to.have.members(['svc-fail', 'svc-exp']) + expect(ids).to.not.include('svc-old') + }) + + it('updatedSince: a record with no updatedAt is treated as 0 and excluded once the cursor advances', async () => { + const legacy = makeJob({ serviceId: 'svc-legacy', updatedAt: undefined }) + const { node, engine } = buildFakes() + engine.db.getServiceJob.resolves([legacy]) + const res = await new GetServicesHandler(node).handle({ + ...baseTask, + updatedSince: String(1_700_000_000_000) + } as any) + expect(await body(res)).to.deep.equal([]) + }) + }) + + describe('ServiceStartHandler', () => { + const baseTask = { + command: PROTOCOL_COMMANDS.SERVICE_START, + consumerAddress: OWNER, + nonce: '1', + signature: '0xsig', + environment: 'env-1', + image: 'nginx', + tag: 'alpine', + exposedPorts: [80], + dockerCmd: ['nginx', '-g', 'daemon off;'], + dockerEntrypoint: ['/docker-entrypoint.sh'], + duration: 3600, + payment: { chainId: 8996, token: '0xtoken' } + } + + it('400 when consumerAddress is not a valid address', async () => { + const { node } = buildFakes() + const res = await new ServiceStartHandler(node).handle({ + ...baseTask, + consumerAddress: 'not-an-address' + } as any) + expect(res.status.httpStatus).to.equal(400) + }) + + it('400 when environment is unknown (getC2DByEnvId throws)', async () => { + const { node, engines } = buildFakes() + engines.getC2DByEnvId.rejects(new Error('not found')) + const res = await new ServiceStartHandler(node).handle({ ...baseTask } as any) + expect(res.status.httpStatus).to.equal(400) + }) + + it('400 with a clear message when the escrow cannot cover the cost (fail fast)', async () => { + const { node, engine } = buildFakes() + engine.escrow.getUserAvailableFunds.resolves(0n) + const res = await new ServiceStartHandler(node).handle({ ...baseTask } as any) + expect(res.status.httpStatus).to.equal(400) + expect(String(res.status.error)).to.contain('Insufficient escrow funds') + // no job record may be created for a start that was refused upfront + expect(engine.createServiceJob.called).to.equal(false) + }) + + it('the funds pre-check is best-effort: an RPC failure does not block the start', async () => { + const { node, engine } = buildFakes() + engine.escrow.getUserAvailableFunds.rejects(new Error('rpc down')) + const res = await new ServiceStartHandler(node).handle({ ...baseTask } as any) + expect(res.status.httpStatus).to.equal(200) + expect(engine.createServiceJob.calledOnce).to.equal(true) + }) + + it('403 when services are disabled on the environment', async () => { + const { node } = buildFakes({ serviceEnabled: false }) + const res = await new ServiceStartHandler(node).handle({ ...baseTask } as any) + expect(res.status.httpStatus).to.equal(403) + }) + + it('400 when image is missing', async () => { + const { node } = buildFakes() + const { image, ...noImage } = baseTask + const res = await new ServiceStartHandler(node).handle({ ...noImage } as any) + expect(res.status.httpStatus).to.equal(400) + }) + + it('400 when more than one image mode is set (tag + dockerfile)', async () => { + const { node } = buildFakes() + const res = await new ServiceStartHandler(node).handle({ + ...baseTask, + dockerfile: 'FROM nginx:alpine' + } as any) + expect(res.status.httpStatus).to.equal(400) + }) + + it('400 when duration exceeds maxDurationSeconds', async () => { + const { node } = buildFakes() + const res = await new ServiceStartHandler(node).handle({ + ...baseTask, + duration: 999999 + } as any) + expect(res.status.httpStatus).to.equal(400) + }) + + it('400 when no pricing for the token (cost null)', async () => { + const { node } = buildFakes({ cost: null }) + const res = await new ServiceStartHandler(node).handle({ ...baseTask } as any) + expect(res.status.httpStatus).to.equal(400) + }) + + it('200 returns immediately with a Starting job and does NOT run escrow synchronously', async () => { + const { node, engine, escrow } = buildFakes() + const res = await new ServiceStartHandler(node).handle({ ...baseTask } as any) + expect(res.status.httpStatus).to.equal(200) + const out = await body(res) + // The response is the freshly-persisted Starting record — escrow + container come later. + expect(out[0].status).to.equal(ServiceStatusNumber.Starting) + expect(out[0].endpoints).to.deep.equal([]) + expect(out[0]).to.not.have.property('userData') + // Escrow now runs in the background pipeline, not in the request path. + expect(escrow.createLock.called).to.equal(false) + expect(escrow.claimLock.called).to.equal(false) + // The handler must not invoke the background pipeline itself. + expect(engine.processServiceStart).to.equal(undefined) + }) + + it('200 happy path: calls createServiceJob with env/image/dockerCmd/dockerEntrypoint, strips userData', async () => { + const { node, engine } = buildFakes() + const res = await new ServiceStartHandler(node).handle({ ...baseTask } as any) + expect(res.status.httpStatus).to.equal(200) + assert(engine.createServiceJob.calledOnce, 'createServiceJob should be called') + const { args } = engine.createServiceJob.firstCall + // signature: (environment, image, tag, checksum, dockerfile, additionalDockerFiles, + // dockerCmd, dockerEntrypoint, exposedPorts, resources, duration, owner, + // payment, serviceId, userData) + expect(args[0]).to.equal('env-1') + expect(args[1]).to.equal('nginx') + expect(args[6]).to.deep.equal(['nginx', '-g', 'daemon off;']) + expect(args[7]).to.deep.equal(['/docker-entrypoint.sh']) + // payment carries the server-side cost but no tx hashes yet (filled in by the pipeline). + const payment = args[12] + expect(payment.cost).to.equal(10) + expect(payment.lockTx).to.equal('') + expect(payment.claimTx).to.equal('') + const out = await body(res) + expect(out[0]).to.not.have.property('userData') + }) + }) + + describe('ServiceGetStreamableLogsHandler', () => { + const baseTask = { + command: PROTOCOL_COMMANDS.SERVICE_GET_STREAMABLE_LOGS, + consumerAddress: OWNER, + nonce: '1', + signature: '0xsig', + serviceId: 'svc-1' + } + + it('400 when serviceId is missing', async () => { + const { node } = buildFakes({ serviceJobInDb: makeJob() }) + const { serviceId, ...noServiceId } = baseTask + const res = await new ServiceGetStreamableLogsHandler(node).handle({ + ...noServiceId + } as any) + expect(res.status.httpStatus).to.equal(400) + }) + + it('401 when the signature/token is invalid', async () => { + const { node } = buildFakes({ serviceJobInDb: makeJob() }) + node.getAuth = () => ({ + validateAuthenticationOrToken: () => + Promise.resolve({ valid: false, error: 'bad signature' }) + }) + const res = await new ServiceGetStreamableLogsHandler(node).handle({ + ...baseTask + } as any) + expect(res.status.httpStatus).to.equal(401) + }) + + it('400 when service not found', async () => { + const { node } = buildFakes({ serviceJobInDb: null }) + const res = await new ServiceGetStreamableLogsHandler(node).handle({ + ...baseTask + } as any) + expect(res.status.httpStatus).to.equal(400) + }) + + it('401 when caller is not the owner', async () => { + const { node } = buildFakes({ serviceJobInDb: makeJob({ owner: '0xsomeoneelse' }) }) + const res = await new ServiceGetStreamableLogsHandler(node).handle({ + ...baseTask + } as any) + expect(res.status.httpStatus).to.equal(401) + }) + + it('404 when the engine has no stream to return (not running/error)', async () => { + const { node } = buildFakes({ serviceJobInDb: makeJob(), streamableLogs: null }) + const res = await new ServiceGetStreamableLogsHandler(node).handle({ + ...baseTask + } as any) + expect(res.status.httpStatus).to.equal(404) + }) + + it('200 and returns the engine stream on success', async () => { + const { node, engine } = buildFakes({ serviceJobInDb: makeJob() }) + const res = await new ServiceGetStreamableLogsHandler(node).handle({ + ...baseTask + } as any) + expect(res.status.httpStatus).to.equal(200) + expect(engine.getServiceStreamableLogs.calledOnceWith('svc-1', OWNER)).to.equal( + true + ) + const chunks: Buffer[] = [] + for await (const chunk of res.stream as Readable) { + chunks.push(Buffer.from(chunk)) + } + expect(Buffer.concat(chunks).toString()).to.equal('hello logs') + }) + + it('200 when service is in Error status (crashed container logs still fetchable)', async () => { + const { node, engine } = buildFakes({ + serviceJobInDb: makeJob({ status: ServiceStatusNumber.Error }) + }) + const res = await new ServiceGetStreamableLogsHandler(node).handle({ + ...baseTask + } as any) + expect(res.status.httpStatus).to.equal(200) + expect(engine.getServiceStreamableLogs.calledOnce).to.equal(true) + }) + + it('400 when since has an invalid format', async () => { + const { node } = buildFakes({ serviceJobInDb: makeJob() }) + const res = await new ServiceGetStreamableLogsHandler(node).handle({ + ...baseTask, + since: 'not-a-valid-since' + } as any) + expect(res.status.httpStatus).to.equal(400) + }) + + it('200 and passes an absolute since timestamp straight through', async () => { + const { node, engine } = buildFakes({ serviceJobInDb: makeJob() }) + const res = await new ServiceGetStreamableLogsHandler(node).handle({ + ...baseTask, + since: '1735689600' + } as any) + expect(res.status.httpStatus).to.equal(200) + expect(engine.getServiceStreamableLogs.firstCall.args).to.deep.equal([ + 'svc-1', + OWNER, + 1735689600 + ]) + }) + + it('200 and converts a relative since duration to a timestamp', async () => { + const { node, engine } = buildFakes({ serviceJobInDb: makeJob() }) + const before = Math.floor(Date.now() / 1000) - 3600 + const res = await new ServiceGetStreamableLogsHandler(node).handle({ + ...baseTask, + since: '1h' + } as any) + const after = Math.floor(Date.now() / 1000) - 3600 + expect(res.status.httpStatus).to.equal(200) + const since = engine.getServiceStreamableLogs.firstCall.args[2] + expect(since).to.be.within(before, after) + }) + }) +}) diff --git a/src/test/unit/service/serviceJobsDatabase.test.ts b/src/test/unit/service/serviceJobsDatabase.test.ts new file mode 100644 index 000000000..2eaf55eab --- /dev/null +++ b/src/test/unit/service/serviceJobsDatabase.test.ts @@ -0,0 +1,478 @@ +import { assert, expect } from 'chai' +import { Readable } from 'stream' +import { C2DDatabase } from '../../../components/database/C2DDatabase.js' +import { typesenseSchemas } from '../../../components/database/TypesenseSchemas.js' +import { getConfiguration } from '../../../utils/config.js' +import { + buildEnvOverrideConfig, + OverrideEnvConfig, + setupEnvironment, + tearDownEnvironment, + TEST_ENV_CONFIG_FILE +} from '../../utils/utils.js' +import { ENVIRONMENT_VARIABLES } from '../../../utils/constants.js' +import { OceanNodeConfig } from '../../../@types/OceanNode.js' +import { + C2DClusterType, + ComputeEnvironment, + ComputeJob, + ComputeResource +} from '../../../@types/C2D/C2D.js' +import { ServiceJob, ServiceStatusNumber } from '../../../@types/C2D/ServiceOnDemand.js' +import { C2DEngine } from '../../../components/c2d/compute_engine_base.js' +import { ValidateParams } from '../../../components/httpRoutes/validateCommands.js' + +const CLUSTER_HASH = 'svc-test-cluster' + +/* eslint-disable require-await */ +// Minimal concrete engine bound to a real DB + cluster hash, so we can exercise +// getUsedResources() (which reads running compute + service jobs from the DB). +class SharedAccountingEngine extends C2DEngine { + constructor(db: C2DDatabase) { + super({ type: C2DClusterType.DOCKER, hash: CLUSTER_HASH }, db, null, null, null) + } + + setLimits(limits: Map) { + this.physicalLimits = limits + } + + async getComputeEnvironments(): Promise { + return [] + } + + async checkDockerImage(): Promise { + return { valid: true, reason: null as string, status: 200 } + } + + async startComputeJob(): Promise { + return [] + } + + async stopComputeJob(): Promise { + return [] + } + + async getComputeJobStatus(): Promise { + return [] + } + + async getComputeJobResult(): Promise<{ stream: Readable; headers: any }> { + return null + } + + async cleanupExpiredStorage(): Promise { + return true + } +} +/* eslint-enable require-await */ + +function makeEnv(id: string, resources: ComputeResource[]): ComputeEnvironment { + return { + id, + resources, + runningJobs: 0, + runningfreeJobs: 0, + queuedJobs: 0, + queuedFreeJobs: 0, + queMaxWaitTime: 0, + queMaxWaitTimeFree: 0, + runMaxWaitTime: 0, + runMaxWaitTimeFree: 0, + consumerAddress: '0x0', + fees: {}, + access: { addresses: [], accessLists: null }, + platform: { architecture: 'x86_64', os: 'linux' }, + minJobDuration: 60, + maxJobDuration: 3600, + maxJobs: 10 + } as ComputeEnvironment +} + +function makeServiceJob(overrides: Partial = {}): ServiceJob { + return { + serviceId: 'svc-' + Math.random().toString(36).slice(2), + clusterHash: CLUSTER_HASH, + environment: 'env-a', + owner: '0xowner', + image: 'quay.io/jupyter/datascience-notebook', + tag: 'latest', + containerImage: 'quay.io/jupyter/datascience-notebook:latest', + containerId: 'container-1', + networkId: 'network-1', + status: ServiceStatusNumber.Running, + statusText: 'Running', + dateCreated: new Date(0).toISOString(), + expiresAt: Date.now() + 3600_000, + duration: 3600, + exposedPorts: [8888], + endpoints: [{ containerPort: 8888, hostPort: 31000, url: 'http://localhost:31000' }], + userData: 'ENCRYPTED_BLOB', + resources: [{ id: 'cpu', amount: 2 }], + payment: { + chainId: 8996, + token: '0x123', + lockTx: '0xlock', + claimTx: '0xclaim', + cancelTx: '', + cost: 5 + }, + ...overrides + } +} + +describe('Service Jobs Database', () => { + let envOverrides: OverrideEnvConfig[] + let config: OceanNodeConfig + let db: C2DDatabase = null + + before(async () => { + envOverrides = buildEnvOverrideConfig( + [ENVIRONMENT_VARIABLES.DOCKER_COMPUTE_ENVIRONMENTS], + [ + '[{"socketPath":"/var/run/docker.sock","environments":[{"storageExpiry":604800,"maxJobDuration":3600,"minJobDuration":60,"resources":[{"id":"cpu","total":4,"max":4,"min":1,"type":"cpu"}],"fees":{"1":[{"feeToken":"0x123","prices":[{"id":"cpu","price":1}]}]}}]}]' + ] + ) + envOverrides = await setupEnvironment(TEST_ENV_CONFIG_FILE, envOverrides) + config = await getConfiguration(true) + db = await new C2DDatabase(config.dbConfig, typesenseSchemas.c2dSchemas) + }) + + after(async () => { + await tearDownEnvironment(envOverrides) + }) + + describe('service lifecycle locks (cross-process lease)', () => { + const STALE_MS = 60_000 + + it('grants the lock to exactly one holder and rejects a second acquire', async () => { + const serviceId = `lock-svc-${Date.now()}-a` + expect(await db.acquireServiceLock(serviceId, 'proc-A', STALE_MS)).to.equal(true) + // second process: fresh lock held by A → refused + expect(await db.acquireServiceLock(serviceId, 'proc-B', STALE_MS)).to.equal(false) + // A releasing frees it for B + await db.releaseServiceLock(serviceId, 'proc-A') + expect(await db.acquireServiceLock(serviceId, 'proc-B', STALE_MS)).to.equal(true) + await db.releaseServiceLock(serviceId, 'proc-B') + }) + + it('steals a stale lock (crashed holder) but not a fresh one', async () => { + const serviceId = `lock-svc-${Date.now()}-b` + // staleMs = 0 makes A's row instantly stale — simulates a dead process + expect(await db.acquireServiceLock(serviceId, 'proc-A', STALE_MS)).to.equal(true) + expect(await db.acquireServiceLock(serviceId, 'proc-B', 0)).to.equal(true) + // now B's row is fresh: A must not get it back with a normal window + expect(await db.acquireServiceLock(serviceId, 'proc-A', STALE_MS)).to.equal(false) + await db.releaseServiceLock(serviceId, 'proc-B') + }) + + it('release is holder-scoped: a stale-steal victim cannot delete the new lock', async () => { + const serviceId = `lock-svc-${Date.now()}-c` + expect(await db.acquireServiceLock(serviceId, 'proc-A', STALE_MS)).to.equal(true) + expect(await db.acquireServiceLock(serviceId, 'proc-B', 0)).to.equal(true) // steal + // A (which lost the lock) releasing must be a no-op for B's row + await db.releaseServiceLock(serviceId, 'proc-A') + expect(await db.isServiceLocked(serviceId, STALE_MS)).to.equal(true) + await db.releaseServiceLock(serviceId, 'proc-B') + expect(await db.isServiceLocked(serviceId, STALE_MS)).to.equal(false) + }) + + it('refreshServiceLocks re-stamps a holder’s rows so they stay unstealable', async () => { + const serviceId = `lock-svc-${Date.now()}-d` + expect(await db.acquireServiceLock(serviceId, 'proc-A', STALE_MS)).to.equal(true) + // age the ORIGINAL stamp past the staleness window B will use below, so the + // refusal can only be explained by the refresh re-stamping the row — without the + // aging, a freshly-acquired row would be "unstealable" even if refresh did nothing + await new Promise((resolve) => setTimeout(resolve, 150)) + await db.refreshServiceLocks('proc-A') + // B treats anything older than 100ms as stale: the original stamp (≥150ms old) + // would be stolen; the re-stamped one (a few ms old) must not be + expect(await db.acquireServiceLock(serviceId, 'proc-B', 100)).to.equal(false) + await db.releaseServiceLock(serviceId, 'proc-A') + }) + }) + + it('inserts and reads back a service job by serviceId', async () => { + const job = makeServiceJob() + await db.newServiceJob(job) + const [found] = await db.getServiceJob(job.serviceId) + assert(found, 'service job not found') + expect(found.serviceId).to.equal(job.serviceId) + expect(found.environment).to.equal('env-a') + expect(found.userData).to.equal('ENCRYPTED_BLOB') + expect(found.resources[0]).to.deep.equal({ id: 'cpu', amount: 2 }) + }) + + it('filters service jobs by owner', async () => { + const mine = makeServiceJob({ owner: '0xalice' }) + const theirs = makeServiceJob({ owner: '0xbob' }) + await db.newServiceJob(mine) + await db.newServiceJob(theirs) + const aliceJobs = await db.getServiceJob(undefined, '0xalice') + expect(aliceJobs.every((j) => j.owner === '0xalice')).to.equal(true) + expect(aliceJobs.find((j) => j.serviceId === mine.serviceId)).to.not.equal(undefined) + }) + + it('updates a service job (status + expiresAt + body)', async () => { + const job = makeServiceJob() + await db.newServiceJob(job) + job.status = ServiceStatusNumber.Stopped + job.statusText = 'Stopped' + job.expiresAt = 123456 + const updated = await db.updateServiceJob(job) + expect(updated).to.equal(1) + const [found] = await db.getServiceJob(job.serviceId) + expect(found.status).to.equal(ServiceStatusNumber.Stopped) + expect(found.expiresAt).to.equal(123456) + }) + + it('stamps updatedAt on insert and bumps it on every status change', async () => { + const before = Date.now() + const job = makeServiceJob() + await db.newServiceJob(job) + const [inserted] = await db.getServiceJob(job.serviceId) + expect(inserted.updatedAt).to.be.a('number') + expect(inserted.updatedAt).to.be.at.least(before) + + // a later transition must advance updatedAt — this is the incremental-sync cursor + await new Promise((resolve) => setTimeout(resolve, 5)) + job.status = ServiceStatusNumber.Stopped + await db.updateServiceJob(job) + const [updated] = await db.getServiceJob(job.serviceId) + expect(updated.updatedAt).to.be.greaterThan(inserted.updatedAt) + }) + + it('getRunningServiceJobs returns only active statuses for the cluster', async () => { + const running = makeServiceJob({ status: ServiceStatusNumber.Running }) + const starting = makeServiceJob({ status: ServiceStatusNumber.Starting }) + const locking = makeServiceJob({ status: ServiceStatusNumber.Locking }) + const claiming = makeServiceJob({ status: ServiceStatusNumber.Claiming }) + const restarting = makeServiceJob({ status: ServiceStatusNumber.Restarting }) + const error = makeServiceJob({ status: ServiceStatusNumber.Error }) + const stopped = makeServiceJob({ status: ServiceStatusNumber.Stopped }) + // never paid: escrow lock failed outright (all payment fields empty) + const unpaidError = makeServiceJob({ + status: ServiceStatusNumber.Error, + payment: { + chainId: 8996, + token: '0x123', + lockTx: '', + claimTx: '', + cancelTx: '', + cost: 5 + } + }) + // never paid: lock was refunded before being claimed + const refundedStopped = makeServiceJob({ + status: ServiceStatusNumber.Stopped, + payment: { + chainId: 8996, + token: '0x123', + lockTx: '0xlock', + claimTx: '', + cancelTx: '0xcancel', + cost: 5 + } + }) + const otherCluster = makeServiceJob({ + status: ServiceStatusNumber.Running, + clusterHash: 'other-cluster' + }) + await db.newServiceJob(running) + await db.newServiceJob(starting) + await db.newServiceJob(locking) + await db.newServiceJob(claiming) + await db.newServiceJob(restarting) + await db.newServiceJob(error) + await db.newServiceJob(stopped) + await db.newServiceJob(unpaidError) + await db.newServiceJob(refundedStopped) + await db.newServiceJob(otherCluster) + + const active = await db.getRunningServiceJobs(CLUSTER_HASH) + const ids = active.map((j) => j.serviceId) + expect(ids).to.include(running.serviceId) + expect(ids).to.include(starting.serviceId) + // the new start-pipeline states must reserve resources too + expect(ids).to.include(locking.serviceId) + expect(ids).to.include(claiming.serviceId) + // a mid-restart service keeps holding its resources + expect(ids).to.include(restarting.serviceId) + // a service whose container died on its own still reserves its resources until + // restarted or expired + expect(ids).to.include(error.serviceId) + // an explicitly-stopped service KEEPS its reservation too: the consumer paid for + // the whole window and may restart it anytime — only Expired releases resources + expect(ids).to.include(stopped.serviceId) + // …but the reservation is tied to PAYMENT: a terminal job whose payment was never + // claimed (lock failed / refunded) must not squat resources + expect(ids).to.not.include(unpaidError.serviceId) + expect(ids).to.not.include(refundedStopped.serviceId) + expect(ids).to.not.include(otherCluster.serviceId) + }) + + it('getPendingServiceStarts returns mid-start jobs (not Running/terminal) for the cluster', async () => { + const starting = makeServiceJob({ status: ServiceStatusNumber.Starting }) + const locking = makeServiceJob({ status: ServiceStatusNumber.Locking }) + const claiming = makeServiceJob({ status: ServiceStatusNumber.Claiming }) + const restarting = makeServiceJob({ status: ServiceStatusNumber.Restarting }) + const running = makeServiceJob({ status: ServiceStatusNumber.Running }) + const stopped = makeServiceJob({ status: ServiceStatusNumber.Stopped }) + const otherCluster = makeServiceJob({ + status: ServiceStatusNumber.Starting, + clusterHash: 'other-cluster' + }) + await db.newServiceJob(starting) + await db.newServiceJob(locking) + await db.newServiceJob(claiming) + await db.newServiceJob(restarting) + await db.newServiceJob(running) + await db.newServiceJob(stopped) + await db.newServiceJob(otherCluster) + + const pending = await db.getPendingServiceStarts(CLUSTER_HASH) + const ids = pending.map((j) => j.serviceId) + expect(ids).to.include(starting.serviceId) + expect(ids).to.include(locking.serviceId) + expect(ids).to.include(claiming.serviceId) + // a crash mid-restart must be orphan-recovered at boot exactly like a crash mid-start + expect(ids).to.include(restarting.serviceId) + expect(ids).to.not.include(running.serviceId) // already started + expect(ids).to.not.include(stopped.serviceId) + expect(ids).to.not.include(otherCluster.serviceId) + }) + + it('getExpiredServiceJobs returns Running, Error and Stopped jobs past expiry', async () => { + const expired = makeServiceJob({ + status: ServiceStatusNumber.Running, + expiresAt: Date.now() - 1000 + }) + const expiredError = makeServiceJob({ + status: ServiceStatusNumber.Error, + expiresAt: Date.now() - 1000 + }) + const future = makeServiceJob({ + status: ServiceStatusNumber.Running, + expiresAt: Date.now() + 3600_000 + }) + const expiredStopped = makeServiceJob({ + status: ServiceStatusNumber.Stopped, + expiresAt: Date.now() - 1000 + }) + const futureStopped = makeServiceJob({ + status: ServiceStatusNumber.Stopped, + expiresAt: Date.now() + 3600_000 + }) + await db.newServiceJob(expired) + await db.newServiceJob(expiredError) + await db.newServiceJob(future) + await db.newServiceJob(expiredStopped) + await db.newServiceJob(futureStopped) + + const expiredJobs = await db.getExpiredServiceJobs(CLUSTER_HASH) + const ids = expiredJobs.map((j) => j.serviceId) + expect(ids).to.include(expired.serviceId) + // an abandoned Error'd service must still be swept once past expiresAt, or its + // reserved resources/ports would leak forever + expect(ids).to.include(expiredError.serviceId) + // a Stopped service past its paid window must flip to Expired instead of sitting + // at Stopped forever (where it would read as restartable) + expect(ids).to.include(expiredStopped.serviceId) + expect(ids).to.not.include(future.serviceId) + expect(ids).to.not.include(futureStopped.serviceId) + }) + + describe('shared resource accounting (compute + service)', () => { + let engine: SharedAccountingEngine + + before(async () => { + engine = new SharedAccountingEngine(db) + engine.setLimits( + new Map([ + ['cpu', 10], + ['gpu0', 2] + ]) + ) + // Clean slate: retire any leftover jobs from earlier tests by marking them Expired — + // the only status that releases the reservation (Stopped still counts as active). + const leftovers = await db.getRunningServiceJobs(CLUSTER_HASH) + for (const j of leftovers) { + j.status = ServiceStatusNumber.Expired + await db.updateServiceJob(j) + } + }) + + it('a running service occupies fungible resources in its own env', async () => { + await db.newServiceJob( + makeServiceJob({ + environment: 'env-shared', + status: ServiceStatusNumber.Running, + resources: [{ id: 'cpu', amount: 3 }] + }) + ) + const env = makeEnv('env-shared', [ + { id: 'cpu', kind: 'fungible', total: 10, min: 1, max: 10 } as ComputeResource + ]) + const used = await engine.getUsedResources(env) + expect(used.usedResources.cpu).to.equal(3) + }) + + it('fungible usage is NOT counted against a different env', async () => { + // The cpu service above is bound to 'env-shared'; a different env sees 0 cpu used. + const otherEnv = makeEnv('env-other', [ + { id: 'cpu', kind: 'fungible', total: 10, min: 1, max: 10 } as ComputeResource + ]) + const used = await engine.getUsedResources(otherEnv) + expect(used.usedResources.cpu ?? 0).to.equal(0) + }) + + it('a running service occupies discrete resources GLOBALLY (any env)', async () => { + await db.newServiceJob( + makeServiceJob({ + environment: 'env-gpu', + status: ServiceStatusNumber.Running, + resources: [{ id: 'gpu0', amount: 1 }] + }) + ) + // Query a DIFFERENT env: discrete usage is global, so it still shows up. + const env = makeEnv('env-elsewhere', [ + { + id: 'gpu0', + kind: 'discrete', + type: 'gpu', + total: 2, + min: 0, + max: 2 + } as ComputeResource + ]) + const used = await engine.getUsedResources(env) + expect(used.usedResources.gpu0).to.equal(1) + }) + + it('checkIfResourcesAreAvailable blocks a compute request when a service holds the GPU', async () => { + // gpu0 total:1 in the env, already 1 in use by the service above (global discrete). + const env = makeEnv('env-elsewhere', [ + { + id: 'gpu0', + kind: 'discrete', + type: 'gpu', + total: 1, + min: 0, + max: 1, + inUse: 1 + } as ComputeResource + ]) + let threw = false + try { + await engine.checkIfResourcesAreAvailable( + [{ id: 'gpu0', amount: 1 }], + env, + false, + [env] + ) + } catch { + threw = true + } + expect(threw).to.equal(true) + }) + }) +}) diff --git a/src/test/unit/service/serviceNetworkCleanup.test.ts b/src/test/unit/service/serviceNetworkCleanup.test.ts new file mode 100644 index 000000000..db1d59e95 --- /dev/null +++ b/src/test/unit/service/serviceNetworkCleanup.test.ts @@ -0,0 +1,228 @@ +import { expect } from 'chai' +import sinon from 'sinon' +import { C2DEngineDocker } from '../../../components/c2d/compute_engine_docker.js' +import { ServiceStatusNumber, ServiceJob } from '../../../@types/C2D/ServiceOnDemand.js' + +const OWNER = '0x0000000000000000000000000000000000000001' +const SERVICE_ID = 'svc-1' +const NETWORK_NAME = `ocean-svc-${SERVICE_ID}` + +function makeJob(overrides: Partial = {}): ServiceJob { + return { + serviceId: SERVICE_ID, + clusterHash: 'hash-1', + environment: 'env-1', + owner: OWNER, + image: 'img', + tag: 'latest', + containerImage: 'img:latest', + containerId: 'c1', + networkId: 'n1', + status: ServiceStatusNumber.Running, + statusText: 'Running', + dateCreated: new Date(0).toISOString(), + expiresAt: Date.now() + 3600_000, + duration: 3600, + exposedPorts: [8888], + endpoints: [{ containerPort: 8888, hostPort: 31000, url: 'http://localhost:31000' }], + userData: 'ENCRYPTED', + resources: [{ id: 'cpu', amount: 2 }], + payment: { + chainId: 8996, + token: '0xtoken', + lockTx: '0xl', + claimTx: '0xc', + cancelTx: '', + cost: 5 + }, + ...overrides + } +} + +function benignError(statusCode: number = 404) { + const e: any = new Error(`docker ${statusCode}`) + e.statusCode = statusCode + return e +} + +// Fake dockerode network handle: inspect() result (or rejection) and remove() behavior +// are configurable per test. +function fakeNetwork(opts: { + inspect?: any + inspectError?: any + removeError?: any +}): any { + return { + inspect: opts.inspectError + ? sinon.stub().rejects(opts.inspectError) + : sinon.stub().resolves(opts.inspect ?? { Containers: {} }), + remove: opts.removeError + ? sinon.stub().rejects(opts.removeError) + : sinon.stub().resolves(undefined) + } +} + +// Bypass the Docker-specific constructor while retaining the prototype chain (same +// pattern as compute.test.ts) so private methods can be exercised with a stubbed docker. +function makeEngine(): any { + const engine: any = Object.create(C2DEngineDocker.prototype) + engine.docker = { + getNetwork: sinon.stub(), + getContainer: sinon.stub(), + createNetwork: sinon.stub() + } + engine.cpuAllocations = new Map() + engine.serviceOpsInFlight = new Set() + engine.serviceOpPromises = new Set() + return engine +} + +describe('service network cleanup (leaked ocean-svc- networks)', () => { + afterEach(() => sinon.restore()) + + describe('removeServiceNetwork()', () => { + it('removes the network by deterministic name when networkId is empty (leak scenario)', async () => { + const engine = makeEngine() + const network = fakeNetwork({}) + engine.docker.getNetwork.returns(network) + + await engine.removeServiceNetwork(SERVICE_ID) + + expect(engine.docker.getNetwork.calledOnceWith(NETWORK_NAME)).to.equal(true) + expect(network.remove.calledOnce).to.equal(true) + }) + + it('tries both the persisted id and the deterministic name', async () => { + const engine = makeEngine() + const byName = fakeNetwork({}) + // after the first ref is removed, the second ref's inspect 404s and is skipped + const byId = fakeNetwork({ inspectError: benignError(404) }) + engine.docker.getNetwork.withArgs(NETWORK_NAME).returns(byName) + engine.docker.getNetwork.withArgs('n1').returns(byId) + + await engine.removeServiceNetwork(SERVICE_ID, 'n1') + + expect(engine.docker.getNetwork.calledWith(NETWORK_NAME)).to.equal(true) + expect(engine.docker.getNetwork.calledWith('n1')).to.equal(true) + expect(byName.remove.calledOnce).to.equal(true) + expect(byId.remove.called).to.equal(false) + }) + + it('force-removes stale attached containers before removing the network', async () => { + const engine = makeEngine() + const network = fakeNetwork({ inspect: { Containers: { cid1: {}, cid2: {} } } }) + engine.docker.getNetwork.returns(network) + const containerRemove = sinon.stub().resolves(undefined) + engine.docker.getContainer.returns({ remove: containerRemove }) + + await engine.removeServiceNetwork(SERVICE_ID) + + expect(engine.docker.getContainer.calledWith('cid1')).to.equal(true) + expect(engine.docker.getContainer.calledWith('cid2')).to.equal(true) + expect(containerRemove.alwaysCalledWith({ force: true })).to.equal(true) + expect(network.remove.calledOnce).to.equal(true) + expect(containerRemove.calledBefore(network.remove)).to.equal(true) + }) + + it('resolves silently when the network is already gone (404 inspect / 404 remove)', async () => { + const engine = makeEngine() + const gone = fakeNetwork({ inspectError: benignError(404) }) + engine.docker.getNetwork.returns(gone) + await engine.removeServiceNetwork(SERVICE_ID) + expect(gone.remove.called).to.equal(false) + + const goneOnRemove = fakeNetwork({ removeError: benignError(404) }) + engine.docker.getNetwork.returns(goneOnRemove) + await engine.removeServiceNetwork(SERVICE_ID) + expect(goneOnRemove.remove.calledOnce).to.equal(true) + }) + + it('propagates non-benign docker errors', async () => { + const engine = makeEngine() + engine.docker.getNetwork.returns(fakeNetwork({ inspectError: benignError(500) })) + try { + await engine.removeServiceNetwork(SERVICE_ID) + expect.fail('expected removeServiceNetwork to reject') + } catch (e: any) { + expect(e.statusCode).to.equal(500) + } + + engine.docker.getNetwork.returns(fakeNetwork({ removeError: benignError(500) })) + try { + await engine.removeServiceNetwork(SERVICE_ID) + expect.fail('expected removeServiceNetwork to reject') + } catch (e: any) { + expect(e.statusCode).to.equal(500) + } + }) + }) + + describe('createServiceNetwork()', () => { + it('returns the created network on the happy path', async () => { + const engine = makeEngine() + const created = { id: 'newnet' } + engine.docker.createNetwork.resolves(created) + + const result = await engine.createServiceNetwork(SERVICE_ID) + + expect(result).to.equal(created) + expect(engine.docker.createNetwork.calledOnceWith({ Name: NETWORK_NAME })).to.equal( + true + ) + }) + + it('on 409 removes the stale network and retries once (self-heal)', async () => { + const engine = makeEngine() + const created = { id: 'newnet' } + engine.docker.createNetwork + .onFirstCall() + .rejects(benignError(409)) + .onSecondCall() + .resolves(created) + const stale = fakeNetwork({}) + engine.docker.getNetwork.returns(stale) + + const result = await engine.createServiceNetwork(SERVICE_ID) + + expect(result).to.equal(created) + expect(stale.remove.calledOnce).to.equal(true) + expect(engine.docker.createNetwork.calledTwice).to.equal(true) + }) + + it('does not retry on non-409 errors', async () => { + const engine = makeEngine() + engine.docker.createNetwork.rejects(benignError(500)) + + try { + await engine.createServiceNetwork(SERVICE_ID) + expect.fail('expected createServiceNetwork to reject') + } catch (e: any) { + expect(e.statusCode).to.equal(500) + } + expect(engine.docker.createNetwork.calledOnce).to.equal(true) + expect(engine.docker.getNetwork.called).to.equal(false) + }) + }) + + describe('stopService() with a leaked network (empty networkId)', () => { + it('removes the network by deterministic name and ends Stopped', async () => { + const engine = makeEngine() + const job = makeJob({ containerId: '', networkId: '' }) + engine.db = { + getServiceJob: sinon.stub().resolves([job]), + updateServiceJob: sinon.stub().resolves(1), + // lifecycle lease (fail-closed: stopService rejects without it) + acquireServiceLock: sinon.stub().resolves(true), + releaseServiceLock: sinon.stub().resolves(undefined) + } + const network = fakeNetwork({}) + engine.docker.getNetwork.returns(network) + + const result = await engine.stopService(SERVICE_ID, OWNER) + + expect(engine.docker.getNetwork.calledWith(NETWORK_NAME)).to.equal(true) + expect(network.remove.calledOnce).to.equal(true) + expect(result.status).to.equal(ServiceStatusNumber.Stopped) + }) + }) +}) diff --git a/src/test/unit/service/serviceRestartRace.test.ts b/src/test/unit/service/serviceRestartRace.test.ts new file mode 100644 index 000000000..f8bc6104e --- /dev/null +++ b/src/test/unit/service/serviceRestartRace.test.ts @@ -0,0 +1,563 @@ +import { expect, assert } from 'chai' +import sinon from 'sinon' +import { C2DEngineDocker } from '../../../components/c2d/compute_engine_docker.js' +import { ServiceStatusNumber, ServiceJob } from '../../../@types/C2D/ServiceOnDemand.js' +import { + allocateHostPort, + releaseHostPort, + reserveHostPort +} from '../../../components/core/service/utils.js' + +const OWNER = '0x0000000000000000000000000000000000000001' +const SERVICE_ID = 'svc-race-1' +const CLUSTER_HASH = 'hash-1' + +// Flushes the microtask queue so an unawaited async call can progress through its +// (immediately-resolving) stubbed awaits up to the next genuinely pending promise. +function flush(): Promise { + return new Promise((resolve) => setImmediate(resolve)) +} + +function makeJob(overrides: Partial = {}): ServiceJob { + return { + serviceId: SERVICE_ID, + clusterHash: CLUSTER_HASH, + environment: 'env-1', + owner: OWNER, + image: 'img', + tag: 'latest', + containerImage: 'img:latest', + containerId: 'c1', + networkId: '', + status: ServiceStatusNumber.Running, + statusText: 'Running', + dateCreated: new Date(0).toISOString(), + expiresAt: Date.now() + 3600_000, + duration: 3600, + exposedPorts: [8888], + endpoints: [{ containerPort: 8888, hostPort: 31000, url: 'http://localhost:31000' }], + userData: undefined, // decryptUserData(undefined) → {} — keyManager never touched + resources: [], // no cpu request → allocateCpus is skipped + payment: { + chainId: 8996, + token: '0xtoken', + lockTx: '0xl', + claimTx: '0xc', + cancelTx: '', + cost: 5 + }, + ...overrides + } +} + +function benignError(statusCode: number = 404) { + const e: any = new Error(`docker ${statusCode}`) + e.statusCode = statusCode + return e +} + +// Bypass the Docker-specific constructor while retaining the prototype chain (same +// pattern as serviceNetworkCleanup.test.ts) so private methods/fields can be exercised +// with a stubbed docker + db. +function makeEngine(): any { + const engine: any = Object.create(C2DEngineDocker.prototype) + engine.docker = { + getNetwork: sinon.stub(), + getContainer: sinon.stub(), + createNetwork: sinon.stub(), + createContainer: sinon.stub() + } + // networks are "already gone" by default (benign 404 on inspect) + engine.docker.getNetwork.returns({ + inspect: sinon.stub().rejects(benignError(404)), + remove: sinon.stub().resolves(undefined) + }) + engine.db = { + getServiceJob: sinon.stub().resolves([]), + updateServiceJob: sinon.stub().resolves(1), + getRunningJobs: sinon.stub().resolves([]), + getPendingServiceStarts: sinon.stub().resolves([]), + getExpiredServiceJobs: sinon.stub().resolves([]), + // cross-process lifecycle lease (service_locks table) — free by default + acquireServiceLock: sinon.stub().resolves(true), + releaseServiceLock: sinon.stub().resolves(undefined), + refreshServiceLocks: sinon.stub().resolves(undefined), + isServiceLocked: sinon.stub().resolves(false) + } + engine.serviceLockHolderId = 'test-holder' + engine.cpuAllocations = new Map() + engine.serviceOpsInFlight = new Set() + engine.serviceOpPromises = new Set() + engine.clusterConfig = { + hash: CLUSTER_HASH, + connection: { resources: [], serviceOnDemand: {} } + } + engine.stopped = false + engine.isInternalLoopRunning = false + engine.cronTimer = null + engine.setNewTimer = sinon.stub() + engine.checkRunningServices = sinon.stub().resolves() + return engine +} + +function stubContainer(overrides: Record = {}) { + return { + stop: sinon.stub().resolves(undefined), + remove: sinon.stub().resolves(undefined), + start: sinon.stub().resolves(undefined), + ...overrides + } +} + +describe('service lifecycle lock (restart/stop vs InternalLoop races)', () => { + afterEach(() => sinon.restore()) + + it('restartService returns Restarting immediately and holds the lock until the background op settles', async () => { + const engine = makeEngine() + const job = makeJob() + engine.db.getServiceJob.resolves([job]) + engine.docker.getContainer.returns(stubContainer()) + engine.docker.createNetwork.resolves({ id: 'newnet' }) + const newContainer = stubContainer() + ;(newContainer as any).id = 'newc' + engine.docker.createContainer.resolves(newContainer) + let resolvePull: () => void + engine.pullImageRef = () => + new Promise((resolve) => { + resolvePull = resolve + }) + + // Non-blocking: resolves as soon as the job is validated + persisted Restarting, + // while the teardown/pull/start continue in the background under the lock. + const snapshot = await engine.restartService(SERVICE_ID, OWNER) + expect(snapshot.status).to.equal(ServiceStatusNumber.Restarting) + expect(engine.serviceOpsInFlight.has(SERVICE_ID)).to.equal(true) + + await flush() // let the background op reach the pending pull + resolvePull!() + await Promise.allSettled([...engine.serviceOpPromises]) + expect(engine.serviceOpsInFlight.has(SERVICE_ID)).to.equal(false) + expect(job.status).to.equal(ServiceStatusNumber.Running) + expect(job.containerId).to.equal('newc') + }) + + it('restartService releases the lock and persists Error on the failure path', async () => { + const engine = makeEngine() + const job = makeJob() + engine.db.getServiceJob.resolves([job]) + engine.docker.getContainer.returns(stubContainer()) + engine.pullImageRef = sinon.stub().rejects(new Error('pull failed')) + + const snapshot = await engine.restartService(SERVICE_ID, OWNER) + expect(snapshot.status).to.equal(ServiceStatusNumber.Restarting) + await Promise.allSettled([...engine.serviceOpPromises]) + expect(engine.serviceOpsInFlight.has(SERVICE_ID)).to.equal(false) + expect(job.status).to.equal(ServiceStatusNumber.Error) + expect(job.statusText).to.equal('pull failed') + // the failure must be PERSISTED, not just mutated in memory — status polls read the DB + expect( + engine.db.updateServiceJob.calledWith( + sinon.match({ + serviceId: SERVICE_ID, + status: ServiceStatusNumber.Error, + statusText: 'pull failed' + }) + ), + 'Error status must be written to the DB' + ).to.equal(true) + // the DB lease must be handed back on the failure path too + expect( + engine.db.releaseServiceLock.calledWith(SERVICE_ID, 'test-holder'), + 'DB lease must be released after the failed restart' + ).to.equal(true) + }) + + it('restartService rejects a service whose payment was refunded (never claimed)', async () => { + const engine = makeEngine() + engine.db.getServiceJob.resolves([ + makeJob({ + status: ServiceStatusNumber.Error, + payment: { + chainId: 8996, + token: '0xtoken', + lockTx: '0xl', + claimTx: '', + cancelTx: '0xcancel', + cost: 5 + } + }) + ]) + + try { + await engine.restartService(SERVICE_ID, OWNER) + expect.fail('expected restartService to reject') + } catch (e: any) { + expect(e.message).to.contain('refunded') + } + expect(engine.serviceOpsInFlight.has(SERVICE_ID)).to.equal(false) + expect(engine.docker.getContainer.called).to.equal(false) + }) + + it('restartService rejects a service that was never paid at all (escrow lock failed)', async () => { + // The free-compute vector: start against an empty escrow account → createLock fails + // → Error with ALL payment fields empty → a restart must not run the service anyway. + const engine = makeEngine() + engine.db.getServiceJob.resolves([ + makeJob({ + status: ServiceStatusNumber.Error, + statusText: 'User 0x… does not have enough funds', + payment: { + chainId: 8996, + token: '0xtoken', + lockTx: '', + claimTx: '', + cancelTx: '', + cost: 5 + } + }) + ]) + + try { + await engine.restartService(SERVICE_ID, OWNER) + expect.fail('expected restartService to reject') + } catch (e: any) { + expect(e.message).to.contain('never claimed') + } + expect(engine.serviceOpsInFlight.has(SERVICE_ID)).to.equal(false) + expect(engine.docker.getContainer.called).to.equal(false) + expect(engine.docker.createNetwork.called).to.equal(false) + }) + + it('rejects a concurrent restart/stop while the lock is held, without touching docker or the DB', async () => { + const engine = makeEngine() + engine.serviceOpsInFlight.add(SERVICE_ID) + + for (const call of [ + () => engine.restartService(SERVICE_ID, OWNER), + () => engine.stopService(SERVICE_ID, OWNER) + ]) { + try { + await call() + expect.fail('expected the call to reject while the lock is held') + } catch (e: any) { + expect(e.message).to.contain('operation in progress') + } + } + expect(engine.docker.getContainer.called).to.equal(false) + expect(engine.docker.getNetwork.called).to.equal(false) + expect(engine.db.getServiceJob.called).to.equal(false) + expect(engine.db.updateServiceJob.called).to.equal(false) + // a failed acquire must not clear a lock held by the in-flight operation + expect(engine.serviceOpsInFlight.has(SERVICE_ID)).to.equal(true) + }) + + it('stopService holds the lock while stopping (a mid-stop restart rejects) and releases it after', async () => { + const engine = makeEngine() + const job = makeJob() + engine.db.getServiceJob.resolves([job]) + let resolveStop: () => void + const pendingStop = new Promise((resolve) => { + resolveStop = resolve + }) + engine.docker.getContainer.returns(stubContainer({ stop: () => pendingStop })) + + const stop = engine.stopService(SERVICE_ID, OWNER) + await flush() + expect(engine.serviceOpsInFlight.has(SERVICE_ID)).to.equal(true) + try { + await engine.restartService(SERVICE_ID, OWNER) + expect.fail('expected restartService to reject while a stop is in flight') + } catch (e: any) { + expect(e.message).to.contain('operation in progress') + } + + resolveStop!() + const stopped = await stop + expect(stopped.status).to.equal(ServiceStatusNumber.Stopped) + expect(engine.serviceOpsInFlight.has(SERVICE_ID)).to.equal(false) + }) + + it('InternalLoop does not launch processServiceStart for a locked service (the orphan-recovery race)', async () => { + const engine = makeEngine() + const midRestart = makeJob({ + status: ServiceStatusNumber.PullImage, + statusText: 'PullImage' + }) + engine.db.getPendingServiceStarts.resolves([midRestart]) + engine.processServiceStart = sinon.stub().resolves() + + // serviceId locked (restartService in flight) → the loop must skip it + engine.serviceOpsInFlight.add(SERVICE_ID) + await engine.InternalLoop() + expect(engine.processServiceStart.called).to.equal(false) + + // control: with the lock free the loop DOES pick the job up (orphan recovery), + // proving this test fails without the lock + engine.serviceOpsInFlight.clear() + engine.isInternalLoopRunning = false + await engine.InternalLoop() + await flush() + expect(engine.processServiceStart.calledOnceWith(midRestart)).to.equal(true) + }) + + it('engine stop() drains an in-flight handler-driven stopService before returning', async () => { + const engine = makeEngine() + engine.db.getServiceJob.resolves([makeJob()]) + let resolveStop: () => void + const pendingStop = new Promise((resolve) => { + resolveStop = resolve + }) + engine.docker.getContainer.returns(stubContainer({ stop: () => pendingStop })) + + const serviceStop = engine.stopService(SERVICE_ID, OWNER) + await flush() + expect(engine.serviceOpPromises.size).to.equal(1) + + let drained = false + const engineStop = engine.stop().then(() => { + drained = true + }) + await flush() + expect(drained).to.equal(false) // must wait on the in-flight stopService + + resolveStop!() + await engineStop + expect(drained).to.equal(true) + expect((await serviceStop).status).to.equal(ServiceStatusNumber.Stopped) + expect(engine.serviceOpPromises.size).to.equal(0) + }) + + it('the expiry sweep defers a locked service instead of marking it Expired without teardown', async () => { + const engine = makeEngine() + const expired = makeJob({ expiresAt: Date.now() - 1000 }) + engine.db.getExpiredServiceJobs.resolves([expired]) + engine.db.getServiceJob.resolves([expired]) + engine.docker.getContainer.returns(stubContainer()) + + // locked (e.g. a user restart is mid-pull) → nothing stopped, nothing persisted + engine.serviceOpsInFlight.add(SERVICE_ID) + await engine.InternalLoop() + expect(engine.db.updateServiceJob.called).to.equal(false) + expect(engine.docker.getContainer.called).to.equal(false) + assert(expired.status === ServiceStatusNumber.Running, 'status must be untouched') + + // lock free → the sweep stops the service and marks it Expired + engine.serviceOpsInFlight.clear() + engine.isInternalLoopRunning = false + await engine.InternalLoop() + expect(expired.status).to.equal(ServiceStatusNumber.Expired) + }) + + it('restartService rejects when another process holds the DB lease', async () => { + const engine = makeEngine() + engine.db.acquireServiceLock = sinon.stub().resolves(false) // held elsewhere + + try { + await engine.restartService(SERVICE_ID, OWNER) + expect.fail('expected restartService to reject') + } catch (e: any) { + expect(e.message).to.contain('operation in progress') + } + // the in-memory reservation must be rolled back so a later acquire can succeed + expect(engine.serviceOpsInFlight.has(SERVICE_ID)).to.equal(false) + expect(engine.db.getServiceJob.called).to.equal(false) + }) + + it('lease acquisition fails CLOSED when the lock DB errors (no docker/DB mutations)', async () => { + const engine = makeEngine() + engine.db.acquireServiceLock = sinon.stub().rejects(new Error('SQLITE_BUSY')) + + try { + await engine.restartService(SERVICE_ID, OWNER) + expect.fail('expected restartService to reject') + } catch (e: any) { + expect(e.message).to.contain('operation in progress') + } + // rolled back, nothing touched — the operation is deferred, not run lock-less + expect(engine.serviceOpsInFlight.has(SERVICE_ID)).to.equal(false) + expect(engine.db.getServiceJob.called).to.equal(false) + expect(engine.db.updateServiceJob.called).to.equal(false) + expect(engine.docker.getContainer.called).to.equal(false) + + // the loop's tryAcquire path defers too (skips the pending job this tick) + engine.db.getPendingServiceStarts.resolves([ + makeJob({ status: ServiceStatusNumber.PullImage, statusText: 'PullImage' }) + ]) + engine.processServiceStart = sinon.stub().resolves() + await engine.InternalLoop() + await flush() + expect(engine.processServiceStart.called).to.equal(false) + expect(engine.serviceOpsInFlight.has(SERVICE_ID)).to.equal(false) + }) + + it('InternalLoop skips a pending job whose DB lease is held by another process', async () => { + const engine = makeEngine() + engine.db.getPendingServiceStarts.resolves([ + makeJob({ status: ServiceStatusNumber.PullImage, statusText: 'PullImage' }) + ]) + engine.db.acquireServiceLock = sinon.stub().resolves(false) + engine.processServiceStart = sinon.stub().resolves() + + await engine.InternalLoop() + await flush() + expect(engine.processServiceStart.called).to.equal(false) + expect(engine.serviceOpsInFlight.has(SERVICE_ID)).to.equal(false) + }) + + it('restartService releases the DB lease once the operation settles', async () => { + const engine = makeEngine() + engine.db.getServiceJob.resolves([makeJob()]) + engine.docker.getContainer.returns(stubContainer()) + engine.docker.createNetwork.resolves({ id: 'newnet' }) + const newContainer = stubContainer() + ;(newContainer as any).id = 'newc' + engine.docker.createContainer.resolves(newContainer) + engine.pullImageRef = sinon.stub().resolves() + + await engine.restartService(SERVICE_ID, OWNER) + await Promise.allSettled([...engine.serviceOpPromises]) + expect( + engine.db.releaseServiceLock.calledWith(SERVICE_ID, 'test-holder'), + 'DB lease must be released with the holder id' + ).to.equal(true) + }) + + it('an explicit stop KEEPS the host ports reserved; only expiry releases them', async () => { + // The user-B scenario: A reserves for 1h, starts, stops after 30min. B must NOT be + // able to grab A's port/resources — A can restart anytime until expiresAt. + const PORT = 31555 + const engine = makeEngine() + const job = makeJob({ + endpoints: [ + { containerPort: 8888, hostPort: PORT, url: `http://localhost:${PORT}` } + ] + }) + engine.db.getServiceJob.resolves([job]) + engine.docker.getContainer.returns(stubContainer()) + reserveHostPort(PORT) // as the original start allocation did + try { + const stopped = await engine.stopService(SERVICE_ID, OWNER) + expect(stopped.status).to.equal(ServiceStatusNumber.Stopped) + // "user B": the allocator must refuse the stopped service's port + let refused = false + try { + await allocateHostPort(PORT, PORT) + } catch { + refused = true + } + expect(refused, "a stopped service's port must stay reserved").to.equal(true) + + // past expiresAt the sweep flips it to Expired and releases the port + job.expiresAt = Date.now() - 1000 + engine.db.getExpiredServiceJobs.resolves([job]) + engine.isInternalLoopRunning = false + await engine.InternalLoop() + expect(job.status).to.equal(ServiceStatusNumber.Expired) + expect(await allocateHostPort(PORT, PORT)).to.equal(PORT) + } finally { + // idempotent — guarantees a failing assertion can't leak the reservation into + // other tests (the allocator set is module-level) + releaseHostPort(PORT) + } + }) + + it('the expiry sweep skips a service that was extended after the expiry snapshot', async () => { + const engine = makeEngine() + // the sweep's snapshot says "expired", but the FRESH row (re-read under the lock by + // doStopService) was extended in the meantime — teardown must not happen + const staleSnapshot = makeJob({ expiresAt: Date.now() - 1000 }) + const extended = makeJob({ expiresAt: Date.now() + 3600_000 }) + engine.db.getExpiredServiceJobs.resolves([staleSnapshot]) + engine.db.getServiceJob.resolves([extended]) + engine.docker.getContainer.returns(stubContainer()) + + await engine.InternalLoop() + + expect(extended.status).to.equal(ServiceStatusNumber.Running) + expect(engine.docker.getContainer.called, 'no teardown may happen').to.equal(false) + expect(engine.db.updateServiceJob.called, 'no state may be written').to.equal(false) + }) + + it('the expiry sweep does NOT mark Expired while teardown fails (no resource leak), then retries', async () => { + const engine = makeEngine() + const expired = makeJob({ expiresAt: Date.now() - 1000 }) + engine.db.getExpiredServiceJobs.resolves([expired]) + engine.db.getServiceJob.resolves([expired]) + // teardown fails hard (docker daemon down — NOT a benign 404) + engine.docker.getContainer.returns( + stubContainer({ stop: sinon.stub().rejects(benignError(500)) }) + ) + + await engine.InternalLoop() + // Expired is terminal and never swept again — a failed stop must leave the job + // OUT of Expired (as Error "stop failed") so the container/ports aren't leaked. + expect(expired.status).to.not.equal(ServiceStatusNumber.Expired) + expect(String(expired.statusText)).to.contain('stop failed') + + // docker recovers → the next tick completes the teardown and marks Expired + engine.docker.getContainer.returns(stubContainer()) + engine.isInternalLoopRunning = false + await engine.InternalLoop() + expect(expired.status).to.equal(ServiceStatusNumber.Expired) + }) + + it('the expiry sweep flips a Stopped service past expiresAt to Expired without touching docker', async () => { + const engine = makeEngine() + const stopped = makeJob({ + status: ServiceStatusNumber.Stopped, + statusText: 'Stopped', + containerId: '', + networkId: '', + expiresAt: Date.now() - 1000 + }) + engine.db.getExpiredServiceJobs.resolves([stopped]) + engine.db.getServiceJob.resolves([stopped]) + + await engine.InternalLoop() + + expect(stopped.status).to.equal(ServiceStatusNumber.Expired) + // resources were already released at stop time — no docker teardown must happen + expect(engine.docker.getContainer.called).to.equal(false) + }) + + it('processServiceStart ignores a stale pending snapshot (job already Running again)', async () => { + const engine = makeEngine() + // Snapshot captured while a restart was mid-pull... + const staleSnapshot = makeJob({ + status: ServiceStatusNumber.PullImage, + statusText: 'PullImage', + containerId: '', + networkId: '' + }) + // ...but by the time the loop processes it, the restart finished: the fresh DB row + // is Running with live docker refs. Pre-guard, orphan-recovery would tear that + // container + network down and clobber the status with Error. + engine.db.getServiceJob.resolves([ + makeJob({ + status: ServiceStatusNumber.Running, + containerId: 'c-live', + networkId: 'n-live' + }) + ]) + + await engine.processServiceStart(staleSnapshot) + + expect(engine.docker.getNetwork.called).to.equal(false) + expect(engine.docker.getContainer.called).to.equal(false) + expect(engine.db.updateServiceJob.called).to.equal(false) + }) + + it('a stopped engine refuses new lifecycle operations', async () => { + const engine = makeEngine() + engine.stopped = true + + try { + await engine.restartService(SERVICE_ID, OWNER) + expect.fail('expected restartService to reject') + } catch (e: any) { + expect(e.message).to.contain('stopped') + } + expect(engine.db.getServiceJob.called).to.equal(false) + expect(engine.db.acquireServiceLock.called).to.equal(false) + }) +}) diff --git a/src/test/unit/service/serviceSchemas.test.ts b/src/test/unit/service/serviceSchemas.test.ts new file mode 100644 index 000000000..1377e7f31 --- /dev/null +++ b/src/test/unit/service/serviceSchemas.test.ts @@ -0,0 +1,137 @@ +import { expect } from 'chai' +import { + ServiceTemplateSchema, + ServiceOnDemandConfigSchema, + C2DEnvironmentConfigSchema +} from '../../../utils/config/schemas.js' + +const baseTemplate = { + id: 'jupyter-cpu', + image: 'quay.io/jupyter/datascience-notebook', + exposedPorts: [8888] +} + +describe('ServiceTemplateSchema', () => { + it('image + tag → valid', () => { + expect( + ServiceTemplateSchema.safeParse({ ...baseTemplate, tag: 'latest' }).success + ).to.equal(true) + }) + it('image + checksum (sha256) → valid', () => { + const checksum = 'sha256:' + 'a'.repeat(64) + expect( + ServiceTemplateSchema.safeParse({ ...baseTemplate, checksum }).success + ).to.equal(true) + }) + it('image + dockerfile → valid', () => { + expect( + ServiceTemplateSchema.safeParse({ ...baseTemplate, dockerfile: 'FROM x' }).success + ).to.equal(true) + }) + it('tag + dockerfile together → invalid', () => { + expect( + ServiceTemplateSchema.safeParse({ ...baseTemplate, tag: 'l', dockerfile: 'FROM x' }) + .success + ).to.equal(false) + }) + it('additionalDockerFiles without dockerfile → invalid', () => { + expect( + ServiceTemplateSchema.safeParse({ + ...baseTemplate, + tag: 'l', + additionalDockerFiles: { 'a.txt': 'x' } + }).success + ).to.equal(false) + }) + it('no tag/checksum/dockerfile → valid (defaults to image:latest at runtime)', () => { + expect(ServiceTemplateSchema.safeParse(baseTemplate).success).to.equal(true) + }) + it('bad id → invalid', () => { + expect( + ServiceTemplateSchema.safeParse({ ...baseTemplate, id: 'Bad Id!' }).success + ).to.equal(false) + }) + it('requiredResources: neither id nor kind → invalid', () => { + expect( + ServiceTemplateSchema.safeParse({ + ...baseTemplate, + requiredResources: [{ min: 1 }] + }).success + ).to.equal(false) + }) + it('requiredResources: both id and kind → invalid', () => { + expect( + ServiceTemplateSchema.safeParse({ + ...baseTemplate, + requiredResources: [{ id: 'cpu', kind: 'fungible', min: 1 }] + }).success + ).to.equal(false) + }) + it('recommended < min → invalid', () => { + expect( + ServiceTemplateSchema.safeParse({ + ...baseTemplate, + requiredResources: [{ id: 'cpu', min: 4, recommended: 2 }] + }).success + ).to.equal(false) + }) + it('valid required + recommended resources → valid', () => { + expect( + ServiceTemplateSchema.safeParse({ + ...baseTemplate, + requiredResources: [{ id: 'cpu', min: 2, recommended: 4 }], + recommendedResources: [{ kind: 'discrete', type: 'gpu', min: 1, recommended: 2 }] + }).success + ).to.equal(true) + }) +}) + +describe('ServiceOnDemandConfigSchema', () => { + it('applies defaults', () => { + const parsed = ServiceOnDemandConfigSchema.parse({ + enabled: true, + nodeHost: 'localhost' + }) + expect(parsed.maxDurationSeconds).to.equal(86400) + expect(parsed.allowImageBuild).to.equal(false) + }) + it('requires nodeHost', () => { + expect(ServiceOnDemandConfigSchema.safeParse({ enabled: true }).success).to.equal( + false + ) + }) + it('rejects unknown keys (strict)', () => { + expect( + ServiceOnDemandConfigSchema.safeParse({ + enabled: true, + nodeHost: 'localhost', + bogus: 1 + }).success + ).to.equal(false) + }) +}) + +describe('C2DEnvironmentConfigSchema features', () => { + const base: any = { + fees: { '8996': [{ feeToken: '0x0', prices: [] as any[] }] }, + storageExpiry: 604800, + maxJobDuration: 3600 + } + it('no features block → both default true', () => { + const parsed: any = C2DEnvironmentConfigSchema.parse(base) + expect(parsed.features).to.deep.equal({ computeJobs: true, services: true }) + }) + it('partial features { computeJobs:false } → services defaults true', () => { + const parsed: any = C2DEnvironmentConfigSchema.parse({ + ...base, + features: { computeJobs: false } + }) + expect(parsed.features).to.deep.equal({ computeJobs: false, services: true }) + }) + it('unknown feature key → invalid (strict)', () => { + expect( + C2DEnvironmentConfigSchema.safeParse({ ...base, features: { compute: true } }) + .success + ).to.equal(false) + }) +}) diff --git a/src/test/unit/service/serviceUtils.test.ts b/src/test/unit/service/serviceUtils.test.ts new file mode 100644 index 000000000..eb7562f85 --- /dev/null +++ b/src/test/unit/service/serviceUtils.test.ts @@ -0,0 +1,124 @@ +import { expect } from 'chai' +import type { ServiceJob } from '../../../@types/C2D/ServiceOnDemand.js' +import { + userDataToEnv, + toPublicServiceJob, + decryptUserData, + allocateHostPort, + releaseHostPort, + parseSinceParam +} from '../../../components/core/service/utils.js' + +describe('service utils', () => { + describe('userDataToEnv', () => { + it('maps a decrypted userData object into a stringified env map', () => { + expect(userDataToEnv({ MODEL_ID: 'm', PORT: 8000, FLAG: true })).to.deep.equal({ + MODEL_ID: 'm', + PORT: '8000', + FLAG: 'true' + }) + }) + it('skips null/undefined values', () => { + expect(userDataToEnv({ A: 'x', B: null, C: undefined })).to.deep.equal({ A: 'x' }) + }) + it('empty object → {}', () => { + expect(userDataToEnv({})).to.deep.equal({}) + }) + }) + + describe('toPublicServiceJob', () => { + it('strips userData, keeps other fields', () => { + const job = { + serviceId: 's1', + userData: 'ENCRYPTED', + owner: '0xabc', + endpoints: [] + } as unknown as ServiceJob + const pub = toPublicServiceJob(job) + expect(pub).to.not.have.property('userData') + expect(pub).to.have.property('serviceId', 's1') + }) + it('is null-safe', () => { + expect(toPublicServiceJob(null)).to.equal(null) + }) + }) + + describe('decryptUserData', () => { + const fakeKeyManager = { + decrypt: (data: Uint8Array) => Promise.resolve(Buffer.from(data)) + } as any + it('returns {} when undefined', async () => { + expect(await decryptUserData(undefined, fakeKeyManager)).to.deep.equal({}) + }) + it('decrypts + JSON-parses a hex payload', async () => { + const payload = JSON.stringify({ MODEL_ID: 'm' }) + const hex = Buffer.from(payload).toString('hex') + const out = await decryptUserData(hex, fakeKeyManager) + expect(out).to.deep.equal({ MODEL_ID: 'm' }) + }) + it('propagates when decrypted payload is not valid JSON', async () => { + const hex = Buffer.from('not-json').toString('hex') + let threw = false + try { + await decryptUserData(hex, fakeKeyManager) + } catch { + threw = true + } + expect(threw).to.equal(true) + }) + }) + + describe('parseSinceParam', () => { + it('returns undefined when omitted', () => { + expect(parseSinceParam(undefined)).to.equal(undefined) + }) + it('parses an all-digit string as an absolute Unix timestamp', () => { + expect(parseSinceParam('1735689600')).to.equal(1735689600) + }) + for (const [input, seconds] of [ + ['30s', 30], + ['45m', 45 * 60], + ['2h', 2 * 3600], + ['7d', 7 * 86400] + ] as [string, number][]) { + it(`converts relative duration "${input}" to now - ${seconds}s`, () => { + const before = Math.floor(Date.now() / 1000) - seconds + const result = parseSinceParam(input) + const after = Math.floor(Date.now() / 1000) - seconds + expect(result).to.be.within(before, after) + }) + } + it('throws on an invalid format', () => { + expect(() => parseSinceParam('not-valid')).to.throw(/Invalid "since" parameter/) + }) + it('throws on an unsupported unit', () => { + expect(() => parseSinceParam('5w')).to.throw(/Invalid "since" parameter/) + }) + }) + + describe('allocateHostPort', () => { + it('never hands out the same port to concurrent callers (TOCTOU)', async () => { + // Property under test: concurrent allocations are always unique. allocateHostPort + // reserves a candidate synchronously (allocatedPorts.add) before the async + // isPortFree() check, so no two concurrent callers can return the same port. + // Use a range far larger than the request count — the allocator probes randomly with + // a bounded retry budget, so an exact-fit range would flake (and CI may already hold + // some ports); ample headroom isolates the uniqueness guarantee from exhaustion. + const rangeStart = 41000 + const rangeEnd = 41999 // 1000 ports + const count = 25 + + const ports = await Promise.all( + Array.from({ length: count }, () => allocateHostPort(rangeStart, rangeEnd)) + ) + try { + expect(new Set(ports).size).to.equal(count) // all unique + ports.forEach((p) => + expect(p).to.be.within(rangeStart, rangeEnd, `port ${p} out of range`) + ) + } finally { + ports.forEach((p) => releaseHostPort(p)) + } + }) + }) +}) diff --git a/src/test/unit/service/templateLoader.test.ts b/src/test/unit/service/templateLoader.test.ts new file mode 100644 index 000000000..791f5275f --- /dev/null +++ b/src/test/unit/service/templateLoader.test.ts @@ -0,0 +1,86 @@ +import { expect } from 'chai' +import { mkdtempSync, writeFileSync, rmSync } from 'fs' +import { join } from 'path' +import { tmpdir } from 'os' +import { loadServiceTemplates } from '../../../components/core/service/templateLoader.js' + +const valid = (id: string) => ({ + id, + image: 'quay.io/jupyter/datascience-notebook', + tag: 'latest', + exposedPorts: [8888] +}) + +describe('loadServiceTemplates', () => { + let dir: string + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'svc-templates-')) + }) + afterEach(() => { + rmSync(dir, { recursive: true, force: true }) + }) + + it('1. undefined dir → []', async () => { + expect(await loadServiceTemplates(undefined)).to.deep.equal([]) + }) + + it('2. non-existent dir → [] (quiet)', async () => { + expect(await loadServiceTemplates(join(dir, 'does-not-exist'))).to.deep.equal([]) + }) + + it('3. two valid single-template files → both returned', async () => { + writeFileSync(join(dir, 'a.json'), JSON.stringify(valid('jupyter-cpu'))) + writeFileSync(join(dir, 'b.json'), JSON.stringify(valid('jupyter-gpu'))) + const templates = await loadServiceTemplates(dir) + expect(templates.map((t) => t.id).sort()).to.deep.equal([ + 'jupyter-cpu', + 'jupyter-gpu' + ]) + }) + + it('4. file containing an array of templates → all returned', async () => { + writeFileSync(join(dir, 'multi.json'), JSON.stringify([valid('one'), valid('two')])) + const templates = await loadServiceTemplates(dir) + expect(templates.map((t) => t.id).sort()).to.deep.equal(['one', 'two']) + }) + + it('5. malformed JSON skipped; others still load', async () => { + writeFileSync(join(dir, 'bad.json'), '{ not json') + writeFileSync(join(dir, 'good.json'), JSON.stringify(valid('good'))) + const templates = await loadServiceTemplates(dir) + expect(templates.map((t) => t.id)).to.deep.equal(['good']) + }) + + it('6. schema-invalid template skipped (tag + dockerfile together)', async () => { + writeFileSync( + join(dir, 'invalid.json'), + JSON.stringify({ ...valid('inv'), dockerfile: 'FROM x' }) + ) + writeFileSync(join(dir, 'ok.json'), JSON.stringify(valid('ok'))) + const templates = await loadServiceTemplates(dir) + expect(templates.map((t) => t.id)).to.deep.equal(['ok']) + }) + + it('7. duplicate id → first (filename-sorted) wins', async () => { + writeFileSync(join(dir, 'a.json'), JSON.stringify({ ...valid('dup'), tag: 'first' })) + writeFileSync(join(dir, 'b.json'), JSON.stringify({ ...valid('dup'), tag: 'second' })) + const templates = await loadServiceTemplates(dir) + expect(templates).to.have.length(1) + expect(templates[0].tag).to.equal('first') + }) + + it('8. non-.json files ignored (e.g. README.md)', async () => { + writeFileSync(join(dir, 'readme.txt'), 'hello') + writeFileSync(join(dir, 'README.md'), '# docs, not a template') + writeFileSync(join(dir, 'good.json'), JSON.stringify(valid('good'))) + const templates = await loadServiceTemplates(dir) + expect(templates.map((t) => t.id)).to.deep.equal(['good']) + }) + + it('9. re-read picks up newly added files (no caching)', async () => { + writeFileSync(join(dir, 'a.json'), JSON.stringify(valid('a'))) + expect(await loadServiceTemplates(dir)).to.have.length(1) + writeFileSync(join(dir, 'b.json'), JSON.stringify(valid('b'))) + expect(await loadServiceTemplates(dir)).to.have.length(2) + }) +}) diff --git a/src/test/unit/sqliteClient.test.ts b/src/test/unit/sqliteClient.test.ts new file mode 100644 index 000000000..6425b3a89 --- /dev/null +++ b/src/test/unit/sqliteClient.test.ts @@ -0,0 +1,110 @@ +import { expect } from 'chai' +import fs from 'fs' +import os from 'os' +import path from 'path' +import { SqliteClient } from '../../components/database/sqliteClient.js' + +describe('SqliteClient', () => { + let tmpDir: string + let client: SqliteClient + + before(() => { + // Nested subdir on purpose: exercises the client's eager mkdir of the parent dir. + tmpDir = path.join( + os.tmpdir(), + `ocean-node-sqliteclient-${process.pid}-${Date.now()}`, + 'nested' + ) + client = new SqliteClient(path.join(tmpDir, 'test.sqlite')) + client.exec(` + CREATE TABLE IF NOT EXISTS kv ( + id TEXT PRIMARY KEY, + n INTEGER, + flag INTEGER, + blob BLOB + ); + `) + }) + + after(() => { + // best-effort cleanup of the temp DB dir (incl. any -wal/-shm siblings) + fs.rmSync(path.dirname(tmpDir), { recursive: true, force: true }) + }) + + it('creates the parent directory eagerly', () => { + expect(fs.existsSync(tmpDir)).to.equal(true) + }) + + it('run() reports changes: 1 on a hit, 0 on a miss', () => { + const inserted = client.run('INSERT INTO kv (id, n) VALUES (?, ?)', ['a', 1]) + expect(inserted.changes).to.equal(1) + expect(inserted.changes).to.be.a('number') + + const hit = client.run('UPDATE kv SET n = ? WHERE id = ?', [2, 'a']) + expect(hit.changes).to.equal(1) + + const miss = client.run('UPDATE kv SET n = ? WHERE id = ?', [9, 'does-not-exist']) + expect(miss.changes).to.equal(0) + }) + + it('get() returns undefined when no row matches', () => { + const row = client.get('SELECT * FROM kv WHERE id = ?', ['missing']) + expect(row).to.equal(undefined) + }) + + it('all() always returns an array', () => { + const rows = client.all('SELECT * FROM kv WHERE id = ?', ['still-missing']) + expect(rows).to.be.an('array').with.lengthOf(0) + }) + + it('sanitizes undefined -> NULL and boolean -> 1/0 bindings', () => { + // undefined would otherwise throw ERR_INVALID_ARG_TYPE in node:sqlite + client.run('INSERT INTO kv (id, n, flag) VALUES (?, ?, ?)', [ + 'sanitize', + undefined, + true + ]) + const row = client.get<{ n: number | null; flag: number }>( + 'SELECT n, flag FROM kv WHERE id = ?', + ['sanitize'] + ) + expect(row?.n).to.equal(null) + expect(row?.flag).to.equal(1) + + client.run('INSERT INTO kv (id, flag) VALUES (?, ?)', ['sanitize-false', false]) + const row2 = client.get<{ flag: number }>('SELECT flag FROM kv WHERE id = ?', [ + 'sanitize-false' + ]) + expect(row2?.flag).to.equal(0) + }) + + it('round-trips a BLOB written as Buffer (comes back as Uint8Array)', () => { + const original = JSON.stringify({ hello: 'world', n: 123 }) + client.run('INSERT INTO kv (id, blob) VALUES (?, ?)', ['blob', Buffer.from(original)]) + const row = client.get<{ blob: Uint8Array }>('SELECT blob FROM kv WHERE id = ?', [ + 'blob' + ]) + expect(row?.blob).to.be.an.instanceOf(Uint8Array) + // This is exactly the decode the compute provider must do: Buffer.from(blob).toString() + expect(Buffer.from(row!.blob).toString()).to.equal(original) + expect(JSON.parse(Buffer.from(row!.blob).toString())).to.deep.equal({ + hello: 'world', + n: 123 + }) + }) + + it('supports INSERT ... ON CONFLICT upsert (nonce/config pattern)', () => { + const sql = ` + INSERT INTO kv (id, n) VALUES (?, ?) + ON CONFLICT(id) DO UPDATE SET n = excluded.n; + ` + client.run(sql, ['upsert', 10]) + expect( + client.get<{ n: number }>('SELECT n FROM kv WHERE id = ?', ['upsert'])?.n + ).to.equal(10) + client.run(sql, ['upsert', 20]) + expect( + client.get<{ n: number }>('SELECT n FROM kv WHERE id = ?', ['upsert'])?.n + ).to.equal(20) + }) +}) diff --git a/src/test/utils/signature.ts b/src/test/utils/signature.ts index 9706b516f..b4dfda32b 100644 --- a/src/test/utils/signature.ts +++ b/src/test/utils/signature.ts @@ -10,15 +10,19 @@ import { ethers, Signer, JsonRpcSigner } from 'ethers' * @param consumer - The consumer's Ethereum address (hex string) * @param nonce - A unique nonce value (string or number) to prevent replay attacks * @param command - The protocol command being requested (e.g., 'COMPUTE_START') + * @param issuerPeerId - The target node's peerId, binding the signature to that node * @returns A Uint8Array containing the message hash bytes ready for signing * */ export function createHashForSignature( consumer: string, nonce: string | number, - command: string + command: string, + issuerPeerId: string = '' ): Uint8Array { - const message = String(String(consumer) + String(nonce) + String(command)) + const message = String( + String(consumer) + String(nonce) + String(command) + String(issuerPeerId) + ) const consumerMessage = ethers.solidityPackedKeccak256( ['bytes'], [ethers.hexlify(ethers.toUtf8Bytes(message))] diff --git a/src/utils/asset.ts b/src/utils/asset.ts index 897e8283e..0941e9f27 100644 --- a/src/utils/asset.ts +++ b/src/utils/asset.ts @@ -1,4 +1,3 @@ -import axios from 'axios' import { Service, DDOManager, DDO } from '@oceanprotocol/ddo-js' import { DDO_IDENTIFIER_PREFIX } from './constants.js' import { CORE_LOGGER } from './logging/common.js' @@ -10,6 +9,7 @@ import ERC20Template from '@oceanprotocol/contracts/artifacts/contracts/interfac import ERC20Template4 from '@oceanprotocol/contracts/artifacts/contracts/templates/ERC20Template4.sol/ERC20Template4.json' with { type: 'json' } import { getContractAddress, getNFTFactory } from '../components/Indexer/utils.js' import { HeadersObject } from '../@types/fileObject.js' +import { fetchHeadersTimeout } from './http.js' // Notes: // Asset as per asset.py on provider, is a class there, while on ocean.Js we only have a type @@ -55,16 +55,22 @@ export async function fetchFileMetadata( const maxLength = isNaN(maxLengthInt) ? 10 * 1024 * 1024 : maxLengthInt try { - const response = await axios({ + const response = await fetchHeadersTimeout( url, - method: method || 'get', - headers, - responseType: 'stream', - timeout: 30000 - }) - contentType = response.headers['content-type'] + { method: method || 'GET', headers }, + 30000 + ) + if (!response.ok) { + // axios threw on non-2xx before hashing — avoid checksumming an error page. + // cancel the undrained body so undici releases the socket back to the pool. + await response.body?.cancel().catch(() => {}) + throw new Error(`Request failed with status code ${response.status} (${url})`) + } + contentType = response.headers.get('content-type') ?? '' let totalSize = 0 - for await (const chunk of response.data) { + // web ReadableStream is async-iterable in Node 22; chunks are Uint8Array. + // break invokes the iterator's return() which cancels + releases the socket. + for await (const chunk of response.body ?? []) { totalSize += chunk.length contentChecksum.update(chunk) if (totalSize > maxLength && !forceChecksum) { diff --git a/src/utils/config/constants.ts b/src/utils/config/constants.ts index f43d878d6..7ab77eba4 100644 --- a/src/utils/config/constants.ts +++ b/src/utils/config/constants.ts @@ -30,6 +30,7 @@ export const ENV_TO_CONFIG_MAPPING = { ALLOWED_ADMINS: 'allowedAdmins', ALLOWED_ADMINS_LIST: 'allowedAdminsList', DOCKER_COMPUTE_ENVIRONMENTS: 'dockerComputeEnvironments', + SERVICE_TEMPLATES_PATH: 'serviceTemplatesPath', DOCKER_REGISTRY_AUTHS: 'dockerRegistrysAuth', P2P_BOOTSTRAP_NODES: 'p2pConfig.bootstrapNodes', P2P_BOOTSTRAP_TIMEOUT: 'p2pConfig.bootstrapTimeout', diff --git a/src/utils/config/schemas.ts b/src/utils/config/schemas.ts index cfdf57ce5..5b4820384 100644 --- a/src/utils/config/schemas.ts +++ b/src/utils/config/schemas.ts @@ -167,27 +167,269 @@ export const DockerRegistryAuthSchema = z export const DockerRegistrysSchema = z.record(z.string(), DockerRegistryAuthSchema) -const ResourceConstraintSchema = z.object({ - id: z.string(), - min: z.number().optional(), - max: z.number().optional() -}) +// A constraint targets EITHER a single resource by `id` OR a group of resources by `type` +// (aggregated). `perUnit` (default true) keeps the historical ratio semantics +// (requiredMin = parentAmount * min); `perUnit:false` makes min/max an absolute floor/ceiling +// enforced only when the parent resource is requested. +// NOTE: keep `perUnit` optional() with no .default() — a default would inject `perUnit:true` +// into every parsed constraint and break existing deep-equal expectations on constraints. +const ResourceConstraintSchema = z + .object({ + id: z.string().optional(), + type: z.string().optional(), + min: z.number().optional(), + max: z.number().optional(), + perUnit: z.boolean().optional(), + aggregate: z.boolean().optional() + }) + .refine((c) => c.id !== undefined || c.type !== undefined, { + message: 'Each resource constraint must specify either "id" or "type"' + }) + .refine((c) => !(c.id !== undefined && c.type !== undefined), { + message: '"id" and "type" are mutually exclusive in a resource constraint' + }) + .refine((c) => !(c.aggregate === true && c.type !== undefined), { + message: 'aggregate constraints must target a single "id", not a "type" group' + }) -export const ComputeResourceSchema = z.object({ - id: z.string(), - total: z.number().optional(), - description: z.string().optional(), - type: z.string().optional(), - kind: z.string().optional(), - min: z.number().optional(), - max: z.number().optional(), - inUse: z.number().optional(), - init: z.any().optional(), - platform: z.string().optional(), - memoryTotal: z.string().optional(), - driverVersion: z.string().optional(), - constraints: z.array(ResourceConstraintSchema).optional() -}) +// cpuList shape: comma-separated core IDs and/or integer ranges ("3", "0-1,3") — no +// spaces, signs or floats. Every dash-separated piece of a part must be a plain +// integer; the semantic rules (range right > left, parts ascending and +// non-overlapping) are checked in validateCpuList below. +const CPU_ID_REGEX = /^\d+$/ + +export function validateCpuList( + value: string, + ctx: z.RefinementCtx, + path: (string | number)[] +): void { + let prevEnd = -1 + let prevPart = '' + for (const part of value.split(',')) { + const pieces = part.split('-') + if (pieces.length > 2 || !pieces.every((p) => CPU_ID_REGEX.test(p))) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `cpuList "${value}" is invalid: expected comma-separated core IDs and/or integer ranges like "3", "0-1,3" or "0-15,32-47" (spaces, floats and negative values are not allowed)`, + path + }) + return + } + const isRange = pieces.length === 2 + const [start, end] = isRange + ? pieces.map(Number) + : [Number(pieces[0]), Number(pieces[0])] + if (!Number.isSafeInteger(start) || !Number.isSafeInteger(end)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `cpuList range "${part}": core IDs are too large to be valid CPU IDs`, + path + }) + continue + } + if (isRange && end <= start) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `cpuList range "${part}": right side must be strictly greater than left side`, + path + }) + } + if (prevPart && start <= prevEnd) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `cpuList ranges "${prevPart}" and "${part}": ranges must be ascending and non-overlapping`, + path + }) + } + if (end > 8192) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `cpuList range "${part}": core ID exceeds maximum supported limit`, + path + }) + } + prevEnd = Math.max(prevEnd, end) + prevPart = part + } +} + +export const ComputeResourceSchema = z + .object({ + id: z.string(), + total: z.number().optional(), + cpuList: z.string().optional(), + description: z.string().optional(), + type: z.string().optional(), + kind: z.enum(['discrete', 'fungible']).optional(), + shareable: z.boolean().optional(), + min: z.number().optional(), + max: z.number().optional(), + inUse: z.number().optional(), + init: z.any().optional(), + platform: z.string().optional(), + memoryTotal: z.string().optional(), + driverVersion: z.string().optional(), + constraints: z.array(ResourceConstraintSchema).optional() + }) + .superRefine((res, ctx) => { + if (res.cpuList === undefined) return + if (res.id !== 'cpu') { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `Resource "${res.id}": "cpuList" is only valid on the cpu resource`, + path: ['cpuList'] + }) + return + } + if (res.total !== undefined) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `Resource "cpu": specify either "total" or "cpuList", not both`, + path: ['cpuList'] + }) + } + validateCpuList(res.cpuList, ctx, ['cpuList']) + }) + +export const EnvironmentResourceRefSchema = z + .object({ + id: z.string(), + total: z.number().optional(), + min: z.number().optional(), + max: z.number().optional(), + constraints: z.array(ResourceConstraintSchema).optional() + }) + .passthrough() + +// ── Per-environment capability flags ────────────────────────────────── + +const ComputeEnvFeaturesSchema = z + .object({ + computeJobs: z.boolean().optional().default(true), + services: z.boolean().optional().default(true) + }) + .strict() // reject unknown feature keys (catches typos like "computejobs") + +// ── Template resource requirements ──────────────────────────────────── + +const TemplateResourceRequirementSchema = z + .object({ + id: z.string().optional(), + kind: z.enum(['discrete', 'fungible']).optional(), + type: z.string().optional(), + min: z.number().min(0), + recommended: z.number().min(0).optional(), + unit: z.string().optional(), + description: z.string().optional() + }) + .strict() + .refine((r) => r.id !== undefined || r.kind !== undefined, { + message: 'Each resource requirement must specify either "id" or "kind"' + }) + .refine((r) => !(r.id !== undefined && r.kind !== undefined), { + message: '"id" and "kind" are mutually exclusive in a resource requirement' + }) + .refine((r) => r.recommended === undefined || r.recommended >= r.min, { + message: '"recommended" must be >= "min"' + }) + +// ── Template ────────────────────────────────────────────────────────── + +const UserConfigurableEnvVarSchema = z + .object({ + key: z.string().min(1), + validation: z.string().optional(), + sensitive: z.boolean().optional() + }) + .strict() + +export const ServiceTemplateSchema = z + .object({ + id: z.string().regex(/^[a-z0-9][a-z0-9_-]{0,63}$/, { + message: 'Template id must match [a-z0-9][a-z0-9_-]{0,63}' + }), + name: z.string().optional(), + description: z.string().optional(), + image: z.string().min(1), + tag: z.string().min(1).optional(), + checksum: z + .string() + .regex(/^sha256:[a-f0-9]{64}$/) + .optional(), + dockerfile: z.string().min(1).optional(), + additionalDockerFiles: z.record(z.string()).optional(), + exposedPorts: z.array(z.number().int().min(1).max(65535)).min(1), + envVars: z.record(z.string()).optional(), + userConfigurableEnvVars: z.array(UserConfigurableEnvVarSchema).optional(), + command: z.array(z.string()).optional(), + entrypoint: z.array(z.string()).optional(), + requiredResources: z.array(TemplateResourceRequirementSchema).optional(), + recommendedResources: z.array(TemplateResourceRequirementSchema).optional() + }) + .strict() + .superRefine((tmpl, ctx) => { + // Validate each regex in userConfigurableEnvVars.validation compiles + ;(tmpl.userConfigurableEnvVars ?? []).forEach((uvar, i) => { + if (uvar.validation) { + try { + // eslint-disable-next-line no-new + new RegExp(uvar.validation) + } catch { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `userConfigurableEnvVars[${i}].validation is not a valid regex: "${uvar.validation}"`, + path: ['userConfigurableEnvVars', i, 'validation'] + }) + } + } + }) + // Warn on shell-injection-prone command patterns (security plan #3) + ;(tmpl.command ?? []).forEach((arg, i) => { + if (/sh\s+-c|`/.test(arg)) { + CONFIG_LOGGER.warn( + `Template "${tmpl.id}" command[${i}] contains shell invocation. ` + + 'This enables injection when userData values are substituted.' + ) + } + }) + + // Image spec mutual exclusion + const imageModesSet = [!!tmpl.tag, !!tmpl.checksum, !!tmpl.dockerfile].filter( + Boolean + ).length + if (imageModesSet > 1) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: + '"tag", "checksum", and "dockerfile" are mutually exclusive — set at most one', + path: ['image'] + }) + } + if (tmpl.additionalDockerFiles && !tmpl.dockerfile) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: '"additionalDockerFiles" requires "dockerfile"', + path: ['additionalDockerFiles'] + }) + } + }) + +// ── Per-daemon service config (no templates here) ───────────────────── + +export const ServiceOnDemandConfigSchema = z + .object({ + enabled: z.boolean(), + nodeHost: z.string().min(1), + hostPortRange: z + .tuple([z.number().int().min(1024), z.number().int().max(65535)]) + .refine((r) => !r || r[0] < r[1], { + message: 'hostPortRange[0] must be less than hostPortRange[1]' + }) + .optional(), + maxDurationSeconds: z.number().int().min(60).optional().default(86400), + allowImageBuild: z.boolean().optional().default(false) + }) + .strict() export const ComputeResourcesPricingInfoSchema = z.object({ id: z.string(), @@ -216,6 +458,24 @@ export const ComputeEnvironmentFreeOptionsSchema = z.object({ allowImageBuild: z.boolean().optional().default(false) }) +// Config-time schema for the free block — resources are refs, not full ComputeResource objects. +export const C2DEnvironmentFreeConfigSchema = z.object({ + minJobDuration: z.number().int().optional().default(60), + maxJobDuration: z.number().int().optional().default(3600), + maxJobs: z.number().int().optional().default(3), + resources: z.array(EnvironmentResourceRefSchema).optional(), + access: z + .object({ + addresses: z.array(z.string()), + accessLists: z + .array(z.record(z.string(), z.array(z.string()))) + .nullable() + .optional() + }) + .optional(), + allowImageBuild: z.boolean().optional().default(false) +}) + export const C2DEnvironmentConfigSchema = z .object({ id: z.string().optional(), @@ -234,9 +494,13 @@ export const C2DEnvironmentConfigSchema = z .optional() }) .optional(), - free: ComputeEnvironmentFreeOptionsSchema.optional(), - resources: z.array(ComputeResourceSchema).optional(), - enableNetwork: z.boolean().optional().default(false) + free: C2DEnvironmentFreeConfigSchema.optional(), + resources: z.array(EnvironmentResourceRefSchema).optional(), + enableNetwork: z.boolean().optional().default(false), + features: ComputeEnvFeaturesSchema.optional().default({ + computeJobs: true, + services: true + }) }) .refine( (data) => @@ -250,29 +514,109 @@ export const C2DEnvironmentConfigSchema = z .refine((data) => data.storageExpiry >= data.maxJobDuration, { message: '"storageExpiry" should be greater than "maxJobDuration"' }) - .refine( - (data) => { - if (!data.resources) return false - return data.resources.some((r) => r.id === 'disk' && r.total) - }, - { message: 'There is no "disk" resource configured. This is mandatory' } - ) export const C2DDockerConfigSchema = z.array( - z.object({ - socketPath: z.string().optional(), - protocol: z.string().optional(), - host: z.string().optional(), - port: z.number().optional(), - caPath: z.string().optional(), - certPath: z.string().optional(), - keyPath: z.string().optional(), - imageRetentionDays: z.number().int().min(1).optional().default(7), - imageCleanupInterval: z.number().int().min(3600).optional().default(86400), // min 1 hour, default 24 hours - scanImages: z.boolean().optional().default(false), - scanImageDBUpdateInterval: z.number().int().min(3600).optional().default(43200), // default 43200 (12 hours) - environments: z.array(C2DEnvironmentConfigSchema).min(1) - }) + z + .object({ + socketPath: z.string().optional(), + protocol: z.string().optional(), + host: z.string().optional(), + port: z.number().optional(), + caPath: z.string().optional(), + certPath: z.string().optional(), + keyPath: z.string().optional(), + imageRetentionDays: z.number().int().min(1).optional().default(7), + imageCleanupInterval: z.number().int().min(3600).optional().default(86400), + paymentClaimInterval: z.number().int().min(60).optional().default(3600), + scanImages: z.boolean().optional().default(false), + scanImageDBUpdateInterval: z.number().int().min(3600).optional().default(43200), + resources: z.array(ComputeResourceSchema).optional(), + environments: z.array(C2DEnvironmentConfigSchema).min(1), + serviceOnDemand: ServiceOnDemandConfigSchema.optional() + }) + .superRefine((dockerConfig, ctx) => { + // Reject old format: env-level resources with init/driverVersion/platform indicate full ComputeResource objects + // that should have been moved to connection-level resources. + dockerConfig.environments.forEach((env, envIdx) => { + ;(env.resources || []).forEach((ref, i) => { + if ( + (ref as any).init !== undefined || + (ref as any).driverVersion !== undefined + ) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `environments[${envIdx}].resources[${i}]: hardware fields (init, driverVersion, platform, etc.) must be defined at connection level in "resources", not inside an environment. See migration guide.`, + path: ['environments', envIdx, 'resources', i] + }) + } + }) + }) + + // Validate env resource refs point to known pool ids. + // cpu, ram, disk are always valid (auto-detected from host). + const autoDetected = new Set(['cpu', 'ram', 'disk']) + const poolIds = new Set([ + ...autoDetected, + ...(dockerConfig.resources ?? []).map((r) => r.id) + ]) + dockerConfig.environments.forEach((env, envIdx) => { + ;(env.resources || []).forEach((ref, i) => { + if (!poolIds.has(ref.id)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `environments[${envIdx}].resources[${i}].id "${ref.id}" not found in connection-level resources`, + path: ['environments', envIdx, 'resources', i, 'id'] + }) + } + }) + }) + + // Connection-level cpu entry must define its size exactly one way: total or cpuList. + // (Having both is already rejected by ComputeResourceSchema.) + ;(dockerConfig.resources ?? []).forEach((res, i) => { + if (res.id === 'cpu' && res.total === undefined && res.cpuList === undefined) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `resources[${i}]: the cpu resource must specify either "total" or "cpuList"`, + path: ['resources', i] + }) + } + }) + + // cpuList is a connection-level hardware field — reject it in env-level refs, + // same as init/driverVersion above. + dockerConfig.environments.forEach((env, envIdx) => { + ;(env.resources || []).forEach((ref, i) => { + if ((ref as any).cpuList !== undefined) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `environments[${envIdx}].resources[${i}]: "cpuList" must be defined at connection level in "resources", not inside an environment`, + path: ['environments', envIdx, 'resources', i] + }) + } + }) + }) + + // Reject shareable:true on gpu/fpga type resources — these require exclusive access. + ;(dockerConfig.resources ?? []).forEach((res, i) => { + if (res.shareable === true && (res.type === 'gpu' || res.type === 'fpga')) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `Resource "${res.id}": shareable:true is not allowed for type "${res.type}" — GPUs and FPGAs require exclusive access per job`, + path: ['resources', i] + }) + } + }) + + // Warn (not error) if shareable:true on a fungible resource — it has no effect. + ;(dockerConfig.resources ?? []).forEach((res) => { + if (res.shareable === true && res.kind === 'fungible') { + CONFIG_LOGGER.warn( + `Resource "${res.id}": shareable:true has no effect on fungible resources` + ) + } + }) + }) ) export const C2DClusterInfoSchema = z.object({ @@ -363,6 +707,8 @@ export const OceanNodeConfigSchema = z .optional() .default([]), + serviceTemplatesPath: z.string().optional().default('databases/serviceTemplates/'), + dockerRegistrysAuth: jsonFromString(DockerRegistrysSchema).optional().default({}), authorizedDecrypters: addressArrayFromString.optional().default([]), diff --git a/src/utils/constants.ts b/src/utils/constants.ts index 09959d26a..507b1b1bb 100644 --- a/src/utils/constants.ts +++ b/src/utils/constants.ts @@ -36,6 +36,7 @@ export const PROTOCOL_COMMANDS = { FIND_PEER: 'findPeer', CREATE_AUTH_TOKEN: 'createAuthToken', INVALIDATE_AUTH_TOKEN: 'invalidateAuthToken', + VALIDATE_AUTH_TOKEN: 'validateAuthToken', FETCH_CONFIG: 'fetchConfig', PUSH_CONFIG: 'pushConfig', GET_LOGS: 'getLogs', @@ -49,7 +50,15 @@ export const PROTOCOL_COMMANDS = { PERSISTENT_STORAGE_DELETE_FILE: 'persistentStorageDeleteFile', GET_ACCESS_LIST: 'getAccessList', SEARCH_ACCESS_LIST: 'searchAccessList', - GET_ESCROW_EVENTS: 'getEscrowEvents' + GET_ESCROW_EVENTS: 'getEscrowEvents', + SERVICE_GET_TEMPLATES: 'serviceGetTemplates', + SERVICE_START: 'serviceStart', + SERVICE_STOP: 'serviceStop', + SERVICE_RESTART: 'serviceRestart', + SERVICE_GET_STATUS: 'serviceGetStatus', + SERVICE_LIST: 'serviceList', + SERVICE_EXTEND: 'serviceExtend', + SERVICE_GET_STREAMABLE_LOGS: 'serviceGetStreamableLogs' } // more visible, keep then close to make sure we always update both export const SUPPORTED_PROTOCOL_COMMANDS: string[] = [ @@ -87,6 +96,7 @@ export const SUPPORTED_PROTOCOL_COMMANDS: string[] = [ PROTOCOL_COMMANDS.FIND_PEER, PROTOCOL_COMMANDS.CREATE_AUTH_TOKEN, PROTOCOL_COMMANDS.INVALIDATE_AUTH_TOKEN, + PROTOCOL_COMMANDS.VALIDATE_AUTH_TOKEN, PROTOCOL_COMMANDS.FETCH_CONFIG, PROTOCOL_COMMANDS.PUSH_CONFIG, PROTOCOL_COMMANDS.GET_LOGS, @@ -100,7 +110,15 @@ export const SUPPORTED_PROTOCOL_COMMANDS: string[] = [ PROTOCOL_COMMANDS.PERSISTENT_STORAGE_DELETE_FILE, PROTOCOL_COMMANDS.GET_ACCESS_LIST, PROTOCOL_COMMANDS.SEARCH_ACCESS_LIST, - PROTOCOL_COMMANDS.GET_ESCROW_EVENTS + PROTOCOL_COMMANDS.GET_ESCROW_EVENTS, + PROTOCOL_COMMANDS.SERVICE_GET_TEMPLATES, + PROTOCOL_COMMANDS.SERVICE_START, + PROTOCOL_COMMANDS.SERVICE_STOP, + PROTOCOL_COMMANDS.SERVICE_RESTART, + PROTOCOL_COMMANDS.SERVICE_GET_STATUS, + PROTOCOL_COMMANDS.SERVICE_LIST, + PROTOCOL_COMMANDS.SERVICE_EXTEND, + PROTOCOL_COMMANDS.SERVICE_GET_STREAMABLE_LOGS ] export const MetadataStates = { @@ -132,6 +150,7 @@ export const EVENTS = { // Escrow contract events. Values must equal the on-chain event name. ESCROW_AUTH: 'Auth', ESCROW_LOCK: 'Lock', + ESCROW_RELOCK: 'ReLock', ESCROW_CLAIMED: 'Claimed', ESCROW_CANCELED: 'Canceled', ESCROW_DEPOSIT: 'Deposit', @@ -141,6 +160,7 @@ export const EVENTS = { export const ESCROW_EVENTS = [ EVENTS.ESCROW_AUTH, EVENTS.ESCROW_LOCK, + EVENTS.ESCROW_RELOCK, EVENTS.ESCROW_CLAIMED, EVENTS.ESCROW_CANCELED, EVENTS.ESCROW_DEPOSIT, @@ -227,14 +247,18 @@ export const EVENT_HASHES: Hashes = { type: EVENTS.NEW_ACCESS_LIST, text: 'NewAccessList(address,address)' }, - '0x118cb6c6a02e26bfdb39cab8d70573499942c4ee3f0d7616d3c4100fe9163d9d': { + '0x5a3021f46552b1ac3c96e967ff1ecfeb100603ccc2940941cad97db3ee2baec7': { type: EVENTS.ESCROW_AUTH, - text: 'Auth(address,address,uint256,uint256,uint256)' + text: 'Auth(address,address,address,uint256,uint256,uint256)' }, '0xb746b0421b0b98debe76bb312ec9fb701603af22ddb107f7e639b0187e4ff880': { type: EVENTS.ESCROW_LOCK, text: 'Lock(address,address,uint256,uint256,uint256,address)' }, + '0x1ccec59cf72b2788d240da471a5a929af5babf4b9f1f5edafbe4ccb832d20211': { + type: EVENTS.ESCROW_RELOCK, + text: 'ReLock(address,address,uint256,uint256,uint256,uint256,address)' + }, '0x77aeb72af8b0efaf7fd8c746d2fb78653ae489dd88dea7a851cb354e4cdc4eed': { type: EVENTS.ESCROW_CLAIMED, text: 'Claimed(address,uint256,address,address,uint256,bytes)' @@ -472,6 +496,11 @@ export const ENVIRONMENT_VARIABLES: Record = { value: process.env.DOCKER_COMPUTE_ENVIRONMENTS, required: false }, + SERVICE_TEMPLATES_PATH: { + name: 'SERVICE_TEMPLATES_PATH', + value: process.env.SERVICE_TEMPLATES_PATH, + required: false + }, DOCKER_REGISTRY_AUTHS: { name: 'DOCKER_REGISTRY_AUTHS', value: process.env.DOCKER_REGISTRY_AUTHS, diff --git a/src/utils/http.ts b/src/utils/http.ts new file mode 100644 index 000000000..1551f98b9 --- /dev/null +++ b/src/utils/http.ts @@ -0,0 +1,82 @@ +import { Readable } from 'stream' + +/** + * Convert a fetch `Headers` instance into a plain object with lowercase keys, + * matching the shape axios used to expose on `response.headers`. Consumers that + * iterate with `Object.entries(...)` (e.g. downloadHandler) rely on a plain + * object here — `Object.entries(new Headers())` would yield `[]`. + */ +export function headersToObject(headers: Headers): Record { + return Object.fromEntries(headers) +} + +/** + * fetch with axios-like timeout semantics for streaming responses. + * + * The timer guards the connection + response-headers phase only. `fetch()` + * resolves as soon as the response headers arrive (the body may still be + * streaming), so clearing the timer in `finally` means long-running body + * streams (downloads, checksums) are never aborted mid-flight — the same + * behaviour axios had with `responseType: 'stream'` + `timeout`. + * + * Buffered callers that want the timeout to cover the full body should use + * plain `fetch(url, { signal: AbortSignal.timeout(ms) })` instead. + */ +export async function fetchHeadersTimeout( + url: string, + init: RequestInit, + timeoutMs: number +): Promise { + const controller = new AbortController() + const timer = setTimeout( + () => controller.abort(new Error('Headers timeout')), + timeoutMs + ) + try { + return await fetch(url, { ...init, signal: controller.signal }) + } finally { + clearTimeout(timer) + } +} + +/** + * GET/POST a URL and adapt the response to the `StorageReadable` contract: + * a Node `Readable` stream plus a plain lowercase-keyed headers object. + * + * Throws on non-2xx (axios threw on non-2xx by default; every storage caller + * relied on that) and on an empty response body. + */ +export async function fetchStream( + url: string, + init: RequestInit = {}, + timeoutMs = 30000 +): Promise<{ httpStatus: number; stream: Readable; headers: Record }> { + const response = await fetchHeadersTimeout(url, init, timeoutMs) + if (!response.ok) { + // preserve axios's throw-on-non-2xx contract for all callers + await response.body?.cancel().catch(() => {}) + throw new Error(`Request failed with status code ${response.status} (${url})`) + } + if (!response.body) { + throw new Error(`Empty response body (${url})`) + } + const headers = headersToObject(response.headers) + // undici transparently decodes gzip/deflate/br but leaves the original + // content-encoding + (now-wrong, compressed) content-length on the headers. + // The stream we return is already decoded, so strip those stale headers — + // otherwise a consumer re-serving them (e.g. downloadHandler) makes the client + // try to decompress plain bytes ("incorrect header check"). axios's stream + // adapter deleted these on decompress too. + const encoding = headers['content-encoding'] + if (encoding && encoding.toLowerCase() !== 'identity') { + delete headers['content-encoding'] + delete headers['content-length'] + } + return { + httpStatus: response.status, + // `response.body` is typed as the DOM ReadableStream; Node's fromWeb wants + // the node:stream/web ReadableStream — structurally identical, cast locally. + stream: Readable.fromWeb(response.body as any), + headers + } +}