diff --git a/.changeset/fix-d1-json-null-values.md b/.changeset/fix-d1-json-null-values.md new file mode 100644 index 000000000000..8323902d01c0 --- /dev/null +++ b/.changeset/fix-d1-json-null-values.md @@ -0,0 +1,7 @@ +--- +"wrangler": patch +--- + +Fix `wrangler d1 execute --json` returning `"null"` (string) instead of `null` (JSON null) for SQL NULL values + +When using `wrangler d1 execute --json` with local execution, SQL NULL values were incorrectly serialized as the string `"null"` instead of JSON `null`. This produced invalid JSON output that violated RFC 4627. The fix removes the explicit null-to-string conversion so NULL values are preserved as proper JSON null in the output. diff --git a/.changeset/local-explorer-cors.md b/.changeset/local-explorer-cors.md new file mode 100644 index 000000000000..8bc4670562d5 --- /dev/null +++ b/.changeset/local-explorer-cors.md @@ -0,0 +1,7 @@ +--- +"miniflare": patch +--- + +local explorer: validate origin and host headers + +The local explorer is a WIP experimental feature. diff --git a/.changeset/tidy-hairs-notice.md b/.changeset/tidy-hairs-notice.md new file mode 100644 index 000000000000..c0515ec10e2f --- /dev/null +++ b/.changeset/tidy-hairs-notice.md @@ -0,0 +1,8 @@ +--- +"@cloudflare/local-explorer-ui": minor +"miniflare": minor +--- + +Add cross-process support to the local explorer + +When running multiple miniflare processes, the local explorer will now be able to view and edit resources that are bound to workers in other miniflare instances. diff --git a/fixtures/worker-with-resources/package.json b/fixtures/worker-with-resources/package.json index f29bc0c38df4..afd5912b120e 100644 --- a/fixtures/worker-with-resources/package.json +++ b/fixtures/worker-with-resources/package.json @@ -5,6 +5,7 @@ "cf-typegen": "wrangler types --no-include-runtime", "deploy": "wrangler deploy", "start": "X_LOCAL_EXPLORER=true wrangler dev", + "start:worker-b": "X_LOCAL_EXPLORER=true wrangler dev -c worker-b/wrangler.jsonc", "test:ci": "vitest run", "test:watch": "vitest" }, diff --git a/fixtures/worker-with-resources/tests/index.test.ts b/fixtures/worker-with-resources/tests/index.test.ts index fe9efd52c0ef..d26c6e6fb391 100644 --- a/fixtures/worker-with-resources/tests/index.test.ts +++ b/fixtures/worker-with-resources/tests/index.test.ts @@ -44,9 +44,6 @@ describe("local explorer", () => { ], result_info: { count: 2, - page: 1, - per_page: 20, - total_count: 2, }, success: true, }); diff --git a/fixtures/worker-with-resources/worker-b/index.ts b/fixtures/worker-with-resources/worker-b/index.ts new file mode 100644 index 000000000000..21d16b329bcc --- /dev/null +++ b/fixtures/worker-with-resources/worker-b/index.ts @@ -0,0 +1,45 @@ +import { DurableObject } from "cloudflare:workers"; + +export default { + async fetch(request, env) { + const url = new URL(request.url); + switch (url.pathname) { + case "/kv/seed": { + await env.KV_B.put("worker-b-key-1", "value from worker B"); + await env.KV_B.put("worker-b-key-2", "another value from worker B"); + return new Response("Seeded Worker B KV"); + } + case "/d1/seed": { + await env.DB_B.exec(` + DROP TABLE IF EXISTS products; + CREATE TABLE products (id INTEGER PRIMARY KEY, name TEXT, price REAL); + INSERT INTO products (id, name, price) VALUES + (1, 'Widget', 9.99), + (2, 'Gadget', 19.99), + (3, 'Gizmo', 29.99); + `); + return new Response("Seeded Worker B D1"); + } + case "/do/seed": { + const doId = env.DO_B.idFromName("worker-b-do"); + const stub = env.DO_B.get(doId); + return stub.fetch(request); + } + } + return new Response("Hello from Worker B!"); + }, +} satisfies ExportedHandler; + +export class WorkerBDurableObject extends DurableObject { + async fetch(_request: Request): Promise { + this.ctx.storage.sql.exec(` + DROP TABLE IF EXISTS orders; + CREATE TABLE orders (id INTEGER PRIMARY KEY, product_id INTEGER, quantity INTEGER); + INSERT INTO orders (id, product_id, quantity) VALUES + (1, 1, 5), + (2, 2, 3), + (3, 3, 10); + `); + return new Response("Seeded Worker B Durable Object"); + } +} diff --git a/fixtures/worker-with-resources/worker-b/tsconfig.json b/fixtures/worker-with-resources/worker-b/tsconfig.json new file mode 100644 index 000000000000..348bb661fe00 --- /dev/null +++ b/fixtures/worker-with-resources/worker-b/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "@cloudflare/workers-tsconfig/tsconfig.json", + "compilerOptions": { + "types": ["@cloudflare/workers-types"] + }, + "include": ["./**/*.ts"] +} diff --git a/fixtures/worker-with-resources/worker-b/worker-configuration.d.ts b/fixtures/worker-with-resources/worker-b/worker-configuration.d.ts new file mode 100644 index 000000000000..65ba8e89418a --- /dev/null +++ b/fixtures/worker-with-resources/worker-b/worker-configuration.d.ts @@ -0,0 +1,7 @@ +// Generated by Wrangler by running `wrangler types --no-include-runtime` + +interface Env { + KV_B: KVNamespace; + DB_B: D1Database; + DO_B: DurableObjectNamespace; +} diff --git a/fixtures/worker-with-resources/worker-b/wrangler.jsonc b/fixtures/worker-with-resources/worker-b/wrangler.jsonc new file mode 100644 index 000000000000..9ca552d6f67c --- /dev/null +++ b/fixtures/worker-with-resources/worker-b/wrangler.jsonc @@ -0,0 +1,32 @@ +{ + "$schema": "../node_modules/wrangler/config-schema.json", + "name": "worker-b", + "main": "index.ts", + "compatibility_date": "2023-05-04", + "kv_namespaces": [ + { + "binding": "KV_B", + "id": "worker-b-kv-id", + }, + ], + "d1_databases": [ + { + "binding": "DB_B", + "database_id": "worker-b-db-id", + }, + ], + "durable_objects": { + "bindings": [ + { + "name": "DO_B", + "class_name": "WorkerBDurableObject", + }, + ], + }, + "migrations": [ + { + "new_sqlite_classes": ["WorkerBDurableObject"], + "tag": "v1", + }, + ], +} diff --git a/packages/local-explorer-ui/src/components/Sidebar.tsx b/packages/local-explorer-ui/src/components/Sidebar.tsx index 830f1c38e51f..28677af4b5a5 100644 --- a/packages/local-explorer-ui/src/components/Sidebar.tsx +++ b/packages/local-explorer-ui/src/components/Sidebar.tsx @@ -154,7 +154,7 @@ export function Sidebar({ /> { diff --git a/packages/local-explorer-ui/src/drivers/d1.ts b/packages/local-explorer-ui/src/drivers/d1.ts index abcb7d7b8b26..87d4d1763379 100644 --- a/packages/local-explorer-ui/src/drivers/d1.ts +++ b/packages/local-explorer-ui/src/drivers/d1.ts @@ -1,4 +1,4 @@ -import { cloudflareD1RawDatabaseQuery } from "../api"; +import { d1RawDatabaseQuery } from "../api"; import { transformStudioArrayBasedResult } from "../utils/studio"; import { StudioSQLiteDriver } from "./sqlite"; import type { D1RawResultResponse } from "../api"; @@ -52,7 +52,7 @@ export class LocalD1Connection implements IStudioConnection { s.trim().replace(/;+$/, "") ); - const response = await cloudflareD1RawDatabaseQuery({ + const response = await d1RawDatabaseQuery({ body: { sql: trimmedStatements.join(";"), }, diff --git a/packages/local-explorer-ui/src/routes/__root.tsx b/packages/local-explorer-ui/src/routes/__root.tsx index 2d0b36f57e37..cc71b38c1826 100644 --- a/packages/local-explorer-ui/src/routes/__root.tsx +++ b/packages/local-explorer-ui/src/routes/__root.tsx @@ -4,7 +4,7 @@ import { useRouterState, } from "@tanstack/react-router"; import { - cloudflareD1ListDatabases, + d1ListDatabases, durableObjectsNamespaceListNamespaces, workersKvNamespaceListNamespaces, } from "../api"; @@ -20,7 +20,7 @@ export const Route = createRootRoute({ loader: async () => { const [kvResponse, d1Response, doResponse] = await Promise.allSettled([ workersKvNamespaceListNamespaces(), - cloudflareD1ListDatabases(), + d1ListDatabases(), durableObjectsNamespaceListNamespaces(), ]); diff --git a/packages/miniflare/scripts/filter-openapi.ts b/packages/miniflare/scripts/filter-openapi.ts index 99333b79d074..648a0b449c7b 100644 --- a/packages/miniflare/scripts/filter-openapi.ts +++ b/packages/miniflare/scripts/filter-openapi.ts @@ -98,10 +98,19 @@ export interface RequestBodyIgnore { properties: string[]; } +export interface ResponsePropertyIgnore { + path: string; + method: string; + /** Dot-notation path to the property to remove, e.g. "result_info.total_count" */ + propertyPath: string; +} + export interface IgnoresConfig { parameters?: ParameterIgnore[]; requestBodyProperties?: RequestBodyIgnore[]; schemaProperties?: Record; + /** Properties to remove from inline response schemas */ + responseProperties?: ResponsePropertyIgnore[]; } interface OpenAPIOperation { parameters?: Array<{ name: string; [key: string]: unknown }>; @@ -168,6 +177,9 @@ function filterOpenAPISpec( // Apply request body ignores (if any) applyRequestBodyIgnores(operation, originalPath, method, ignores); + // Apply response property ignores (if any) + applyResponsePropertyIgnores(operation, originalPath, method, ignores); + // Remove security from operation since we implement that differently locally delete operation.security; @@ -422,15 +434,38 @@ function removeSchemaProperties( schema: OpenAPISchema, propsToRemove: string[] ): void { - if (!schema.properties) { - return; + // Handle direct properties + if (schema.properties) { + const props = schema.properties; + for (const prop of propsToRemove) { + delete props[prop]; + } + if (schema.required) { + schema.required = schema.required.filter( + (r) => !propsToRemove.includes(r) + ); + } } - const props = schema.properties; - for (const prop of propsToRemove) { - delete props[prop]; + + // Handle allOf - recurse into each sub-schema + if (Array.isArray(schema.allOf)) { + for (const subSchema of schema.allOf) { + removeSchemaProperties(subSchema as OpenAPISchema, propsToRemove); + } + } + + // Handle oneOf + if (Array.isArray(schema.oneOf)) { + for (const subSchema of schema.oneOf) { + removeSchemaProperties(subSchema as OpenAPISchema, propsToRemove); + } } - if (schema.required) { - schema.required = schema.required.filter((r) => !propsToRemove.includes(r)); + + // Handle anyOf + if (Array.isArray(schema.anyOf)) { + for (const subSchema of schema.anyOf) { + removeSchemaProperties(subSchema as OpenAPISchema, propsToRemove); + } } } @@ -480,6 +515,110 @@ function applySchemaIgnores( } } +/** + * Recursively find and process all inline schemas in response content. + * Applies property removal at specified paths. + */ +function applyResponsePropertyIgnores( + operation: OpenAPIOperation, + path: string, + method: string, + ignores: IgnoresConfig +): void { + if (!ignores.responseProperties) { + return; + } + + const pathIgnores = ignores.responseProperties.filter( + (p) => p.path === path && p.method === method + ); + + if (pathIgnores.length === 0) { + return; + } + + const responses = operation.responses as + | Record }> + | undefined; + if (!responses) { + return; + } + + for (const response of Object.values(responses)) { + if (!response.content) { + continue; + } + for (const mediaType of Object.values(response.content)) { + if (!mediaType.schema) { + continue; + } + // Walk through the schema and apply ignores + for (const ignore of pathIgnores) { + removePropertyFromSchema(mediaType.schema, ignore.propertyPath); + } + } + } +} + +/** + * Remove a property from an OpenAPI schema at a dot-notation path. + * Handles allOf/oneOf/anyOf and nested properties. + * E.g., "result_info.total_count" removes total_count from result_info's properties. + */ +function removePropertyFromSchema(schema: unknown, propertyPath: string): void { + if (!schema || typeof schema !== "object") { + return; + } + + const schemaObj = schema as Record; + + // Handle allOf - search in each sub-schema + if (Array.isArray(schemaObj.allOf)) { + for (const subSchema of schemaObj.allOf) { + removePropertyFromSchema(subSchema, propertyPath); + } + } + + // Handle oneOf + if (Array.isArray(schemaObj.oneOf)) { + for (const subSchema of schemaObj.oneOf) { + removePropertyFromSchema(subSchema, propertyPath); + } + } + + // Handle anyOf + if (Array.isArray(schemaObj.anyOf)) { + for (const subSchema of schemaObj.anyOf) { + removePropertyFromSchema(subSchema, propertyPath); + } + } + + // Handle direct properties + if (schemaObj.properties && typeof schemaObj.properties === "object") { + const props = schemaObj.properties as Record; + const parts = propertyPath.split("."); + + if (parts.length === 1) { + // Simple property removal + delete props[parts[0]]; + // Also remove from required array if present + if (Array.isArray(schemaObj.required)) { + schemaObj.required = (schemaObj.required as string[]).filter( + (r) => r !== parts[0] + ); + } + } else { + // Nested property removal: e.g., "result_info.total_count" + // Navigate to the parent property's schema and remove from its properties + const [parentProp, ...rest] = parts; + const parentSchema = props[parentProp] as Record; + if (parentSchema) { + removePropertyFromSchema(parentSchema, rest.join(".")); + } + } + } +} + function removeAccountPathParam(path: string): string { return path.replace("/accounts/{account_id}", ""); } diff --git a/packages/miniflare/scripts/openapi-filter-config.ts b/packages/miniflare/scripts/openapi-filter-config.ts index b9efff923223..06ebfdd6e93c 100644 --- a/packages/miniflare/scripts/openapi-filter-config.ts +++ b/packages/miniflare/scripts/openapi-filter-config.ts @@ -49,6 +49,39 @@ const config = { ignores: { // Query/path parameters not implemented parameters: [ + // List KV namespaces - pagination not implemented (aggregated endpoint returns all) + { + path: "/accounts/{account_id}/storage/kv/namespaces", + method: "get", + name: "page", + }, + { + path: "/accounts/{account_id}/storage/kv/namespaces", + method: "get", + name: "per_page", + }, + // List D1 databases - pagination not implemented (aggregated endpoint returns all) + { + path: "/accounts/{account_id}/d1/database", + method: "get", + name: "page", + }, + { + path: "/accounts/{account_id}/d1/database", + method: "get", + name: "per_page", + }, + // List DO namespaces - pagination not implemented (aggregated endpoint returns all) + { + path: "/accounts/{account_id}/workers/durable_objects/namespaces", + method: "get", + name: "page", + }, + { + path: "/accounts/{account_id}/workers/durable_objects/namespaces", + method: "get", + name: "per_page", + }, // Put value - expiration options not implemented { path: "/accounts/{account_id}/storage/kv/namespaces/{namespace_id}/values/{key_name}", @@ -86,7 +119,34 @@ const config = { "served_by_primary", "served_by_region", ], + // Aggregated list endpoints don't use pagination, so total_count is redundant + // (it always equals count since we return all results) + // minor hack: we don't remove the type for DO's (WorkersApiResponseCollection) since it + // is used by object and namespace listing, and we do need pagination for object listing + // but thankfully the pagination fields are optional so we just pretend they are not there + "workers-kv_api-response-collection": ["total_count", "page", "per_page"], + "workers-kv_result_info": ["total_count", "page", "per_page"], }, + + // Properties to remove from inline response schemas (not in named schemas) + responseProperties: [ + // D1 list databases has inline result_info with pagination fields + { + path: "/accounts/{account_id}/d1/database", + method: "get", + propertyPath: "result_info.total_count", + }, + { + path: "/accounts/{account_id}/d1/database", + method: "get", + propertyPath: "result_info.page", + }, + { + path: "/accounts/{account_id}/d1/database", + method: "get", + propertyPath: "result_info.per_page", + }, + ], }, // Local-only extensions (not in upstream Cloudflare API) diff --git a/packages/miniflare/src/index.ts b/packages/miniflare/src/index.ts index b24c49d50407..2544a2c46b76 100644 --- a/packages/miniflare/src/index.ts +++ b/packages/miniflare/src/index.ts @@ -113,7 +113,7 @@ import { parseWithRootPath, stripAnsi, } from "./shared"; -import { DevRegistry, WorkerDefinition } from "./shared/dev-registry"; +import { DevRegistry, getWorkerRegistry } from "./shared/dev-registry"; import { createInboundDoProxyService, createOutboundDoProxyService, @@ -139,6 +139,7 @@ import { } from "./workers"; import { ADMIN_API } from "./workers/secrets-store/constants"; import { formatZodError } from "./zod-format"; +import type { WorkerDefinition } from "./shared/dev-registry-types"; import type { CacheStorage, D1Database, @@ -1346,6 +1347,11 @@ export class Miniflare { response = new Response(filePath, { status: 200 }); } else if (url.pathname.startsWith("/core/do-storage/")) { response = await this.#handleLoopbackDOStorageRequest(url); + } else if (url.pathname === "/core/dev-registry") { + // Used by the local explorer to aggregate resources across instances + const registryPath = this.#devRegistry.getRegistryPath(); + const registry = registryPath ? getWorkerRegistry(registryPath) : {}; + response = Response.json(registry); } } catch (e: any) { this.#log.error(e); @@ -2823,9 +2829,11 @@ export * from "./shared"; export * from "./workers"; export * from "./merge"; export * from "./zod-format"; +export type { + WorkerRegistry, + WorkerDefinition, +} from "./shared/dev-registry-types"; export { - type WorkerRegistry, - type WorkerDefinition, getDefaultDevRegistryPath, getWorkerRegistry, } from "./shared/dev-registry"; diff --git a/packages/miniflare/src/plugins/core/explorer.ts b/packages/miniflare/src/plugins/core/explorer.ts index 4e3b105e5b60..99e5d0947640 100644 --- a/packages/miniflare/src/plugins/core/explorer.ts +++ b/packages/miniflare/src/plugins/core/explorer.ts @@ -20,6 +20,7 @@ export interface ExplorerServicesOptions { proxyBindings: Worker_Binding[]; bindingIdMap: BindingIdMap; hasDurableObjects: boolean; + workerNames: string[]; } /** @@ -33,6 +34,7 @@ export function getExplorerServices( proxyBindings, bindingIdMap, hasDurableObjects, + workerNames, } = options; const explorerBindings: Worker_Binding[] = [ @@ -46,14 +48,18 @@ export function getExplorerServices( name: CoreBindings.EXPLORER_DISK, service: { name: LOCAL_EXPLORER_DISK }, }, + // Loopback service for accessing Node.js endpoints: + // - /core/dev-registry for cross-instance aggregation + // - /core/do-storage for using DO storage to list objects + WORKER_BINDING_SERVICE_LOOPBACK, + // Worker names for this instance, used to filter self from registry during aggregation + { + name: CoreBindings.JSON_LOCAL_EXPLORER_WORKER_NAMES, + json: JSON.stringify(workerNames), + }, ]; if (hasDurableObjects) { - // Add loopback service binding if DOs are configured - // The explorer worker uses this to call the /core/do-storage endpoint - // which reads the filesystem using Node.js (bypassing workerd disk service issues on Windows) - explorerBindings.push(WORKER_BINDING_SERVICE_LOOPBACK); - // Add Durable Object namespace bindings for the explorer // Yes we are binding to 'unbound' DOs, but that has no effect // on the user's access via ctx.exports diff --git a/packages/miniflare/src/plugins/core/index.ts b/packages/miniflare/src/plugins/core/index.ts index bab0e67c0d3a..2b3db8446c78 100644 --- a/packages/miniflare/src/plugins/core/index.ts +++ b/packages/miniflare/src/plugins/core/index.ts @@ -77,7 +77,7 @@ import { kCurrentWorker, ServiceDesignatorSchema, } from "./services"; -import type { WorkerRegistry } from "../../shared/dev-registry"; +import type { WorkerRegistry } from "../../shared/dev-registry-types"; import type { BindingIdMap } from "./types"; // `workerd`'s `trustBrowserCas` should probably be named `trustSystemCas`. @@ -1144,9 +1144,6 @@ export function getGlobalServices({ proxyBindings, durableObjectClassNames ); - // Add loopback service binding if DOs are configured - // The explorer worker uses this to call the /core/do-storage endpoint - // which reads the filesystem using Node.js (bypassing workerd disk service issues on Windows) const hasDurableObjects = Object.keys(IDToBindingMap.do).length > 0; services.push( ...getExplorerServices({ @@ -1154,6 +1151,7 @@ export function getGlobalServices({ proxyBindings, bindingIdMap: IDToBindingMap, hasDurableObjects, + workerNames, }) ); } diff --git a/packages/miniflare/src/shared/dev-registry-types.ts b/packages/miniflare/src/shared/dev-registry-types.ts new file mode 100644 index 000000000000..079abf258e8e --- /dev/null +++ b/packages/miniflare/src/shared/dev-registry-types.ts @@ -0,0 +1,12 @@ +export type WorkerRegistry = Record; + +export type WorkerDefinition = { + protocol: "http" | "https"; + host: string; + port: number; + entrypointAddresses: Record< + "default" | string, + { host: string; port: number } | undefined + >; + durableObjects: { name: string; className: string }[]; +}; diff --git a/packages/miniflare/src/shared/dev-registry.ts b/packages/miniflare/src/shared/dev-registry.ts index d2f6a349fe61..229655dcf365 100644 --- a/packages/miniflare/src/shared/dev-registry.ts +++ b/packages/miniflare/src/shared/dev-registry.ts @@ -15,19 +15,7 @@ import { SocketPorts } from "../runtime"; import { getProxyFallbackServiceSocketName } from "./external-service"; import { Log } from "./log"; import { getGlobalWranglerConfigPath } from "./wrangler"; - -export type WorkerRegistry = Record; - -export type WorkerDefinition = { - protocol: "http" | "https"; - host: string; - port: number; - entrypointAddresses: Record< - "default" | string, - { host: string; port: number } | undefined - >; - durableObjects: { name: string; className: string }[]; -}; +import type { WorkerDefinition, WorkerRegistry } from "./dev-registry-types"; export class DevRegistry { private heartbeats = new Map(); @@ -173,6 +161,10 @@ export class DevRegistry { return this.isEnabled() && this.enableDurableObjectProxy; } + public getRegistryPath(): string | undefined { + return this.registryPath; + } + public async updateRegistryPath( registryPath: string | undefined, enableDurableObjectProxy: boolean, diff --git a/packages/miniflare/src/shared/dev-registry.worker.ts b/packages/miniflare/src/shared/dev-registry.worker.ts index 4b9a5a3bd5bb..9e7da144dbd8 100644 --- a/packages/miniflare/src/shared/dev-registry.worker.ts +++ b/packages/miniflare/src/shared/dev-registry.worker.ts @@ -10,7 +10,7 @@ import { INBOUND_DO_PROXY_SERVICE_PATH, } from "../shared/external-service"; import { Log } from "./log"; -import type { WorkerDefinition } from "../shared/dev-registry"; +import type { WorkerDefinition } from "../shared/dev-registry-types"; import type { MessagePort } from "node:worker_threads"; interface ProxyAddress { diff --git a/packages/miniflare/src/workers/core/constants.ts b/packages/miniflare/src/workers/core/constants.ts index cb2d057c44ab..865a45cfb392 100644 --- a/packages/miniflare/src/workers/core/constants.ts +++ b/packages/miniflare/src/workers/core/constants.ts @@ -47,6 +47,7 @@ export const CoreBindings = { SERVICE_LOCAL_EXPLORER: "MINIFLARE_LOCAL_EXPLORER", EXPLORER_DISK: "MINIFLARE_EXPLORER_DISK", JSON_LOCAL_EXPLORER_BINDING_MAP: "LOCAL_EXPLORER_BINDING_MAP", + JSON_LOCAL_EXPLORER_WORKER_NAMES: "LOCAL_EXPLORER_WORKER_NAMES", SERVICE_CACHE: "MINIFLARE_CACHE", } as const; diff --git a/packages/miniflare/src/workers/local-explorer/aggregation.ts b/packages/miniflare/src/workers/local-explorer/aggregation.ts new file mode 100644 index 000000000000..e63d13cb0d1d --- /dev/null +++ b/packages/miniflare/src/workers/local-explorer/aggregation.ts @@ -0,0 +1,109 @@ +/** + * Utilities for cross-instance aggregation in the local explorer. + * + * When multiple Miniflare instances run with a shared dev registry, + * any one instance can aggregate data from all instances. + */ + +import { LOCAL_EXPLORER_API_PATH } from "../../plugins/core/constants"; +import type { WorkerRegistry } from "../../shared/dev-registry-types"; +import type { AppContext } from "./common"; + +/** + * Header that indicates a request should not trigger further aggregation. + * Used to prevent infinite recursion when instance A fetches from instance B. + */ +const NO_AGGREGATE_HEADER = "X-Miniflare-Explorer-No-Aggregate"; + +/** + * Get the unique base URLs of peer instances from the dev registry, + * excluding the current instance (identified by worker names). + */ +function getPeerUrls( + registry: WorkerRegistry, + selfWorkerNames: string[] +): string[] { + const selfSet = new Set(selfWorkerNames); + const urls = Object.entries(registry) + .filter(([name]) => !selfSet.has(name)) + .map(([, def]) => `${def.protocol}://${def.host}:${def.port}`); + // A single Miniflare process with multiple workers registers multiple + // entries in the registry, all sharing the same host:port. We deduplicate + // to avoid fetching from the same peer multiple times. + return [...new Set(urls)]; +} + +export async function getPeerUrlsIfAggregating( + c: AppContext +): Promise { + if (c.req.raw.headers.has(NO_AGGREGATE_HEADER)) { + return []; + } + const loopback = c.env.MINIFLARE_LOOPBACK; + const workerNames = c.env.LOCAL_EXPLORER_WORKER_NAMES; + const response = await loopback.fetch("http://localhost/core/dev-registry"); + const registry = (await response.json()) as WorkerRegistry; + return getPeerUrls(registry, workerNames); +} + +/** + * Fetch data from a peer instance's explorer API. + * Returns null on any error (silent omission policy). + * + * @param peerUrl - Base URL of the peer instance (e.g., "http://127.0.0.1:8788") + * @param apiPath - API path relative to the explorer API base (e.g., "/d1/database") + * @param init - Optional fetch init options + */ +export async function fetchFromPeer( + peerUrl: string, + apiPath: string, + init?: RequestInit +): Promise { + try { + const url = new URL(`${LOCAL_EXPLORER_API_PATH}${apiPath}`, peerUrl); + const response = await fetch(url.toString(), { + ...init, + headers: { + ...(init?.headers as Record | undefined), + [NO_AGGREGATE_HEADER]: "true", + Host: "localhost", + }, + }); + return response; + } catch { + return null; + } +} + +/** + * Aggregate list results from local data and peer instances. + * + * @param c - Hono app context + * @param localResults - Results from the local instance + * @param apiPath - API path relative to the explorer API base + */ +export async function aggregateListResults( + c: AppContext, + localResults: T[], + apiPath: string +): Promise { + const peerUrls = await getPeerUrlsIfAggregating(c); + if (peerUrls.length === 0) { + return localResults; + } + + const peerResults = await Promise.all( + peerUrls.map(async (url) => { + const response = await fetchFromPeer(url, apiPath); + if (!response?.ok) return []; + try { + const data = (await response.json()) as { result: T[] }; + return Array.isArray(data.result) ? data.result : []; + } catch { + return []; + } + }) + ); + + return [...localResults, ...peerResults.flat()]; +} diff --git a/packages/miniflare/src/workers/local-explorer/explorer.worker.ts b/packages/miniflare/src/workers/local-explorer/explorer.worker.ts index fa11709944f7..1e19f99731c4 100644 --- a/packages/miniflare/src/workers/local-explorer/explorer.worker.ts +++ b/packages/miniflare/src/workers/local-explorer/explorer.worker.ts @@ -10,9 +10,8 @@ import { import { CoreBindings } from "../core"; import { errorResponse, validateQuery, validateRequestBody } from "./common"; import { - zCloudflareD1ListDatabasesData, - zCloudflareD1RawDatabaseQueryData, - zDurableObjectsNamespaceListNamespacesData, + zD1ListDatabasesData, + zD1RawDatabaseQueryData, zDurableObjectsNamespaceListObjectsData, zDurableObjectsNamespaceQuerySqliteData, zWorkersKvNamespaceGetMultipleKeyValuePairsData, @@ -35,8 +34,12 @@ export type Env = { [key: string]: unknown; [CoreBindings.JSON_LOCAL_EXPLORER_BINDING_MAP]: BindingIdMap; [CoreBindings.EXPLORER_DISK]: Fetcher; - // Loopback service for calling Node.js endpoints (used for DO storage listing) - [CoreBindings.SERVICE_LOOPBACK]?: Fetcher; + // Loopback service for calling Node.js endpoints: + // - /core/dev-registry for cross-instance aggregation + // - /core/do-storage for DO storage listing + [CoreBindings.SERVICE_LOOPBACK]: Fetcher; + // Worker names for this instance, used to filter self from dev registry during aggregation + [CoreBindings.JSON_LOCAL_EXPLORER_WORKER_NAMES]: string[]; }; export type AppBindings = { Bindings: Env }; @@ -48,6 +51,61 @@ app.onError((err) => { return errorResponse(500, 10000, err.message); }); +// ============================================================================ +// Middleware +// ============================================================================ + +app.use("/api/*", async (c, next) => { + const ALLOWED_HOSTNAMES = ["localhost", "127.0.0.1", "[::1]"]; + + const hostHeader = c.req.header("Host"); + try { + const host = hostHeader ? new URL(`http://${hostHeader}`).hostname : ""; + if (!ALLOWED_HOSTNAMES.includes(host)) { + return errorResponse(403, 10000, "Invalid Host header"); + } + } catch { + return errorResponse(403, 10000, "Invalid Host header"); + } + + const origin = c.req.header("Origin"); + if (origin) { + try { + if (!ALLOWED_HOSTNAMES.includes(new URL(origin).hostname)) { + return errorResponse( + 403, + 10000, + "Cross-origin requests to the local explorer API are not allowed" + ); + } + } catch { + return errorResponse( + 403, + 10000, + "Cross-origin requests to the local explorer API are not allowed" + ); + } + } + + if (c.req.method === "OPTIONS") { + return new Response(null, { + status: 204, + headers: { + "Access-Control-Allow-Origin": origin ?? "*", + "Access-Control-Allow-Methods": "GET, POST, PUT, DELETE, OPTIONS", + "Access-Control-Allow-Headers": "Content-Type", + "Access-Control-Max-Age": "86400", + }, + }); + } + + await next(); + + if (origin) { + c.res.headers.set("Access-Control-Allow-Origin", origin); + } +}); + // ============================================================================ // Static Asset Serving // ============================================================================ @@ -151,13 +209,13 @@ app.post( app.get( "/api/d1/database", - validateQuery(zCloudflareD1ListDatabasesData.shape.query.unwrap()), + validateQuery(zD1ListDatabasesData.shape.query.unwrap()), (c) => listD1Databases(c, c.req.valid("query")) ); app.post( "/api/d1/database/:database_id/raw", - validateRequestBody(zCloudflareD1RawDatabaseQueryData.shape.body), + validateRequestBody(zD1RawDatabaseQueryData.shape.body), (c) => rawD1Database(c, c.req.valid("json")) ); @@ -165,13 +223,7 @@ app.post( // Durable Objects Endpoints // ============================================================================ -app.get( - "/api/workers/durable_objects/namespaces", - validateQuery( - zDurableObjectsNamespaceListNamespacesData.shape.query.unwrap() - ), - (c) => listDONamespaces(c, c.req.valid("query")) -); +app.get("/api/workers/durable_objects/namespaces", (c) => listDONamespaces(c)); app.get( "/api/workers/durable_objects/namespaces/:namespace_id/objects", diff --git a/packages/miniflare/src/workers/local-explorer/generated/index.ts b/packages/miniflare/src/workers/local-explorer/generated/index.ts index 98b5b17ea4fa..0d8da4ca8191 100644 --- a/packages/miniflare/src/workers/local-explorer/generated/index.ts +++ b/packages/miniflare/src/workers/local-explorer/generated/index.ts @@ -2,16 +2,6 @@ export type { ClientOptions, - CloudflareD1ListDatabasesData, - CloudflareD1ListDatabasesError, - CloudflareD1ListDatabasesErrors, - CloudflareD1ListDatabasesResponse, - CloudflareD1ListDatabasesResponses, - CloudflareD1RawDatabaseQueryData, - CloudflareD1RawDatabaseQueryError, - CloudflareD1RawDatabaseQueryErrors, - CloudflareD1RawDatabaseQueryResponse, - CloudflareD1RawDatabaseQueryResponses, D1ApiResponseCommon, D1ApiResponseCommonFailure, D1BatchQuery, @@ -20,9 +10,19 @@ export type { D1DatabaseResponse, D1DatabaseResponseWritable, D1DatabaseVersion, + D1ListDatabasesData, + D1ListDatabasesError, + D1ListDatabasesErrors, + D1ListDatabasesResponse, + D1ListDatabasesResponses, D1Messages, D1Params, D1QueryMeta, + D1RawDatabaseQueryData, + D1RawDatabaseQueryError, + D1RawDatabaseQueryErrors, + D1RawDatabaseQueryResponse, + D1RawDatabaseQueryResponses, D1RawResultResponse, D1SingleQuery, D1Sql, diff --git a/packages/miniflare/src/workers/local-explorer/generated/types.gen.ts b/packages/miniflare/src/workers/local-explorer/generated/types.gen.ts index d07a4009f2e9..d587491083bf 100644 --- a/packages/miniflare/src/workers/local-explorer/generated/types.gen.ts +++ b/packages/miniflare/src/workers/local-explorer/generated/types.gen.ts @@ -352,18 +352,6 @@ export type WorkersKvResultInfo = { * Total number of results for the requested service. */ count?: number; - /** - * Current page within paginated list of results. - */ - page?: number; - /** - * Number of results per page of results. - */ - per_page?: number; - /** - * Total results available without any search parameters. - */ - total_count?: number; }; export type DoSqlWithParams = { @@ -455,14 +443,6 @@ export type WorkersKvNamespaceListNamespacesData = { body?: never; path?: never; query?: { - /** - * Page number of paginated results. - */ - page?: number; - /** - * Maximum number of results per page. - */ - per_page?: number; /** * Field to order results by. */ @@ -668,7 +648,7 @@ export type WorkersKvNamespaceGetMultipleKeyValuePairsResponses = { export type WorkersKvNamespaceGetMultipleKeyValuePairsResponse = WorkersKvNamespaceGetMultipleKeyValuePairsResponses[keyof WorkersKvNamespaceGetMultipleKeyValuePairsResponses]; -export type CloudflareD1ListDatabasesData = { +export type D1ListDatabasesData = { body?: never; path?: never; query?: { @@ -676,29 +656,21 @@ export type CloudflareD1ListDatabasesData = { * a database name to search for. */ name?: string; - /** - * Page number of paginated results. - */ - page?: number; - /** - * Number of items per page. - */ - per_page?: number; }; url: "/d1/database"; }; -export type CloudflareD1ListDatabasesErrors = { +export type D1ListDatabasesErrors = { /** * List D1 databases response failure */ "4XX": D1ApiResponseCommonFailure; }; -export type CloudflareD1ListDatabasesError = - CloudflareD1ListDatabasesErrors[keyof CloudflareD1ListDatabasesErrors]; +export type D1ListDatabasesError = + D1ListDatabasesErrors[keyof D1ListDatabasesErrors]; -export type CloudflareD1ListDatabasesResponses = { +export type D1ListDatabasesResponses = { /** * List D1 databases response */ @@ -709,26 +681,14 @@ export type CloudflareD1ListDatabasesResponses = { * Total number of results for the requested service */ count?: number; - /** - * Current page within paginated list of results - */ - page?: number; - /** - * Number of results per page of results - */ - per_page?: number; - /** - * Total results available without any search parameters - */ - total_count?: number; }; }; }; -export type CloudflareD1ListDatabasesResponse = - CloudflareD1ListDatabasesResponses[keyof CloudflareD1ListDatabasesResponses]; +export type D1ListDatabasesResponse = + D1ListDatabasesResponses[keyof D1ListDatabasesResponses]; -export type CloudflareD1RawDatabaseQueryData = { +export type D1RawDatabaseQueryData = { body: D1BatchQuery; path: { database_id: D1DatabaseIdentifier; @@ -737,17 +697,17 @@ export type CloudflareD1RawDatabaseQueryData = { url: "/d1/database/{database_id}/raw"; }; -export type CloudflareD1RawDatabaseQueryErrors = { +export type D1RawDatabaseQueryErrors = { /** * Query response failure */ "4XX": D1ApiResponseCommonFailure; }; -export type CloudflareD1RawDatabaseQueryError = - CloudflareD1RawDatabaseQueryErrors[keyof CloudflareD1RawDatabaseQueryErrors]; +export type D1RawDatabaseQueryError = + D1RawDatabaseQueryErrors[keyof D1RawDatabaseQueryErrors]; -export type CloudflareD1RawDatabaseQueryResponses = { +export type D1RawDatabaseQueryResponses = { /** * Raw query response */ @@ -756,22 +716,13 @@ export type CloudflareD1RawDatabaseQueryResponses = { }; }; -export type CloudflareD1RawDatabaseQueryResponse = - CloudflareD1RawDatabaseQueryResponses[keyof CloudflareD1RawDatabaseQueryResponses]; +export type D1RawDatabaseQueryResponse = + D1RawDatabaseQueryResponses[keyof D1RawDatabaseQueryResponses]; export type DurableObjectsNamespaceListNamespacesData = { body?: never; path?: never; - query?: { - /** - * Current page. - */ - page?: number; - /** - * Items per-page. - */ - per_page?: number; - }; + query?: never; url: "/workers/durable_objects/namespaces"; }; diff --git a/packages/miniflare/src/workers/local-explorer/generated/zod.gen.ts b/packages/miniflare/src/workers/local-explorer/generated/zod.gen.ts index a0d21080689a..3c7abbcc56b0 100644 --- a/packages/miniflare/src/workers/local-explorer/generated/zod.gen.ts +++ b/packages/miniflare/src/workers/local-explorer/generated/zod.gen.ts @@ -284,9 +284,6 @@ export const zWorkersKvNamespace = z.object({ export const zWorkersKvResultInfo = z.object({ count: z.number().optional(), - page: z.number().optional(), - per_page: z.number().optional(), - total_count: z.number().optional(), }); export const zWorkersKvApiResponseCollection = zWorkersKvApiResponseCommon.and( @@ -359,8 +356,6 @@ export const zWorkersKvNamespaceListNamespacesData = z.object({ path: z.never().optional(), query: z .object({ - page: z.number().gte(1).optional().default(1), - per_page: z.number().gte(1).lte(1000).optional().default(20), order: z.enum(["id", "title"]).optional(), direction: z.enum(["asc", "desc"]).optional(), }) @@ -468,14 +463,12 @@ export const zWorkersKvNamespaceGetMultipleKeyValuePairsResponse = }) ); -export const zCloudflareD1ListDatabasesData = z.object({ +export const zD1ListDatabasesData = z.object({ body: z.never().optional(), path: z.never().optional(), query: z .object({ name: z.string().optional(), - page: z.number().gte(1).optional().default(1), - per_page: z.number().gte(10).lte(10000).optional().default(1000), }) .optional(), }); @@ -483,21 +476,18 @@ export const zCloudflareD1ListDatabasesData = z.object({ /** * List D1 databases response */ -export const zCloudflareD1ListDatabasesResponse = zD1ApiResponseCommon.and( +export const zD1ListDatabasesResponse = zD1ApiResponseCommon.and( z.object({ result: z.array(zD1DatabaseResponse).optional(), result_info: z .object({ count: z.number().optional(), - page: z.number().optional(), - per_page: z.number().optional(), - total_count: z.number().optional(), }) .optional(), }) ); -export const zCloudflareD1RawDatabaseQueryData = z.object({ +export const zD1RawDatabaseQueryData = z.object({ body: zD1BatchQuery, path: z.object({ database_id: zD1DatabaseIdentifier, @@ -508,7 +498,7 @@ export const zCloudflareD1RawDatabaseQueryData = z.object({ /** * Raw query response */ -export const zCloudflareD1RawDatabaseQueryResponse = zD1ApiResponseCommon.and( +export const zD1RawDatabaseQueryResponse = zD1ApiResponseCommon.and( z.object({ result: z.array(zD1RawResultResponse).optional(), }) @@ -517,12 +507,7 @@ export const zCloudflareD1RawDatabaseQueryResponse = zD1ApiResponseCommon.and( export const zDurableObjectsNamespaceListNamespacesData = z.object({ body: z.never().optional(), path: z.never().optional(), - query: z - .object({ - page: z.number().int().gte(1).optional().default(1), - per_page: z.number().int().gte(1).lte(1000).optional().default(20), - }) - .optional(), + query: z.never().optional(), }); /** diff --git a/packages/miniflare/src/workers/local-explorer/openapi.local.json b/packages/miniflare/src/workers/local-explorer/openapi.local.json index 7dc219a6e4eb..341c9dd6bb30 100644 --- a/packages/miniflare/src/workers/local-explorer/openapi.local.json +++ b/packages/miniflare/src/workers/local-explorer/openapi.local.json @@ -17,27 +17,6 @@ "description": "Returns the namespaces owned by an account.", "operationId": "workers-kv-namespace-list-namespaces", "parameters": [ - { - "in": "query", - "name": "page", - "schema": { - "default": 1, - "description": "Page number of paginated results.", - "minimum": 1, - "type": "number" - } - }, - { - "in": "query", - "name": "per_page", - "schema": { - "default": 20, - "description": "Maximum number of results per page.", - "maximum": 1000, - "minimum": 1, - "type": "number" - } - }, { "in": "query", "name": "order", @@ -445,7 +424,7 @@ "/d1/database": { "get": { "description": "Returns a list of D1 databases.", - "operationId": "cloudflare-d1-list-databases", + "operationId": "d1-list-databases", "parameters": [ { "in": "query", @@ -454,27 +433,6 @@ "description": "a database name to search for.", "type": "string" } - }, - { - "in": "query", - "name": "page", - "schema": { - "default": 1, - "description": "Page number of paginated results.", - "minimum": 1, - "type": "number" - } - }, - { - "in": "query", - "name": "per_page", - "schema": { - "default": 1000, - "description": "Number of items per page.", - "maximum": 10000, - "minimum": 10, - "type": "number" - } } ], "responses": { @@ -500,21 +458,6 @@ "description": "Total number of results for the requested service", "example": 1, "type": "number" - }, - "page": { - "description": "Current page within paginated list of results", - "example": 1, - "type": "number" - }, - "per_page": { - "description": "Number of results per page of results", - "example": 20, - "type": "number" - }, - "total_count": { - "description": "Total results available without any search parameters", - "example": 2000, - "type": "number" } }, "type": "object" @@ -545,7 +488,7 @@ "/d1/database/{database_id}/raw": { "post": { "description": "Returns the query result rows as arrays rather than objects. This is a performance-optimized version of the /query endpoint.", - "operationId": "cloudflare-d1-raw-database-query", + "operationId": "d1-raw-database-query", "parameters": [ { "in": "path", @@ -610,29 +553,7 @@ "get": { "description": "Returns the Durable Object namespaces owned by an account.", "operationId": "durable-objects-namespace-list-namespaces", - "parameters": [ - { - "description": "Current page.", - "in": "query", - "name": "page", - "schema": { - "default": 1, - "minimum": 1, - "type": "integer" - } - }, - { - "description": "Items per-page.", - "in": "query", - "name": "per_page", - "schema": { - "default": 20, - "maximum": 1000, - "minimum": 1, - "type": "integer" - } - } - ], + "parameters": [], "responses": { "200": { "content": { @@ -1648,21 +1569,6 @@ "description": "Total number of results for the requested service.", "example": 1, "type": "number" - }, - "page": { - "description": "Current page within paginated list of results.", - "example": 1, - "type": "number" - }, - "per_page": { - "description": "Number of results per page of results.", - "example": 20, - "type": "number" - }, - "total_count": { - "description": "Total results available without any search parameters.", - "example": 2000, - "type": "number" } }, "type": "object" diff --git a/packages/miniflare/src/workers/local-explorer/resources/d1.ts b/packages/miniflare/src/workers/local-explorer/resources/d1.ts index 43fa40bc0d5d..dc403471f08f 100644 --- a/packages/miniflare/src/workers/local-explorer/resources/d1.ts +++ b/packages/miniflare/src/workers/local-explorer/resources/d1.ts @@ -1,18 +1,29 @@ import { z } from "zod"; +import { + aggregateListResults, + fetchFromPeer, + getPeerUrlsIfAggregating, +} from "../aggregation"; import { errorResponse, wrapResponse } from "../common"; import { - zCloudflareD1ListDatabasesData, - zCloudflareD1RawDatabaseQueryData, + zD1ListDatabasesData, + zD1RawDatabaseQueryData, } from "../generated/zod.gen"; import type { AppContext } from "../common"; import type { Env } from "../explorer.worker"; import type { - CloudflareD1ListDatabasesResponse, D1DatabaseResponse, D1RawResultResponse, D1SingleQuery, } from "../generated"; +// ============================================================================ +// Error Codes (matching Cloudflare API) +// ============================================================================ + +/** Error code for D1 database not found */ +const D1_ERROR_DATABASE_NOT_FOUND = 7404; + // ============================================================================ // Helper Functions // ============================================================================ @@ -40,79 +51,106 @@ function getD1Binding(env: Env, databaseId: string): D1Database | null { return env[bindingName] as D1Database; } +/** + * Get local D1 databases from the binding map. + */ +function getLocalD1Databases(env: Env): D1DatabaseResponse[] { + const d1BindingMap = env.LOCAL_EXPLORER_BINDING_MAP.d1; + return Object.entries(d1BindingMap).map(([id, bindingName]) => { + // Use the binding name as the database name since we don't have + // the actual name locally. The ID is the `database_id` or generated from binding. + const databaseName = bindingName.split(":").pop() || bindingName; + + return { + name: databaseName, + uuid: id, + version: "production", + } satisfies D1DatabaseResponse; + }); +} + +async function findD1DatabaseOwner( + c: AppContext, + databaseId: string +): Promise { + const peerUrls = await getPeerUrlsIfAggregating(c); + if (peerUrls.length === 0) return null; + + const responses = await Promise.all( + peerUrls.map(async (url) => { + const response = await fetchFromPeer(url, "/d1/database"); + if (!response?.ok) return null; + const data = (await response.json()) as { + result?: Array<{ uuid: string }>; + }; + const found = data.result?.some((db) => db.uuid === databaseId); + return found ? url : null; + }) + ); + + return responses.find((url) => url !== null) ?? null; +} + // ============================================================================ // API Handlers // ============================================================================ type ListDatabasesQuery = z.output< - ReturnType + ReturnType >; /** - * Lists all D1 databases available in the local environment. + * Lists all D1 databases available across all connected instances. + * + * This is an aggregated endpoint - it fetches databases from the local instance + * and all peer instances in the dev registry, then merges the results. * - * Returns a paginated list of databases with their names and UUIDs. - * Supports filtering by database name and pagination via query parameters. + * Supports filtering by database name via the `name` query parameter. * * @see https://developers.cloudflare.com/api/resources/d1/subresources/database/methods/list/ * * @param c - The Hono application context - * @param query - Query parameters for filtering and pagination - * @param query.page - The page number for pagination - * @param query.per_page - The number of results per page + * @param query - Query parameters for filtering * @param query.name - Optional filter to search databases by name (case-insensitive) * - * @returns A JSON response containing the list of databases and pagination info + * @returns A JSON response containing all databases from all instances */ export async function listD1Databases( c: AppContext, query: ListDatabasesQuery ): Promise { - const { page, per_page, name } = query; + const { name } = query; - const d1BindingMap = c.env.LOCAL_EXPLORER_BINDING_MAP.d1; - let databases = Object.entries(d1BindingMap).map(([id, bindingName]) => { - // Use the binding name as the database name since we don't have - // the actual name locally. The ID is the `database_id` or generated from binding. - const databaseName = bindingName.split(":").pop() || bindingName; - - return { - name: databaseName, - uuid: id, - version: "production", - } satisfies D1DatabaseResponse; - }); + const localDatabases = getLocalD1Databases(c.env); + const aggregatedDatabases = await aggregateListResults( + c, + localDatabases, + "/d1/database" + ); - const totalCount = databases.length; + // deduplicate by id - not totally correct, since local dev can use binding names as an 'id' :/ + // TODO: check persistence path to properly verify local uniqueness + const localIds = new Set(localDatabases.map((db) => db.uuid)); + let allDatabases = aggregatedDatabases.filter( + (db, index) => index < localDatabases.length || !localIds.has(db.uuid) + ); // Filter by name if provided if (name) { - databases = databases.filter((db) => + allDatabases = allDatabases.filter((db) => db.name?.toLowerCase().includes(name.toLowerCase()) ); } - // Paginate - const startIndex = (page - 1) * per_page; - const endIndex = startIndex + per_page; - databases = databases.slice(startIndex, endIndex); - return c.json({ - ...wrapResponse(databases), + ...wrapResponse(allDatabases), result_info: { - count: databases.length, - page, - per_page, - total_count: totalCount, + count: allDatabases.length, }, - } satisfies Omit & { - result: D1DatabaseResponse[]; }); } -type RawDatabaseBody = z.output< - typeof zCloudflareD1RawDatabaseQueryData.shape.body ->; +type RawDatabaseBody = z.output; /** * Executes raw SQL queries against a D1 database. @@ -122,6 +160,9 @@ type RawDatabaseBody = z.output< * Supports both single queries and batch queries. Each query can include * parameterized values for safe SQL execution. * + * If the database is not found locally, this handler will attempt to proxy + * the request to peer instances in the dev registry. + * * @see https://developers.cloudflare.com/api/resources/d1/subresources/database/methods/raw/ * * @param c - The Hono application context (`database_id` is extracted from the request path) @@ -142,11 +183,41 @@ export async function rawD1Database( return errorResponse(400, 10000, "Missing database_id parameter"); } + // Try local first const db = getD1Binding(c.env, databaseId); - if (!db) { - return errorResponse(404, 10000, "Database not found"); + if (db) { + return executeD1Query(c, db, body); + } + + const ownerMiniflare = await findD1DatabaseOwner(c, databaseId); + if (ownerMiniflare) { + const response = await fetchFromPeer( + ownerMiniflare, + `/d1/database/${encodeURIComponent(databaseId)}/raw`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + } + ); + if (response) return response; } + return errorResponse( + 404, + D1_ERROR_DATABASE_NOT_FOUND, + `The database ${databaseId} could not be found` + ); +} + +/** + * Execute a D1 query against a local database binding. + */ +async function executeD1Query( + c: AppContext, + db: D1Database, + body: RawDatabaseBody +): Promise { // Normalize to array of queries const queries: D1SingleQuery[] = "batch" in body && body.batch ? body.batch : [body as D1SingleQuery]; diff --git a/packages/miniflare/src/workers/local-explorer/resources/do.ts b/packages/miniflare/src/workers/local-explorer/resources/do.ts index 35c04a46b0e2..367f92f90c84 100644 --- a/packages/miniflare/src/workers/local-explorer/resources/do.ts +++ b/packages/miniflare/src/workers/local-explorer/resources/do.ts @@ -1,7 +1,11 @@ import { INTROSPECT_SQLITE_METHOD } from "../../../plugins/core/constants"; +import { + aggregateListResults, + fetchFromPeer, + getPeerUrlsIfAggregating, +} from "../aggregation"; import { errorResponse, wrapResponse } from "../common"; import { - zDurableObjectsNamespaceListNamespacesData, zDurableObjectsNamespaceListObjectsData, zDurableObjectsNamespaceQuerySqliteData, } from "../generated/zod.gen"; @@ -10,46 +14,115 @@ import type { AppContext } from "../common"; import type { Env } from "../explorer.worker"; import type { z } from "zod"; -type ListNamespacesQuery = NonNullable< - z.output["query"] ->; +// ============================================================================ +// Error Codes (matching Cloudflare API) +// ============================================================================ -/** - * List Durable Object Namespaces - * https://developers.cloudflare.com/api/resources/durable_objects/subresources/namespaces/methods/list/ - * - * Returns the Durable Object namespaces available locally. - */ -export async function listDONamespaces( - c: AppContext, - query: ListNamespacesQuery -) { - const { page, per_page } = query; +/** Error code for Durable Object namespace not found */ +const DO_ERROR_NAMESPACE_NOT_FOUND = 10066; + +// ============================================================================ +// Helper Functions +// ============================================================================ + +interface DirectoryEntry { + name: string; + type: "file" | "directory"; +} - const doBindingMap = c.env.LOCAL_EXPLORER_BINDING_MAP.do; +interface IntrospectableDurableObject extends Rpc.DurableObjectBranded { + [INTROSPECT_SQLITE_METHOD]: IntrospectSqliteMethod; +} + +function getDOBinding( + env: Env, + namespaceId: string +): { + binding: DurableObjectNamespace; + useSQLite: boolean; +} | null { + const info = env.LOCAL_EXPLORER_BINDING_MAP.do[namespaceId]; + if (!info) return null; + return { + binding: env[ + info.binding + ] as DurableObjectNamespace, + useSQLite: info.useSQLite, + }; +} - // Convert binding map to array of namespace objects - let namespaces = Object.entries(doBindingMap).map(([id, info]) => ({ +/** + * Get local DO namespaces from the binding map. + */ +function getLocalDONamespaces(env: Env): { + id: string; + name: string; + script: string; + class: string; + use_sqlite: boolean; +}[] { + const doBindingMap = env.LOCAL_EXPLORER_BINDING_MAP.do; + return Object.entries(doBindingMap).map(([id, info]) => ({ id, // This is the unsafeUniqueKey - ${scriptName}-${className} name: `${info.scriptName}_${info.className}`, // This is what the API returns... script: info.scriptName, class: info.className, use_sqlite: info.useSQLite, })); +} - const totalCount = namespaces.length; +async function findDONamespaceOwner( + c: AppContext, + namespaceId: string +): Promise { + const peerUrls = await getPeerUrlsIfAggregating(c); + if (peerUrls.length === 0) return null; + + const responses = await Promise.all( + peerUrls.map(async (url) => { + const response = await fetchFromPeer( + url, + "/workers/durable_objects/namespaces" + ); + if (!response?.ok) return null; + const data = (await response.json()) as { + result?: Array<{ id: string }>; + }; + const found = data.result?.some((ns) => ns.id === namespaceId); + return found ? url : null; + }) + ); + + return responses.find((url) => url !== null) ?? null; +} - // Paginate results - const startIndex = (page - 1) * per_page; - namespaces = namespaces.slice(startIndex, startIndex + per_page); +// ============================================================================ +// API Handlers +// ============================================================================ + +/** + * List Durable Object Namespaces across all connected instances. + * + * This is an aggregated endpoint - it fetches namespaces from the local instance + * and all peer instances in the dev registry, then merges the results. + * + * @see https://developers.cloudflare.com/api/resources/durable_objects/subresources/namespaces/methods/list/ + */ +export async function listDONamespaces(c: AppContext) { + const localNamespaces = getLocalDONamespaces(c.env); + // note that we don't have duplication issues here like + // we do for listD1Namespaces etc. because DOs are tied + // to scripts and external DOs have already been filtered out + const allNamespaces = await aggregateListResults( + c, + localNamespaces, + "/workers/durable_objects/namespaces" + ); return c.json({ - ...wrapResponse(namespaces), + ...wrapResponse(allNamespaces), result_info: { - count: namespaces.length, - page, - per_page, - total_count: totalCount, + count: allNamespaces.length, }, }); } @@ -58,17 +131,13 @@ type ListObjectsQuery = NonNullable< z.output["query"] >; -interface DirectoryEntry { - name: string; - type: "file" | "directory"; -} - /** * List Durable Objects in a namespace - * https://developers.cloudflare.com/api/resources/durable_objects/subresources/namespaces/methods/list_objects/ * - * Returns the Durable Objects in a given namespace. - * Objects are enumerated by reading the persist directory for .sqlite files. + * This endpoint keeps pagination as-is since it operates on a single namespace. + * If the namespace is not found locally, it proxies to peer instances. + * + * @see https://developers.cloudflare.com/api/resources/durable_objects/subresources/namespaces/methods/list_objects/ */ export async function listDOObjects( c: AppContext, @@ -77,12 +146,45 @@ export async function listDOObjects( ) { const { limit, cursor } = query; - if ( - !c.env.LOCAL_EXPLORER_BINDING_MAP.do[namespaceId] || - // No loopback service means we can't list DOs - c.env.MINIFLARE_LOOPBACK === undefined - ) { - return errorResponse(404, 10001, `Namespace not found: ${namespaceId}`); + // Check if namespace exists locally + const namespaceExists = c.env.LOCAL_EXPLORER_BINDING_MAP.do[namespaceId]; + + if (namespaceExists) { + return executeListDOObjects(c, namespaceId, { limit, cursor }); + } + + const ownerMiniflare = await findDONamespaceOwner(c, namespaceId); + if (ownerMiniflare) { + const params = new URLSearchParams(); + if (cursor) params.set("cursor", cursor); + if (limit !== undefined) params.set("limit", String(limit)); + const queryString = params.toString(); + const path = `/workers/durable_objects/namespaces/${encodeURIComponent(namespaceId)}/objects${queryString ? `?${queryString}` : ""}`; + + const response = await fetchFromPeer(ownerMiniflare, path); + if (response) return response; + } + + return errorResponse( + 404, + DO_ERROR_NAMESPACE_NOT_FOUND, + `Durable Object namespace ID '${namespaceId}' not found.` + ); +} + +/** + * Execute list DO objects on a local namespace. + */ +async function executeListDOObjects( + c: AppContext, + namespaceId: string, + options: { limit: number; cursor?: string } +): Promise { + const { limit, cursor } = options; + + // No loopback service means we can't list DOs + if (c.env.MINIFLARE_LOOPBACK === undefined) { + return errorResponse(500, 10001, "Loopback service not available"); } // The DO storage structure is: //.sqlite @@ -163,33 +265,14 @@ type QueryBody = z.output< typeof zDurableObjectsNamespaceQuerySqliteData >["body"]; -interface IntrospectableDurableObject extends Rpc.DurableObjectBranded { - [INTROSPECT_SQLITE_METHOD]: IntrospectSqliteMethod; -} - -function getDOBinding( - env: Env, - namespaceId: string -): { - binding: DurableObjectNamespace; - useSQLite: boolean; -} | null { - const info = env.LOCAL_EXPLORER_BINDING_MAP.do[namespaceId]; - if (!info) return null; - return { - binding: env[ - info.binding - ] as DurableObjectNamespace, - useSQLite: info.useSQLite, - }; -} - /** * Query Durable Object SQLite storage * * Executes SQL queries against a specific Durable Object's SQLite storage * using introspection method that is injected into user DO classes. * + * If the namespace is not found locally, it proxies to peer instances. + * * The namespace ID is the uniqueKey: scriptName-className */ export async function queryDOSqlite( @@ -197,13 +280,46 @@ export async function queryDOSqlite( namespaceId: string, body: QueryBody ): Promise { - // Look up namespace in binding map + // Try local first const ns = getDOBinding(c.env, namespaceId); - if (!ns) { - return errorResponse(404, 10001, `Namespace not found: ${namespaceId}`); + if (ns) { + return executeQueryDOSqlite(c, ns, namespaceId, body); } + const ownerMiniflare = await findDONamespaceOwner(c, namespaceId); + if (ownerMiniflare) { + const response = await fetchFromPeer( + ownerMiniflare, + `/workers/durable_objects/namespaces/${encodeURIComponent(namespaceId)}/query`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + } + ); + if (response) return response; + } + + return errorResponse( + 404, + DO_ERROR_NAMESPACE_NOT_FOUND, + `Durable Object namespace ID '${namespaceId}' not found.` + ); +} + +/** + * Execute query on a local DO namespace. + */ +async function executeQueryDOSqlite( + c: AppContext, + ns: { + binding: DurableObjectNamespace; + useSQLite: boolean; + }, + namespaceId: string, + body: QueryBody +): Promise { if (!ns.useSQLite) { return errorResponse( 400, diff --git a/packages/miniflare/src/workers/local-explorer/resources/kv.ts b/packages/miniflare/src/workers/local-explorer/resources/kv.ts index 26811b511c98..bc9c62fae1a2 100644 --- a/packages/miniflare/src/workers/local-explorer/resources/kv.ts +++ b/packages/miniflare/src/workers/local-explorer/resources/kv.ts @@ -1,4 +1,9 @@ import z from "zod"; +import { + aggregateListResults, + fetchFromPeer, + getPeerUrlsIfAggregating, +} from "../aggregation"; import { errorResponse, wrapResponse } from "../common"; import { zWorkersKvNamespaceGetMultipleKeyValuePairsData, @@ -8,6 +13,19 @@ import { import type { AppContext } from "../common"; import type { Env } from "../explorer.worker"; +// ============================================================================ +// Error Codes (matching Cloudflare API) +// ============================================================================ + +/** Error code for key not found in KV namespace */ +const KV_ERROR_KEY_NOT_FOUND = 10009; +/** Error code for KV namespace not found */ +const KV_ERROR_NAMESPACE_NOT_FOUND = 10013; + +// ============================================================================ +// Helper Functions +// ============================================================================ + /** * Get a KV binding by namespace ID */ @@ -21,12 +39,57 @@ function getKVBinding(env: Env, namespace_id: string): KVNamespace | null { return env[bindingName] as KVNamespace; } +async function findKVNamespaceOwner( + c: AppContext, + namespaceId: string +): Promise { + const peerUrls = await getPeerUrlsIfAggregating(c); + if (peerUrls.length === 0) return null; + + const responses = await Promise.all( + peerUrls.map(async (url) => { + const response = await fetchFromPeer(url, "/storage/kv/namespaces"); + if (!response?.ok) return null; + const data = (await response.json()) as { + result?: Array<{ id: string }>; + }; + const found = data.result?.some((ns) => ns.id === namespaceId); + return found ? url : null; + }) + ); + + return responses.find((url) => url !== null) ?? null; +} + +/** + * Get local KV namespaces from the binding map. + */ +function getLocalKVNamespaces(env: Env): Array<{ id: string; title: string }> { + const kvBindingMap = env.LOCAL_EXPLORER_BINDING_MAP.kv; + return Object.entries(kvBindingMap).map(([id, bindingName]) => ({ + id: id, + // this is not technically correct, but the title doesn't exist locally + title: bindingName.split(":").pop() || bindingName, + })); +} + +// ============================================================================ +// API Handlers +// ============================================================================ + type ListNamespacesQuery = NonNullable< z.output["query"] >; + /** - * List Namespaces - * https://developers.cloudflare.com/api/resources/kv/subresources/namespaces/methods/list/ + * List all KV namespaces across all connected instances. + * + * This is an aggregated endpoint - it fetches namespaces from the local instance + * and all peer instances in the dev registry, then merges the results. + * + * Supports sorting via `direction` and `order` query parameters. + * + * @see https://developers.cloudflare.com/api/resources/kv/subresources/namespaces/methods/list/ */ export async function listKVNamespaces( c: AppContext, @@ -34,37 +97,33 @@ export async function listKVNamespaces( ) { const direction = query.direction ?? "asc"; const order = query.order ?? "id"; - const page = query.page; - const per_page = query.per_page; - - const kvBindingMap = c.env.LOCAL_EXPLORER_BINDING_MAP.kv; - let namespaces = Object.entries(kvBindingMap).map(([id, bindingName]) => ({ - id: id, - // this is not technically correct, but the title doesn't exist locally - title: bindingName.split(":").pop() || bindingName, - })); - namespaces.sort((a, b) => { + const localNamespaces = getLocalKVNamespaces(c.env); + const aggregatedNamespaces = await aggregateListResults( + c, + localNamespaces, + "/storage/kv/namespaces" + ); + + // deduplicate by id - not totally correct, since local dev can use binding names as an 'id' :/ + // TODO: check persistence path to properly verify local uniqueness + const localIds = new Set(localNamespaces.map((ns) => ns.id)); + const allNamespaces = aggregatedNamespaces.filter( + (ns, index) => index < localNamespaces.length || !localIds.has(ns.id) + ); + + // Sort results + allNamespaces.sort((a, b) => { const aVal = order === "id" ? a.id : a.title; const bVal = order === "id" ? b.id : b.title; const cmp = aVal.localeCompare(bVal); return direction === "asc" ? cmp : -cmp; }); - const total_count = namespaces.length; - - // Paginate - const startIndex = (page - 1) * per_page; - const endIndex = startIndex + per_page; - namespaces = namespaces.slice(startIndex, endIndex); - return c.json({ - ...wrapResponse(namespaces), + ...wrapResponse(allNamespaces), result_info: { - count: namespaces.length, - page, - per_page, - total_count, + count: allNamespaces.length, }, }); } @@ -74,7 +133,11 @@ type ListKeysQuery = NonNullable< >; /** * List a Namespace's Keys - * https://developers.cloudflare.com/api/resources/kv/subresources/namespaces/subresources/keys/methods/list/ + * + * This endpoint keeps pagination as-is since it operates on a single namespace. + * If the namespace is not found locally, it proxies to peer instances. + * + * @see https://developers.cloudflare.com/api/resources/kv/subresources/namespaces/subresources/keys/methods/list/ */ export async function listKVKeys(c: AppContext, query: ListKeysQuery) { const namespace_id = c.req.param("namespace_id"); @@ -85,12 +148,41 @@ export async function listKVKeys(c: AppContext, query: ListKeysQuery) { const limit = query.limit; const prefix = query.prefix; + // Try local first const kv = getKVBinding(c.env, namespace_id); - if (!kv) { - return errorResponse(404, 10000, "Namespace not found"); + if (kv) { + return executeListKeys(c, kv, { cursor, limit, prefix }); } - const listResult = await kv.list({ cursor, limit, prefix }); + const ownerMiniflare = await findKVNamespaceOwner(c, namespace_id); + if (ownerMiniflare) { + const params = new URLSearchParams(); + if (cursor) params.set("cursor", cursor); + if (limit !== undefined) params.set("limit", String(limit)); + if (prefix) params.set("prefix", prefix); + const queryString = params.toString(); + const path = `/storage/kv/namespaces/${encodeURIComponent(namespace_id)}/keys${queryString ? `?${queryString}` : ""}`; + + const response = await fetchFromPeer(ownerMiniflare, path); + if (response) return response; + } + + return errorResponse( + 404, + KV_ERROR_NAMESPACE_NOT_FOUND, + "list keys: 'namespace not found'" + ); +} + +/** + * Execute list keys on a local KV binding. + */ +async function executeListKeys( + c: AppContext, + kv: KVNamespace, + options: { cursor?: string; limit?: number; prefix?: string } +) { + const listResult = await kv.list(options); const resultCursor = "cursor" in listResult ? listResult.cursor ?? "" : ""; return c.json({ @@ -110,7 +202,10 @@ export async function listKVKeys(c: AppContext, query: ListKeysQuery) { /** * Read key-value pair - * https://developers.cloudflare.com/api/resources/kv/subresources/namespaces/subresources/values/methods/get/ + * + * If the namespace is not found locally, it proxies to peer instances. + * + * @see https://developers.cloudflare.com/api/resources/kv/subresources/namespaces/subresources/values/methods/get/ */ export async function getKVValue(c: AppContext) { const namespace_id = c.req.param("namespace_id"); @@ -119,23 +214,39 @@ export async function getKVValue(c: AppContext) { return errorResponse(400, 10000, "Missing required path parameters"); } + // Try local first const kv = getKVBinding(c.env, namespace_id); - if (!kv) { - return errorResponse(404, 10000, "Namespace not found"); + if (kv) { + const value = await kv.get(key_name, { type: "arrayBuffer" }); + if (value === null) { + return errorResponse(404, KV_ERROR_KEY_NOT_FOUND, "get: 'key not found'"); + } + // this specific API doesn't wrap the response in the envelope + return new Response(value); } - const value = await kv.get(key_name, { type: "arrayBuffer" }); - if (value === null) { - return errorResponse(404, 10000, "Key not found"); + const ownerMiniflare = await findKVNamespaceOwner(c, namespace_id); + if (ownerMiniflare) { + const response = await fetchFromPeer( + ownerMiniflare, + `/storage/kv/namespaces/${encodeURIComponent(namespace_id)}/values/${encodeURIComponent(key_name)}` + ); + if (response) return response; } - // this specific API doesn't wrap the response in the envelope - return new Response(value); + return errorResponse( + 404, + KV_ERROR_NAMESPACE_NOT_FOUND, + "get: 'namespace not found'" + ); } /** * Write key-value pair with optional metadata - * https://developers.cloudflare.com/api/resources/kv/subresources/namespaces/subresources/values/methods/update/ + * + * If the namespace is not found locally, it proxies to peer instances. + * + * @see https://developers.cloudflare.com/api/resources/kv/subresources/namespaces/subresources/values/methods/update/ */ export async function putKVValue(c: AppContext) { const namespace_id = c.req.param("namespace_id"); @@ -144,11 +255,45 @@ export async function putKVValue(c: AppContext) { return errorResponse(400, 10000, "Missing required path parameters"); } + // Try local first const kv = getKVBinding(c.env, namespace_id); - if (!kv) { - return errorResponse(404, 10000, "Namespace not found"); + if (kv) { + return executePutKVValue(c, kv, key_name); + } + + const ownerMiniflare = await findKVNamespaceOwner(c, namespace_id); + if (ownerMiniflare) { + const body = await c.req.arrayBuffer(); + const response = await fetchFromPeer( + ownerMiniflare, + `/storage/kv/namespaces/${encodeURIComponent(namespace_id)}/values/${encodeURIComponent(key_name)}`, + { + method: "PUT", + headers: { + "Content-Type": + c.req.header("content-type") || "application/octet-stream", + }, + body, + } + ); + if (response) return response; } + return errorResponse( + 404, + KV_ERROR_NAMESPACE_NOT_FOUND, + "put: 'namespace not found'" + ); +} + +/** + * Execute put KV value on a local KV binding. + */ +async function executePutKVValue( + c: AppContext, + kv: KVNamespace, + key_name: string +): Promise { let value: ArrayBuffer | string; let metadata: unknown | undefined; @@ -199,7 +344,10 @@ export async function putKVValue(c: AppContext) { /** * Delete key-value pair - * https://developers.cloudflare.com/api/resources/kv/subresources/namespaces/subresources/values/methods/delete/ + * + * If the namespace is not found locally, it proxies to peer instances. + * + * @see https://developers.cloudflare.com/api/resources/kv/subresources/namespaces/subresources/values/methods/delete/ */ export async function deleteKVValue(c: AppContext) { const namespace_id = c.req.param("namespace_id"); @@ -208,13 +356,28 @@ export async function deleteKVValue(c: AppContext) { return errorResponse(400, 10000, "Missing required path parameters"); } + // Try local first const kv = getKVBinding(c.env, namespace_id); - if (!kv) { - return errorResponse(404, 10000, "Namespace not found"); + if (kv) { + await kv.delete(key_name); + return c.json(wrapResponse({})); } - await kv.delete(key_name); - return c.json(wrapResponse({})); + const ownerMiniflare = await findKVNamespaceOwner(c, namespace_id); + if (ownerMiniflare) { + const response = await fetchFromPeer( + ownerMiniflare, + `/storage/kv/namespaces/${encodeURIComponent(namespace_id)}/values/${encodeURIComponent(key_name)}`, + { method: "DELETE" } + ); + if (response) return response; + } + + return errorResponse( + 404, + KV_ERROR_NAMESPACE_NOT_FOUND, + "remove key: 'namespace not found'" + ); } type BulkGetBody = NonNullable< @@ -222,7 +385,10 @@ type BulkGetBody = NonNullable< >; /** * Get multiple key-value pairs - * https://developers.cloudflare.com/api/resources/kv/subresources/namespaces/methods/bulk_get/ + * + * If the namespace is not found locally, it proxies to peer instances. + * + * @see https://developers.cloudflare.com/api/resources/kv/subresources/namespaces/methods/bulk_get/ */ export async function bulkGetKVValues(c: AppContext, body: BulkGetBody) { const namespace_id = c.req.param("namespace_id"); @@ -231,19 +397,38 @@ export async function bulkGetKVValues(c: AppContext, body: BulkGetBody) { } const { keys } = body; + // Try local first const kv = getKVBinding(c.env, namespace_id); - if (!kv) { - return errorResponse(404, 10000, "Namespace not found"); - } + if (kv) { + // Fetch all keys at once - returns Map + const results = await kv.get(keys); + + // Build result object with null for missing keys + const values: Record = {}; + for (const key of keys) { + values[key] = results?.get(key) ?? null; + } - // Fetch all keys at once - returns Map - const results = await kv.get(keys); + return c.json(wrapResponse({ values })); + } - // Build result object with null for missing keys - const values: Record = {}; - for (const key of keys) { - values[key] = results?.get(key) ?? null; + const ownerMiniflare = await findKVNamespaceOwner(c, namespace_id); + if (ownerMiniflare) { + const response = await fetchFromPeer( + ownerMiniflare, + `/storage/kv/namespaces/${encodeURIComponent(namespace_id)}/bulk/get`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + } + ); + if (response) return response; } - return c.json(wrapResponse({ values })); + return errorResponse( + 404, + KV_ERROR_NAMESPACE_NOT_FOUND, + "bulk get keys: 'namespace not found'" + ); } diff --git a/packages/miniflare/test/plugins/local-explorer/aggregation.spec.ts b/packages/miniflare/test/plugins/local-explorer/aggregation.spec.ts new file mode 100644 index 000000000000..f49aded00e8f --- /dev/null +++ b/packages/miniflare/test/plugins/local-explorer/aggregation.spec.ts @@ -0,0 +1,673 @@ +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { removeDirSync } from "@cloudflare/workers-utils"; +import { getWorkerRegistry, Miniflare } from "miniflare"; +import { afterAll, beforeAll, describe, test } from "vitest"; +import { LOCAL_EXPLORER_API_PATH } from "../../../src/plugins/core/constants"; +import { disposeWithRetry } from "../../test-shared"; + +const BASE_URL = `http://localhost${LOCAL_EXPLORER_API_PATH}`; + +/** + * Poll the dev registry until all expected workers are registered. + * This avoids flakey tests that rely on fixed timeouts. + */ +async function waitForWorkersInRegistry( + registryPath: string, + expectedWorkers: string[], + timeoutMs = 5000 +): Promise { + const startTime = Date.now(); + while (Date.now() - startTime < timeoutMs) { + const registry = getWorkerRegistry(registryPath); + const registeredWorkers = Object.keys(registry); + if (expectedWorkers.every((w) => registeredWorkers.includes(w))) { + return; + } + await new Promise((resolve) => setTimeout(resolve, 50)); + } + throw new Error( + `Timed out waiting for workers to register: ${expectedWorkers.join(", ")}` + ); +} + +interface ListResponse { + result?: Array<{ id?: string; uuid?: string; [key: string]: unknown }>; + result_info?: { count?: number }; +} + +/** + * Helper to normalize list responses for snapshot testing. + * Sorts results by id to ensure consistent ordering. + */ +function normalizeListResponse(data: ListResponse) { + const sorted = [...(data.result ?? [])].sort((a, b) => + (a.id ?? a.uuid ?? "").localeCompare(b.id ?? b.uuid ?? "") + ); + return { + result: sorted, + result_info: data.result_info, + }; +} + +describe("Cross-process aggregation", () => { + let registryPath: string; + let instanceA: Miniflare; + let instanceB: Miniflare; + + beforeAll(async () => { + // Create a shared dev registry directory + registryPath = mkdtempSync(path.join(tmpdir(), "mf-registry-")); + + instanceA = new Miniflare({ + name: "worker-a", + inspectorPort: 0, + compatibilityDate: "2025-01-01", + modules: true, + script: ` + export class MyDO { + constructor(state) { this.state = state; } + async fetch() { return new Response("DO A"); } + } + export default { fetch() { return new Response("Worker A"); } } + `, + unsafeLocalExplorer: true, + unsafeDevRegistryPath: registryPath, + kvNamespaces: { + KV_A_1: "kv-a-1", + KV_A_2: "kv-a-2", + }, + d1Databases: { + DB_A: "db-a", + }, + durableObjects: { + MY_DO: "MyDO", + }, + }); + + instanceB = new Miniflare({ + name: "worker-b", + inspectorPort: 0, + compatibilityDate: "2025-01-01", + modules: true, + script: ` + export class OtherDO { + constructor(state) { this.state = state; } + async fetch() { return new Response("DO B"); } + } + export default { fetch() { return new Response("Worker B"); } } + `, + unsafeLocalExplorer: true, + unsafeDevRegistryPath: registryPath, + kvNamespaces: { + KV_B_1: "kv-b-1", + }, + d1Databases: { + DB_B: "db-b", + }, + durableObjects: { + OTHER_DO: "OtherDO", + }, + }); + await instanceA.ready; + await instanceB.ready; + + // Wait for both instances to register in the dev registry + await waitForWorkersInRegistry(registryPath, ["worker-a", "worker-b"]); + }); + + afterAll(async () => { + await Promise.all([ + disposeWithRetry(instanceA), + disposeWithRetry(instanceB), + ]); + removeDirSync(registryPath); + }); + + describe("KV namespace aggregation", () => { + test("lists KV namespaces from both instances when queried from instance A", async ({ + expect, + }) => { + const response = await instanceA.dispatchFetch( + `${BASE_URL}/storage/kv/namespaces` + ); + const data = (await response.json()) as ListResponse; + + expect(normalizeListResponse(data)).toMatchInlineSnapshot(` + { + "result": [ + { + "id": "kv-a-1", + "title": "KV_A_1", + }, + { + "id": "kv-a-2", + "title": "KV_A_2", + }, + { + "id": "kv-b-1", + "title": "KV_B_1", + }, + ], + "result_info": { + "count": 3, + }, + } + `); + }); + + test("lists KV namespaces from both instances when queried from instance B", async ({ + expect, + }) => { + const response = await instanceB.dispatchFetch( + `${BASE_URL}/storage/kv/namespaces` + ); + const data = (await response.json()) as ListResponse; + + expect(normalizeListResponse(data)).toMatchInlineSnapshot(` + { + "result": [ + { + "id": "kv-a-1", + "title": "KV_A_1", + }, + { + "id": "kv-a-2", + "title": "KV_A_2", + }, + { + "id": "kv-b-1", + "title": "KV_B_1", + }, + ], + "result_info": { + "count": 3, + }, + } + `); + }); + + test("proxies KV key list to peer instance when namespace not found locally", async ({ + expect, + }) => { + const kvB = await instanceB.getKVNamespace("KV_B_1"); + await kvB.put("peer-key-1", "value1"); + + const response = await instanceA.dispatchFetch( + `${BASE_URL}/storage/kv/namespaces/kv-b-1/keys` + ); + const data = await response.json(); + + expect(data).toMatchObject({ + success: true, + result: expect.arrayContaining([ + expect.objectContaining({ name: "peer-key-1" }), + ]), + }); + }); + + test("proxies KV value get to peer instance when namespace not found locally", async ({ + expect, + }) => { + const kvB = await instanceB.getKVNamespace("KV_B_1"); + await kvB.put("peer-value-key", "peer-value-content"); + + const response = await instanceA.dispatchFetch( + `${BASE_URL}/storage/kv/namespaces/kv-b-1/values/peer-value-key` + ); + + expect(response.status).toBe(200); + expect(await response.text()).toMatchInlineSnapshot( + `"peer-value-content"` + ); + }); + + test("proxies KV value put to peer instance when namespace not found locally", async ({ + expect, + }) => { + const response = await instanceA.dispatchFetch( + `${BASE_URL}/storage/kv/namespaces/kv-b-1/values/cross-write-key`, + { + method: "PUT", + body: "cross-written-value", + } + ); + + expect(response.status).toBe(200); + await response.json(); // Consume body + + const kvB = await instanceB.getKVNamespace("KV_B_1"); + expect(await kvB.get("cross-write-key")).toMatchInlineSnapshot( + `"cross-written-value"` + ); + }); + + test("proxies KV value delete to peer instance when namespace not found locally", async ({ + expect, + }) => { + const kvB = await instanceB.getKVNamespace("KV_B_1"); + await kvB.put("to-delete-key", "value"); + + const response = await instanceA.dispatchFetch( + `${BASE_URL}/storage/kv/namespaces/kv-b-1/values/to-delete-key`, + { method: "DELETE" } + ); + + expect(response.status).toBe(200); + await response.json(); // Consume body + + expect(await kvB.get("to-delete-key")).toBeNull(); + }); + + test("returns 404 when resource not found locally or on peers", async ({ + expect, + }) => { + const response = await instanceA.dispatchFetch( + `${BASE_URL}/storage/kv/namespaces/non-existent/keys` + ); + + expect(response.status).toBe(404); + expect(await response.json()).toMatchInlineSnapshot(` + { + "errors": [ + { + "code": 10013, + "message": "list keys: 'namespace not found'", + }, + ], + "messages": [], + "result": null, + "success": false, + } + `); + }); + }); + + describe("D1 database aggregation", () => { + test("lists D1 databases from both instances", async ({ expect }) => { + const response = await instanceA.dispatchFetch(`${BASE_URL}/d1/database`); + const data = (await response.json()) as ListResponse; + + expect(normalizeListResponse(data)).toMatchInlineSnapshot(` + { + "result": [ + { + "name": "DB_A", + "uuid": "db-a", + "version": "production", + }, + { + "name": "DB_B", + "uuid": "db-b", + "version": "production", + }, + ], + "result_info": { + "count": 2, + }, + } + `); + }); + + test("proxies D1 raw query to peer instance when database not found locally", async ({ + expect, + }) => { + const dbB = await instanceB.getD1Database("DB_B"); + await dbB.exec( + "CREATE TABLE IF NOT EXISTS test_table (id INTEGER PRIMARY KEY, name TEXT)" + ); + await dbB.exec("DELETE FROM test_table"); // Clean slate + await dbB.exec("INSERT INTO test_table (name) VALUES ('peer-row')"); + + const response = await instanceA.dispatchFetch( + `${BASE_URL}/d1/database/db-b/raw`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + sql: "SELECT name FROM test_table", + params: [], + }), + } + ); + + expect(response.status).toBe(200); + const data = (await response.json()) as { + success: boolean; + result: Array<{ results: { columns: string[]; rows: unknown[][] } }>; + }; + expect(data.success).toBe(true); + expect(data.result[0].results).toMatchInlineSnapshot(` + { + "columns": [ + "name", + ], + "rows": [ + [ + "peer-row", + ], + ], + } + `); + }); + }); + + describe("DO namespace aggregation", () => { + test("lists DO namespaces from both instances", async ({ expect }) => { + const response = await instanceA.dispatchFetch( + `${BASE_URL}/workers/durable_objects/namespaces` + ); + const data = (await response.json()) as ListResponse; + const normalized = normalizeListResponse(data); + + // use_sqlite depends on migrations config, so we check structure without it + expect(normalized.result?.map(({ id, name }) => ({ id, name }))) + .toMatchInlineSnapshot(` + [ + { + "id": "worker-a-MyDO", + "name": "worker-a_MyDO", + }, + { + "id": "worker-b-OtherDO", + "name": "worker-b_OtherDO", + }, + ] + `); + expect(normalized.result_info).toMatchInlineSnapshot(` + { + "count": 2, + } + `); + }); + }); +}); + +describe("Multi-worker peer deduplication", () => { + let registryPath: string; + let instanceA: Miniflare; + let instanceB: Miniflare; + + beforeAll(async () => { + registryPath = mkdtempSync(path.join(tmpdir(), "mf-registry-multiworker-")); + + instanceA = new Miniflare({ + name: "worker-a", + inspectorPort: 0, + compatibilityDate: "2025-01-01", + modules: true, + script: `export default { fetch() { return new Response("Worker A"); } }`, + unsafeLocalExplorer: true, + unsafeDevRegistryPath: registryPath, + kvNamespaces: { + KV_A: "kv-a", + }, + }); + await instanceA.ready; + + // Instance B: TWO workers in the same Miniflare process + // Both register in the dev registry with the same host:port + instanceB = new Miniflare({ + inspectorPort: 0, + compatibilityDate: "2025-01-01", + unsafeLocalExplorer: true, + unsafeDevRegistryPath: registryPath, + workers: [ + { + name: "worker-b1", + modules: true, + script: `export default { fetch() { return new Response("Worker B1"); } }`, + kvNamespaces: { + KV_B1: "kv-b1", + }, + }, + { + name: "worker-b2", + modules: true, + script: `export default { fetch() { return new Response("Worker B2"); } }`, + kvNamespaces: { + KV_B2: "kv-b2", + }, + }, + ], + }); + await instanceB.ready; + + // Wait for all workers to register in the dev registry + await waitForWorkersInRegistry(registryPath, [ + "worker-a", + "worker-b1", + "worker-b2", + ]); + }); + + afterAll(async () => { + await Promise.all([ + disposeWithRetry(instanceA), + disposeWithRetry(instanceB), + ]); + removeDirSync(registryPath); + }); + + test("does not duplicate results when peer has multiple workers", async ({ + expect, + }) => { + const response = await instanceA.dispatchFetch( + `${BASE_URL}/storage/kv/namespaces` + ); + const data = (await response.json()) as ListResponse; + + // Should have exactly 3 namespaces (kv-a, kv-b1, kv-b2) + // NOT 5 which would happen without URL deduplication + expect(normalizeListResponse(data)).toMatchInlineSnapshot(` + { + "result": [ + { + "id": "kv-a", + "title": "KV_A", + }, + { + "id": "kv-b1", + "title": "KV_B1", + }, + { + "id": "kv-b2", + "title": "KV_B2", + }, + ], + "result_info": { + "count": 3, + }, + } + `); + }); +}); + +describe("Same ID across multiple instances with different persistence directories", () => { + let registryPath: string; + let instanceA: Miniflare; + let instanceB: Miniflare; + + beforeAll(async () => { + registryPath = mkdtempSync(path.join(tmpdir(), "mf-registry-same-id-")); + + // Both instances use the SAME KV and D1 ids + // but they have separate storage (different default persistence paths) + // Helpfully, DOs require you to specify a script name, which explicitly + // ties it to a specific instance. + instanceA = new Miniflare({ + name: "worker-a", + inspectorPort: 0, + compatibilityDate: "2025-01-01", + modules: true, + script: `export default { fetch() { return new Response("Worker A"); } }`, + unsafeLocalExplorer: true, + unsafeDevRegistryPath: registryPath, + kvNamespaces: { + MY_KV: "shared-kv-id", + }, + d1Databases: { + MY_DB: "shared-db-id", + }, + durableObjects: { + MY_DO: { + className: "MyDO", + }, + }, + }); + + instanceB = new Miniflare({ + name: "worker-b", + inspectorPort: 0, + compatibilityDate: "2025-01-01", + modules: true, + script: `export default { fetch() { return new Response("Worker B"); } }`, + unsafeLocalExplorer: true, + unsafeDevRegistryPath: registryPath, + kvNamespaces: { + MY_KV: "shared-kv-id", + }, + d1Databases: { + MY_DB: "shared-db-id", + }, + durableObjects: { + MY_DO: { + className: "MyDO", + scriptName: "worker-a", + }, + }, + }); + + await instanceA.ready; + await instanceB.ready; + + await waitForWorkersInRegistry(registryPath, ["worker-a", "worker-b"]); + }); + + afterAll(async () => { + await Promise.all([ + disposeWithRetry(instanceA), + disposeWithRetry(instanceB), + ]); + removeDirSync(registryPath); + }); + + test("listing deduplicates namespaces with the same ID", async ({ + expect, + }) => { + let response = await instanceA.dispatchFetch( + `${BASE_URL}/storage/kv/namespaces` + ); + let data = (await response.json()) as ListResponse; + expect(data.result_info?.count).toBe(1); + + response = await instanceA.dispatchFetch(`${BASE_URL}/d1/database`); + data = (await response.json()) as ListResponse; + expect(data.result_info?.count).toBe(1); + + response = await instanceA.dispatchFetch( + `${BASE_URL}/workers/durable_objects/namespaces` + ); + data = (await response.json()) as ListResponse; + expect(data.result_info?.count).toBe(1); + }); + + // TODO: this is kind of a footgun - we should somehow check persistence paths + // and warn if resources with the same id aren't using the same storage + test("operations will prioritise local storage when ID exists locally", async ({ + expect, + }) => { + // Write different values to the same key in each instance's KV + const kvA = await instanceA.getKVNamespace("MY_KV"); + const kvB = await instanceB.getKVNamespace("MY_KV"); + + await kvA.put("test-key", "value-from-A"); + await kvB.put("test-key", "value-from-B"); + + // Query from instance A - should get A's value, not B's + const responseA = await instanceA.dispatchFetch( + `${BASE_URL}/storage/kv/namespaces/shared-kv-id/values/test-key` + ); + expect(await responseA.text()).toBe("value-from-A"); + + // Query from instance B - should get B's value, not A's + const responseB = await instanceB.dispatchFetch( + `${BASE_URL}/storage/kv/namespaces/shared-kv-id/values/test-key` + ); + expect(await responseB.text()).toBe("value-from-B"); + }); +}); + +describe("Same ID across multiple instances with same persistence directories", () => { + let registryPath: string; + + let instanceA: Miniflare; + let instanceB: Miniflare; + + beforeAll(async () => { + registryPath = mkdtempSync(path.join(tmpdir(), "mf-registry-same-id-")); + const persistencePath = path.join(tmpdir(), "mf-persistence-same-id"); + + // Both instances use the SAME KV and D1 ids + // but they have separate storage (different default persistence paths) + // Helpfully, DOs require you to specify a script name, which explicitly + // ties it to a specific instance. + instanceA = new Miniflare({ + name: "worker-a", + inspectorPort: 0, + compatibilityDate: "2025-01-01", + modules: true, + script: `export default { fetch() { return new Response("Worker A"); } }`, + unsafeLocalExplorer: true, + unsafeDevRegistryPath: registryPath, + defaultPersistRoot: persistencePath, + kvNamespaces: { + MY_KV: "shared-kv-id", + }, + }); + + instanceB = new Miniflare({ + name: "worker-b", + inspectorPort: 0, + compatibilityDate: "2025-01-01", + modules: true, + script: `export default { fetch() { return new Response("Worker B"); } }`, + unsafeLocalExplorer: true, + unsafeDevRegistryPath: registryPath, + defaultPersistRoot: persistencePath, + kvNamespaces: { + MY_KV: "shared-kv-id", + }, + }); + + await instanceA.ready; + await instanceB.ready; + + await waitForWorkersInRegistry(registryPath, ["worker-a", "worker-b"]); + }); + + afterAll(async () => { + await Promise.all([ + disposeWithRetry(instanceA), + disposeWithRetry(instanceB), + ]); + removeDirSync(registryPath); + removeDirSync(path.join(tmpdir(), "mf-persistence-same-id")); + }); + + test("operations will prioritise local storage when ID exists locally", async ({ + expect, + }) => { + // Write different values to the same key in each instance's KV + const kvA = await instanceA.getKVNamespace("MY_KV"); + + await kvA.put("test-key", "value-from-A"); + + // Query from instance B - should get A's value + const responseB = await instanceB.dispatchFetch( + `${BASE_URL}/storage/kv/namespaces/shared-kv-id/values/test-key` + ); + expect(await responseB.text()).toBe("value-from-A"); + }); +}); diff --git a/packages/miniflare/test/plugins/local-explorer/d1.spec.ts b/packages/miniflare/test/plugins/local-explorer/d1.spec.ts index 44a6be518962..b3fec2cf9c27 100644 --- a/packages/miniflare/test/plugins/local-explorer/d1.spec.ts +++ b/packages/miniflare/test/plugins/local-explorer/d1.spec.ts @@ -2,9 +2,9 @@ import { Miniflare } from "miniflare"; import { afterAll, beforeAll, describe, test } from "vitest"; import { LOCAL_EXPLORER_API_PATH } from "../../../src/plugins/core/constants"; import { - zCloudflareD1ListDatabasesResponse, - zCloudflareD1RawDatabaseQueryResponse, zD1ApiResponseCommonFailure, + zD1ListDatabasesResponse, + zD1RawDatabaseQueryResponse, } from "../../../src/workers/local-explorer/generated/zod.gen"; import { disposeWithRetry } from "../../test-shared"; import { expectValidResponse } from "./helpers"; @@ -53,16 +53,14 @@ INSERT INTO users (name, email) VALUES ('Bob', 'bob@example.com'); }); describe("GET /d1/database", () => { - test("lists available D1 databases with default pagination", async ({ - expect, - }) => { + test("lists all available D1 databases", async ({ expect }) => { const response = await mf.dispatchFetch(`${BASE_URL}/d1/database`); expect(response.headers.get("Content-Type")).toBe("application/json"); const data = await expectValidResponse( response, - zCloudflareD1ListDatabasesResponse + zD1ListDatabasesResponse ); expect(data.success).toBe(true); @@ -77,33 +75,6 @@ INSERT INTO users (name, email) VALUES ('Bob', 'bob@example.com'); ); expect(data.result_info).toMatchObject({ count: 2, - page: 1, - per_page: 1000, - total_count: 2, - }); - }); - - test("pagination works", async ({ expect }) => { - // D1 has per_page minimum of 10, so use 10 as smallest page size - const response = await mf.dispatchFetch( - `${BASE_URL}/d1/database?per_page=10&page=1` - ); - - expect(response.headers.get("Content-Type")).toBe("application/json"); - - const data = await expectValidResponse( - response, - zCloudflareD1ListDatabasesResponse - ); - - expect(data).toMatchObject({ - result_info: { - count: 2, - page: 1, - per_page: 10, - total_count: 2, - }, - success: true, }); }); @@ -116,37 +87,13 @@ INSERT INTO users (name, email) VALUES ('Bob', 'bob@example.com'); const data = await expectValidResponse( response, - zCloudflareD1ListDatabasesResponse + zD1ListDatabasesResponse ); expect(data).toMatchObject({ result: [expect.objectContaining({ uuid: "test-db-id" })], result_info: { count: 1, - total_count: 2, - }, - success: true, - }); - }); - - test("returns empty result for page beyond total", async ({ expect }) => { - const response = await mf.dispatchFetch( - `${BASE_URL}/d1/database?page=100` - ); - - expect(response.headers.get("Content-Type")).toBe("application/json"); - - const data = await expectValidResponse( - response, - zCloudflareD1ListDatabasesResponse - ); - - expect(data).toMatchObject({ - result: [], - result_info: { - count: 0, - page: 100, - total_count: 2, }, success: true, }); @@ -172,7 +119,7 @@ INSERT INTO users (name, email) VALUES ('Bob', 'bob@example.com'); const data = await expectValidResponse( response, - zCloudflareD1RawDatabaseQueryResponse + zD1RawDatabaseQueryResponse ); expect(data.success).toBe(true); @@ -205,7 +152,7 @@ INSERT INTO users (name, email) VALUES ('Bob', 'bob@example.com'); const data = await expectValidResponse( response, - zCloudflareD1RawDatabaseQueryResponse + zD1RawDatabaseQueryResponse ); expect(data).toMatchObject({ @@ -243,7 +190,7 @@ INSERT INTO users (name, email) VALUES ('Bob', 'bob@example.com'); const data = await expectValidResponse( response, - zCloudflareD1RawDatabaseQueryResponse + zD1RawDatabaseQueryResponse ); expect(data).toMatchObject({ @@ -290,7 +237,7 @@ INSERT INTO users (name, email) VALUES ('Bob', 'bob@example.com'); ); expect(data).toMatchObject({ - errors: [expect.objectContaining({ message: "Database not found" })], + errors: [expect.objectContaining({ code: 7404 })], success: false, }); }); @@ -325,30 +272,6 @@ INSERT INTO users (name, email) VALUES ('Bob', 'bob@example.com'); }); describe("validation", () => { - test("returns 400 for invalid query parameters", async ({ expect }) => { - const response = await mf.dispatchFetch( - `${BASE_URL}/d1/database?page=invalid` - ); - - expect(response.headers.get("Content-Type")).toBe("application/json"); - - const data = await expectValidResponse( - response, - zD1ApiResponseCommonFailure, - 400 - ); - - expect(data).toMatchObject({ - errors: [ - expect.objectContaining({ - code: 10001, - message: expect.stringContaining("page"), - }), - ], - success: false, - }); - }); - test("returns 400 for invalid batch item", async ({ expect }) => { const response = await mf.dispatchFetch( `${BASE_URL}/d1/database/test-db-id/raw`, diff --git a/packages/miniflare/test/plugins/local-explorer/do.spec.ts b/packages/miniflare/test/plugins/local-explorer/do.spec.ts index 116a78da0687..33691e3b7835 100644 --- a/packages/miniflare/test/plugins/local-explorer/do.spec.ts +++ b/packages/miniflare/test/plugins/local-explorer/do.spec.ts @@ -19,9 +19,6 @@ interface ListNamespacesResponse { result: DONamespace[]; result_info: { count: number; - page: number; - per_page: number; - total_count: number; }; errors: Array<{ code: number; message: string }>; messages: Array<{ code: number; message: string }>; @@ -82,7 +79,7 @@ describe("Durable Objects API", () => { }); describe("GET /workers/durable_objects/namespaces", () => { - test("lists available DO namespaces", async ({ expect }) => { + test("lists all available DO namespaces", async ({ expect }) => { const response = await mf.dispatchFetch( `${BASE_URL}/workers/durable_objects/namespaces` ); @@ -115,26 +112,6 @@ describe("Durable Objects API", () => { `); expect(data.result_info).toMatchObject({ count: 2, - page: 1, - per_page: 20, - total_count: 2, - }); - }); - - test("respects pagination parameters", async ({ expect }) => { - const response = await mf.dispatchFetch( - `${BASE_URL}/workers/durable_objects/namespaces?page=1&per_page=1` - ); - - expect(response.status).toBe(200); - const data = (await response.json()) as ListNamespacesResponse; - - expect(data.result.length).toBe(1); - expect(data.result_info).toMatchObject({ - count: 1, - page: 1, - per_page: 1, - total_count: 2, }); }); }); @@ -149,7 +126,7 @@ describe("Durable Objects API", () => { const data = (await response.json()) as ListObjectsResponse; expect(data.success).toBe(false); - expect(data.errors[0].message).toContain("Namespace not found"); + expect(data.errors[0].code).toBe(10066); }); }); @@ -812,7 +789,7 @@ describe("Durable Objects API", () => { const data = (await response.json()) as ErrorResponse; expect(data.success).toBe(false); - expect(data.errors[0].message).toContain("Namespace not found"); + expect(data.errors[0].code).toBe(10066); }); test("returns 400 for non-SQLite namespace", async ({ expect }) => { diff --git a/packages/miniflare/test/plugins/local-explorer/index.spec.ts b/packages/miniflare/test/plugins/local-explorer/index.spec.ts index 6a0c52a67ae5..d5f5cf023841 100644 --- a/packages/miniflare/test/plugins/local-explorer/index.spec.ts +++ b/packages/miniflare/test/plugins/local-explorer/index.spec.ts @@ -1,3 +1,4 @@ +import http from "node:http"; import { Miniflare } from "miniflare"; import { afterAll, beforeAll, describe, test } from "vitest"; import { LOCAL_EXPLORER_API_PATH } from "../../../src/plugins/core/constants"; @@ -26,27 +27,12 @@ describe("Local Explorer API validation", () => { }); describe("query parameter validation", () => { - test("uses default values when optional params not provided", async ({ - expect, - }) => { - const response = await mf.dispatchFetch( - `${BASE_URL}/storage/kv/namespaces` - ); - - expect(response.status).toBe(200); - expect(await response.json()).toMatchObject({ - success: true, - result_info: { - page: 1, - per_page: 20, - }, - }); - }); test("returns 400 for invalid type in query parameter", async ({ expect, }) => { + // Test query validation on listKVKeys endpoint which still has query params const response = await mf.dispatchFetch( - `${BASE_URL}/storage/kv/namespaces?page=not-a-number` + `${BASE_URL}/storage/kv/namespaces/test-kv-id/keys?limit=not-a-number` ); expect(response.status).toBe(400); @@ -56,7 +42,7 @@ describe("Local Explorer API validation", () => { { code: 10001, message: - 'page: Expected query param to be number but received "not-a-number"', + 'limit: Expected query param to be number but received "not-a-number"', }, ], }); @@ -65,8 +51,9 @@ describe("Local Explorer API validation", () => { test("returns 400 for invalid value (number out of range)", async ({ expect, }) => { + // Test with limit which has a minimum of 10 const response = await mf.dispatchFetch( - `${BASE_URL}/storage/kv/namespaces?page=-1` + `${BASE_URL}/storage/kv/namespaces/test-kv-id/keys?limit=0` ); expect(response.status).toBe(400); @@ -75,15 +62,15 @@ describe("Local Explorer API validation", () => { errors: [ { code: 10001, - message: "page: Number must be greater than or equal to 1", + message: "limit: Number must be greater than or equal to 10", }, ], }); }); - test("returns 400 for per_page exceeding maximum", async ({ expect }) => { + test("returns 400 for limit exceeding maximum", async ({ expect }) => { const response = await mf.dispatchFetch( - `${BASE_URL}/storage/kv/namespaces?per_page=1001` + `${BASE_URL}/storage/kv/namespaces/test-kv-id/keys?limit=1001` ); expect(response.status).toBe(400); @@ -92,7 +79,7 @@ describe("Local Explorer API validation", () => { errors: [ { code: 10001, - message: "per_page: Number must be less than or equal to 1000", + message: "limit: Number must be less than or equal to 1000", }, ], }); @@ -150,7 +137,7 @@ describe("Local Explorer API validation", () => { expect, }) => { const response = await mf.dispatchFetch( - `${BASE_URL}/storage/kv/namespaces?page=invalid` + `${BASE_URL}/storage/kv/namespaces/test-kv-id/keys?limit=invalid` ); expect(response.status).toBe(400); @@ -160,14 +147,13 @@ describe("Local Explorer API validation", () => { { code: 10001, message: - 'page: Expected query param to be number but received "invalid"', + 'limit: Expected query param to be number but received "invalid"', }, ], messages: [], result: null, }); }); - test("not found responses follow Cloudflare API format", async ({ expect, }) => { @@ -178,8 +164,85 @@ describe("Local Explorer API validation", () => { expect(response.status).toBe(404); expect(await response.json()).toMatchObject({ success: false, - errors: [{ code: 10000, message: "Namespace not found" }], + errors: [{ code: 10013, message: "list keys: 'namespace not found'" }], + }); + }); + }); + + test("allows requests from local origins, blocks external", async ({ + expect, + }) => { + // no origin is allowed (non-browser clients) + let res = await mf.dispatchFetch(`${BASE_URL}/storage/kv/namespaces`); + expect(res.status).toBe(200); + await res.arrayBuffer(); + + // localhost, 127.0.0.1, [::1] all allowed + for (const origin of [ + "http://localhost:8787", + "http://127.0.0.1:8787", + "http://[::1]:8787", + ]) { + res = await mf.dispatchFetch(`${BASE_URL}/storage/kv/namespaces`, { + headers: { Origin: origin }, }); + expect(res.status).toBe(200); + expect(res.headers.get("Access-Control-Allow-Origin")).toBe(origin); + await res.arrayBuffer(); + } + + // external origin blocked + res = await mf.dispatchFetch(`${BASE_URL}/storage/kv/namespaces`, { + headers: { Origin: "https://evil.com" }, + }); + expect(res.status).toBe(403); + await res.arrayBuffer(); + + // malformed origin blocked + res = await mf.dispatchFetch(`${BASE_URL}/storage/kv/namespaces`, { + headers: { Origin: "not-a-valid-url" }, + }); + expect(res.status).toBe(403); + await res.arrayBuffer(); + }); + + test("handles CORS preflight", async ({ expect }) => { + // allowed origin + let res = await mf.dispatchFetch(`${BASE_URL}/storage/kv/namespaces`, { + method: "OPTIONS", + headers: { Origin: "http://localhost:5173" }, + }); + expect(res.status).toBe(204); + expect(res.headers.get("Access-Control-Allow-Origin")).toBe( + "http://localhost:5173" + ); + expect(res.headers.get("Access-Control-Allow-Methods")).toBe( + "GET, POST, PUT, DELETE, OPTIONS" + ); + await res.arrayBuffer(); + + // blocked origin + res = await mf.dispatchFetch(`${BASE_URL}/storage/kv/namespaces`, { + method: "OPTIONS", + headers: { Origin: "https://attacker.com" }, + }); + expect(res.status).toBe(403); + await res.arrayBuffer(); + }); + + test("blocks DNS rebinding attacks via Host header", async ({ expect }) => { + const url = await mf.ready; + const status = await new Promise((resolve, reject) => { + const req = http.get( + `${url.origin}${LOCAL_EXPLORER_API_PATH}/storage/kv/namespaces`, + { setHost: false, headers: { Host: "evil.com" } }, + (res) => { + res.resume(); + resolve(res.statusCode ?? 0); + } + ); + req.on("error", reject); }); + expect(status).toBe(403); }); }); diff --git a/packages/miniflare/test/plugins/local-explorer/kv.spec.ts b/packages/miniflare/test/plugins/local-explorer/kv.spec.ts index 0242fc82c2ba..77a8922ab446 100644 --- a/packages/miniflare/test/plugins/local-explorer/kv.spec.ts +++ b/packages/miniflare/test/plugins/local-explorer/kv.spec.ts @@ -36,9 +36,7 @@ describe("KV API", () => { }); describe("GET /storage/kv/namespaces", () => { - test("lists available KV namespaces with default pagination", async ({ - expect, - }) => { + test("lists all available KV namespaces", async ({ expect }) => { const response = await mf.dispatchFetch( `${BASE_URL}/storage/kv/namespaces` ); @@ -56,9 +54,6 @@ describe("KV API", () => { ); expect(data.result_info).toMatchObject({ count: 3, - page: 1, - per_page: 20, - total_count: 3, }); }); @@ -104,42 +99,6 @@ describe("KV API", () => { ], }); }); - - test("pagination works", async ({ expect }) => { - const response = await mf.dispatchFetch( - `${BASE_URL}/storage/kv/namespaces?per_page=2&page=2` - ); - - expect(response.status).toBe(200); - // Sorted by ID: "another-kv-id", "test-kv-id", "zebra-kv-id" - // Page 2 with per_page=2 should return only "zebra-kv-id" - expect(await response.json()).toMatchObject({ - result: [expect.objectContaining({ id: "zebra-kv-id" })], - result_info: { - count: 1, - page: 2, - per_page: 2, - total_count: 3, - }, - }); - }); - - test("returns empty result for page beyond total", async ({ expect }) => { - const response = await mf.dispatchFetch( - `${BASE_URL}/storage/kv/namespaces?per_page=20&page=100` - ); - - expect(response.status).toBe(200); - expect(await response.json()).toMatchObject({ - result: [], - result_info: { - count: 0, - page: 100, - per_page: 20, - total_count: 3, - }, - }); - }); }); describe("GET /storage/kv/namespaces/:namespaceId/keys", () => { @@ -189,7 +148,7 @@ describe("KV API", () => { expect(response.status).toBe(404); expect(await response.json()).toMatchObject({ success: false, - errors: [expect.objectContaining({ message: "Namespace not found" })], + errors: [expect.objectContaining({ code: 10013 })], }); }); @@ -269,7 +228,7 @@ describe("KV API", () => { expect(response.status).toBe(404); expect(await response.json()).toMatchObject({ success: false, - errors: [expect.objectContaining({ message: "Key not found" })], + errors: [expect.objectContaining({ code: 10009 })], }); }); @@ -281,7 +240,7 @@ describe("KV API", () => { expect(response.status).toBe(404); expect(await response.json()).toMatchObject({ success: false, - errors: [expect.objectContaining({ message: "Namespace not found" })], + errors: [expect.objectContaining({ code: 10013 })], }); }); @@ -349,7 +308,7 @@ describe("KV API", () => { expect(response.status).toBe(404); expect(await response.json()).toMatchObject({ success: false, - errors: [expect.objectContaining({ message: "Namespace not found" })], + errors: [expect.objectContaining({ code: 10013 })], }); }); }); @@ -400,7 +359,7 @@ describe("KV API", () => { expect(response.status).toBe(404); expect(await response.json()).toMatchObject({ success: false, - errors: [expect.objectContaining({ message: "Namespace not found" })], + errors: [expect.objectContaining({ code: 10013 })], }); }); @@ -501,7 +460,7 @@ describe("KV API", () => { expect(response.status).toBe(404); expect(await response.json()).toMatchObject({ success: false, - errors: [expect.objectContaining({ message: "Namespace not found" })], + errors: [expect.objectContaining({ code: 10013 })], }); }); }); diff --git a/packages/wrangler/src/__tests__/d1/execute.test.ts b/packages/wrangler/src/__tests__/d1/execute.test.ts index cbcedf385143..316c93b1c29c 100644 --- a/packages/wrangler/src/__tests__/d1/execute.test.ts +++ b/packages/wrangler/src/__tests__/d1/execute.test.ts @@ -230,6 +230,23 @@ describe("execute", () => { ]); }); + it("should output JSON null for SQL NULL values with --json flag", async ({ + expect, + }) => { + setIsTTY(false); + writeWranglerConfig({ + d1_databases: [ + { binding: "DATABASE", database_name: "db", database_id: "xxxx" }, + ], + }); + + await runWrangler( + "d1 execute db --command 'SELECT 1 as id, null as name;' --json" + ); + const parsed = JSON.parse(std.out); + expect(parsed[0].results[0].name).toBeNull(); + }); + describe("duration formatting", () => { mockAccountId({ accountId: "some-account-id" }); mockApiToken(); diff --git a/packages/wrangler/src/d1/execute.ts b/packages/wrangler/src/d1/execute.ts index 51c603fe197b..8f08a72e5b29 100644 --- a/packages/wrangler/src/d1/execute.ts +++ b/packages/wrangler/src/d1/execute.ts @@ -33,7 +33,7 @@ import type { D1Result } from "@cloudflare/workers-types/experimental"; import type { ComplianceConfig, Config } from "@cloudflare/workers-utils"; export type QueryResult = { - results: Record[]; + results: Record[]; success: boolean; meta?: { duration?: number; @@ -328,9 +328,6 @@ async function executeLocally({ if (Array.isArray(value)) { value = `[${value.join(", ")}]`; } - if (value === null) { - value = "null"; - } return [key, value]; }) )