diff --git a/package.json b/package.json index 3ca6fbb0..a949d6cb 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ "type": "module", "imports": { "~docsCatalog": "./src/docs.json", + "#collectionsHost": "./dist/server.collectionsHost.js", "#toolsHost": "./dist/server.toolsHost.js" }, "exports": { diff --git a/src/__tests__/__snapshots__/options.defaults.test.ts.snap b/src/__tests__/__snapshots__/options.defaults.test.ts.snap index d784cfbe..200927e3 100644 --- a/src/__tests__/__snapshots__/options.defaults.test.ts.snap +++ b/src/__tests__/__snapshots__/options.defaults.test.ts.snap @@ -2,6 +2,7 @@ exports[`options defaults should return specific properties: defaults 1`] = ` { + "collectionModules": [], "contextManagement": false, "contextPath": "/", "contextUrl": "file:///", diff --git a/src/__tests__/__snapshots__/server.test.ts.snap b/src/__tests__/__snapshots__/server.test.ts.snap index 6ffe41b4..b14a77fe 100644 --- a/src/__tests__/__snapshots__/server.test.ts.snap +++ b/src/__tests__/__snapshots__/server.test.ts.snap @@ -9,6 +9,9 @@ exports[`runServer should allow server to be stopped, http stop server: diagnost [ "Server stats enabled.", ], + [ + "No external collections loaded.", + ], [ "No external resources loaded.", ], @@ -71,6 +74,9 @@ exports[`runServer should allow server to be stopped, stdio stop server: diagnos [ "Server stats enabled.", ], + [ + "No external collections loaded.", + ], [ "No external resources loaded.", ], @@ -133,6 +139,9 @@ exports[`runServer should attempt to run server, create transport, connect, and [ "Server stats enabled.", ], + [ + "No external collections loaded.", + ], [ "No external resources loaded.", ], @@ -206,6 +215,9 @@ exports[`runServer should attempt to run server, disable SIGINT handler: diagnos [ "Server stats enabled.", ], + [ + "No external collections loaded.", + ], [ "No external resources loaded.", ], @@ -274,6 +286,9 @@ exports[`runServer should attempt to run server, enable SIGINT handler explicitl [ "Server stats enabled.", ], + [ + "No external collections loaded.", + ], [ "No external resources loaded.", ], @@ -347,6 +362,9 @@ exports[`runServer should attempt to run server, log warnings for experimental o [ "Server stats enabled.", ], + [ + "No external collections loaded.", + ], [ "No external resources loaded.", ], @@ -435,6 +453,9 @@ exports[`runServer should attempt to run server, register a tool: diagnostics 1` [ "Server stats enabled.", ], + [ + "No external collections loaded.", + ], [ "No external resources loaded.", ], @@ -516,6 +537,9 @@ exports[`runServer should attempt to run server, register multiple tools: diagno [ "Server stats enabled.", ], + [ + "No external collections loaded.", + ], [ "No external resources loaded.", ], @@ -604,6 +628,9 @@ exports[`runServer should attempt to run server, use custom options: diagnostics [ "Server stats enabled.", ], + [ + "No external collections loaded.", + ], [ "No external resources loaded.", ], @@ -677,6 +704,9 @@ exports[`runServer should attempt to run server, use default tools, http: diagno [ "Server stats enabled.", ], + [ + "No external collections loaded.", + ], [ "No external resources loaded.", ], @@ -759,6 +789,9 @@ exports[`runServer should attempt to run server, use default tools, stdio: diagn [ "Server stats enabled.", ], + [ + "No external collections loaded.", + ], [ "No external resources loaded.", ], diff --git a/src/__tests__/server.collections.test.ts b/src/__tests__/server.collections.test.ts index c4fd399f..78f0d5e0 100644 --- a/src/__tests__/server.collections.test.ts +++ b/src/__tests__/server.collections.test.ts @@ -2,8 +2,9 @@ import { composeCollections } from '../server.collections'; import { getOptions, getSessionOptions } from '../options.context'; jest.mock('../options.context', () => ({ - getOptions: jest.fn(), - getSessionOptions: jest.fn() + getOptions: jest.fn(() => ({})), + getSessionOptions: jest.fn(() => ({ sessionId: 'test' })), + getLoggerOptions: jest.fn(() => ({})) })); describe('composeCollections', () => { diff --git a/src/options.collections.ts b/src/options.collections.ts new file mode 100644 index 00000000..ecbb98be --- /dev/null +++ b/src/options.collections.ts @@ -0,0 +1,17 @@ +import { type ToolOptions, setToolOptions } from './options.tools'; + +/** + * Options for records. A limited subset of options. + * + * @alias ToolOptions + */ +type CollectionOptions = ToolOptions; + +/** + * Return a refined set of options from global options for records. + * + * @alias setToolOptions + */ +const setCollectionOptions = setToolOptions; + +export { setCollectionOptions, type CollectionOptions }; diff --git a/src/options.defaults.ts b/src/options.defaults.ts index 3ee44588..2f38c09c 100644 --- a/src/options.defaults.ts +++ b/src/options.defaults.ts @@ -2,6 +2,7 @@ import { basename, resolve } from 'node:path'; import { pathToFileURL } from 'node:url'; import packageJson from '../package.json'; import { type ToolModule } from './server.toolsUser'; +import { type CollectionModule } from './server.collectionsUser'; import { getNodeMajorVersion } from './options.helpers'; /** @@ -47,6 +48,8 @@ import { getNodeMajorVersion } from './options.helpers'; * @property {typeof TOOL_MEMO_OPTIONS} toolMemoOptions - Tool-specific memoization options. * @property {ToolModule|ToolModule[]} toolModules - Array of external tool modules (ESM specs or paths) to be loaded and * registered with the server. + * @property {CollectionModule|CollectionModule[]} collectionModules - Array of external collection modules + * (ESM specs or paths) to be loaded and registered with the server. * @property urlRegex - Regular expression pattern for URL matching. * @property version - Version of the package. * @property whitelist - Central outbound-URL policy options. @@ -83,6 +86,7 @@ interface DefaultOptions { stats: StatsOptions; toolMemoOptions: Partial; toolModules: ToolModule | ToolModule[]; + collectionModules: CollectionModule | CollectionModule[]; urlRegex: RegExp; version: string; whitelist: WhitelistOptions; @@ -560,6 +564,7 @@ const DEFAULT_OPTIONS: DefaultOptions = { resourceModules: [], toolMemoOptions: TOOL_MEMO_OPTIONS, toolModules: [], + collectionModules: [], separator: DEFAULT_SEPARATOR, urlRegex: URL_REGEX, version: (process.env.NODE_ENV === 'local' && '0.0.0') || packageJson.version, diff --git a/src/server.collections.ts b/src/server.collections.ts index 32540522..09608c1a 100644 --- a/src/server.collections.ts +++ b/src/server.collections.ts @@ -1,21 +1,274 @@ +import { type ChildProcess } from 'node:child_process'; import { type AppSession, type GlobalOptions } from './options'; +import { formatUnknownError, log } from './logger'; +import { + spawnChildProcess, + shutdownChildProcess, + activeChildrenBySession, + type ChildHandle +} from './server.process'; import { getOptions, getSessionOptions } from './options.context'; -import { type McpCollectionCreator } from './collections'; +import { type McpCollectionCreator, type McpCollectionResult } from './collections'; +import { setCollectionOptions } from './options.collections'; +import { type CollectionDescriptor, type IpcResponse } from './server.collectionsIpc'; +import { normalizeCollections, type NormalizedCollectionEntry } from './server.collectionsUser'; + +/** + * Handle for a spawned Host process. + * + * @property manifest - Array of collection descriptors. + */ +type HostHandle = ChildHandle & { + collections: CollectionDescriptor[]; +}; + +/** + * Compute the allowlist for the Tools Host. + * + * @param {GlobalOptions} options - Global options. + * @returns Array of absolute directories to allow read access. + */ +const computeFsReadAllowlist = ({ contextPath }: GlobalOptions = getOptions()): string[] => { + const directories = new Set(); + + if (contextPath) { + directories.add(contextPath); + } + + return [...directories]; +}; + +/** + * Extract the names of built-in collections. + * + * @param builtinCreators - Array of built-in collection creators. + * @returns Set of collection names. + */ +const getBuiltInCollectionNames = (builtinCreators: McpCollectionCreator[]) => + new Set(builtinCreators.map((creator, index) => { + const [name] = creator() || []; + + if (!name) { + log.warn(`Built-in collection at index ${index} is missing the name property`); + } + + return name; + }).filter(Boolean)); + +/** + * Log warnings and errors from Tools' load. + * + * @param warningsErrors - Object containing warnings and errors + * @param warningsErrors.warnings - Log warnings + * @param warningsErrors.errors - Log errors + */ +const logWarningsErrors = ({ warnings = [], errors = [] }: { warnings?: string[], errors?: string[] } = {}) => { + if (Array.isArray(warnings) && warnings.length > 0) { + const lines = warnings.map(warning => ` - ${String(warning)}`); + + log.warn(`Collections load warnings (${warnings.length})\n${lines.join('\n')}`); + } + + if (Array.isArray(errors) && errors.length > 0) { + const lines = errors.map(error => ` - ${String(error)}`); + + log.error(`Collections load errors (${errors.length})\n${lines.join('\n')}`); + } +}; + +/** + * Get normalized "inline" modules. Inline modules can be internal or embedded and are explicitly trusted. + * + * @param {GlobalOptions} options - Global options. + * @param options.collectionModules - Array of modules to normalize + * @returns - Filtered array of normalized "inline" tool modules + */ +const getInlineCollections = ({ collectionModules }: GlobalOptions = getOptions()): NormalizedCollectionEntry[] => + normalizeCollections.memo(collectionModules).filter(module => module.type === 'tuple'); + +/** + * Get normalized "inline" modules. + * + * @param {GlobalOptions} options - Global options. + * @param options.collectionModules - Array of modules to normalize + * @returns - Filtered array of normalized "inline" tool modules + */ +const getInvalidCollections = ({ collectionModules }: GlobalOptions = getOptions()): NormalizedCollectionEntry[] => + normalizeCollections.memo(collectionModules).filter(module => module.type === 'invalid'); + +/** + * Debug a child process' stderr output. + * + * @param child - Child process to debug + * @param {AppSession} sessionOptions - Session options + */ +const debugChild = (child: ChildProcess, { sessionId } = getSessionOptions()) => { + const childPid = child.pid; + + const debugHandler = (chunk: Buffer | string) => { + const raw = String(chunk); + + if (!raw || !raw.trim()) { + return; + } + + // Split multi-line chunks so each line is tagged + const lines = raw.split(/\r?\n/).map(line => line.trim()).filter(Boolean); + + for (const line of lines) { + const tagged = `[collections-host pid=${childPid} sid=${sessionId}] ${line}`; + + // Default: debug-level passthrough + log.debug(tagged); + } + }; + + child.stderr?.on?.('data', debugHandler); + + return () => { + child.stderr?.off?.('data', debugHandler); + }; +}; + +const spawnCollectionHost = async ( + options: GlobalOptions = getOptions() +): Promise => { + const { pluginIsolation, pluginHost, nodeVersion } = options || {}; + const { loadTimeoutMs, invokeTimeoutMs } = pluginHost || {}; + + // const filePackageCollectionModules = []; + // const internalCollectionOptions = options; + const collectionOptions = setCollectionOptions(options); + + const handle = spawnChildProcess({ + importSpecifier: '#collectionsHost', + label: 'Collections Host', + isolation: { + mode: pluginIsolation === 'strict' ? 'strict' : 'none', + nodeVersion, + fsReadAllowlist: computeFsReadAllowlist() + }, + enableStderrDebug: child => debugChild(child) + }); + + // hello + await handle.request({ t: 'hello' }, 'hello:ack', loadTimeoutMs); + + // load + const loadAck = await handle.request>( + { t: 'load', specs: [], invokeTimeoutMs, collectionOptions }, + 'load:ack', + loadTimeoutMs + ); + + logWarningsErrors(loadAck); + + // manifest + const manifest = await handle.request>( + { t: 'manifest:get' }, + 'manifest:result', + loadTimeoutMs + ); + + return { ...handle, collections: manifest.collections as CollectionDescriptor[] }; +}; + +/** + * Dynamically proxies a remote child-process record callback across the IPC boundary. + * + * @param sourceName + * @param handle + * @param globalOpts + * @param handle.pluginHost + */ +const makeProxyCreators = ( + handle: HostHandle, + { pluginHost }: GlobalOptions = getOptions() +): McpCollectionCreator[] => handle.collections.map((collection): McpCollectionCreator => () => { + const name = collection.name; + const invokeTimeoutMs = Math.max(0, Number(pluginHost?.invokeTimeoutMs) || 0); + + const handler = async (args?: unknown): Promise => { + const response = await handle.request>( + { t: 'invoke', collectionId: collection.id, args }, + 'invoke:result', + invokeTimeoutMs + ); + + if ('ok' in response && response.ok === false) { + const invocationError = new Error(response.error?.message || 'Collection invocation failed', { cause: response.error?.cause }) as Error & { + code?: string; + details?: unknown; + }; + + if (response.error?.stack) { + invocationError.stack = response.error.stack; + } + + if (response.error?.code) { + invocationError.code = response.error?.code; + } + + const errorCause = response.error?.cause as { details?: unknown } | undefined; + + invocationError.details = response.error?.details || errorCause?.details; + throw invocationError; + } + + return response.result as McpCollectionResult; + }; + + return [name, handler]; +}); + +/** + * Best-effort Tools Host shutdown for the current session. + * + * Policy: + * - Primary grace defaults to 0 ms (internal-only, from DEFAULT_OPTIONS.pluginHost.gracePeriodMs) + * - Single fallback kill at grace + 200 ms to avoid racing simultaneous kills + * - Close logging for child(ren) stderr + * + * @param {GlobalOptions} options - Global options. + * @param {AppSession} sessionOptions - Session options. + */ +const sendCollectionsHostShutdown = async ( + { pluginHost }: GlobalOptions = getOptions(), + { sessionId }: AppSession = getSessionOptions() +): Promise => { + const handle = activeChildrenBySession.get(sessionId) as HostHandle | undefined; + + await shutdownChildProcess(handle, { + gracePeriodMs: Math.max(0, Number(pluginHost?.gracePeriodMs) || 0), + sessionId, + label: 'Collections Host' + }); +}; /** * Composes multi-source record collections across process boundaries. * * @param builtinCreators - * @param {GlobalOptions} _options - Global options. - * @param {AppSession} _session - Session options. + * @param {GlobalOptions} options - Global options. + * @param {AppSession} session - Session options. * @returns Promise array of collection creators. */ const composeCollections = async ( builtinCreators: McpCollectionCreator[], - _options: GlobalOptions = getOptions(), - _session: AppSession = getSessionOptions() + options: GlobalOptions = getOptions(), + session: AppSession = getSessionOptions() ): Promise => { // Wrap built-in creators to enforce trusted _isInternal. Ties into what options, session values are available. + const { collectionModules, nodeVersion, contextUrl, contextPath } = options; + const { sessionId } = session; + const existingSession = activeChildrenBySession.get(sessionId); + + if (existingSession) { + log.warn(`Existing Collections Host session detected ${sessionId}. Shutting down the existing host before creating a new one.`); + await sendCollectionsHostShutdown(); + } + + // Intercept and wrap built-in creators to enforce trusted isInternal: true status const securedBuiltinCreators = builtinCreators.map((creator): McpCollectionCreator => opt => { const [name, callback, config] = creator(opt); @@ -29,13 +282,147 @@ const composeCollections = async ( ]; }); - if (securedBuiltinCreators.length === 0) { + const updatedCollectionModules = Array.isArray(collectionModules) ? collectionModules : []; + const usedNames = getBuiltInCollectionNames(securedBuiltinCreators); + + if (updatedCollectionModules.length === 0) { + log.info('No external collections loaded.'); + } + + if (updatedCollectionModules.length === 0 && securedBuiltinCreators.length === 0) { return []; } - return securedBuiltinCreators; + const filePackageCreators: NormalizedCollectionEntry[] = []; + const invalidCreators = getInvalidCollections({ collectionModules, contextUrl, contextPath } as GlobalOptions); + const inlineCreators: NormalizedCollectionEntry[] = getInlineCollections({ collectionModules, contextUrl, contextPath } as GlobalOptions); + + const normalizeCollectionName = (collectionName?: string) => collectionName?.trim?.()?.toLowerCase?.(); + + invalidCreators.forEach(({ error }) => { + log.warn(error); + }); + + // collectionCreators.push(...filteredInlineCreators); + + const localCreators: McpCollectionCreator[] = []; + const hostedCreators: McpCollectionCreator[] = []; + + for (const creator of securedBuiltinCreators) { + const [, , config] = creator(options); + const runHost = typeof config?.runInChildProcess === 'function' + ? await config.runInChildProcess(options) + : Boolean(config?.runInChildProcess); + + if (runHost) { + hostedCreators.push(creator); + } else { + localCreators.push(creator); + } + } + + const filteredInlineCreators = inlineCreators.map(collection => + collection.value as McpCollectionCreator).filter(Boolean); + + /* + This is already taken care of as part of the getInlineCollections and normalizeCollections chain + const filteredInlineCreators = inlineCreators.map(collection => { + const creator = collection.value as McpCollectionCreator; + + if (!creator) { + return null; + } + + return (opts?: GlobalOptions) => { + const [name, callback, config] = creator(opts); + + return [ + name, + callback, + { + ...config, + isInternal: false // Override/strip to ensure untrusted collections remain sandboxed + } + ]; + }; + }).filter(Boolean) as McpCollectionCreator[]; + */ + + hostedCreators.push(...filteredInlineCreators); + + if (filePackageCreators.length && (!nodeVersion || nodeVersion < 22)) { + log.warn('External collection plugins require Node >= 22; skipping file-based collections.'); + } + + if (hostedCreators.length === 0) { + return localCreators; + } + + let host: HostHandle | undefined; + + // Clean up on exit or disconnect + const onChildExitOrDisconnect = () => { + if (!host) { + return; + } + + const current = activeChildrenBySession.get(sessionId); + + if (current && current.child === host.child) { + try { + host.closeStderr(); + log.info('Collections Host stderr reader closed.'); + } catch (error) { + log.error(`Failed to close Collections Host stderr reader: ${formatUnknownError(error)}`); + } + + activeChildrenBySession.delete(sessionId); + } + + host.child.off('exit', onChildExitOrDisconnect); + host.child.off('disconnect', onChildExitOrDisconnect); + }; + + try { + host = await spawnCollectionHost(options); + + // Filter manifest by reserved names BEFORE proxying + const filteredCollections = host.collections.filter(collection => { + const collectionName = normalizeCollectionName(collection.name); + + if (collectionName && usedNames.has(collectionName)) { + log.warn(`Skipping collection plugin "${collection.name}" – name already used by built-in/inline collection.`); + + return false; + } + + if (collectionName) { + usedNames.add(collectionName); + } + + return true; + }); + + const filteredHandle = { ...host, collections: filteredCollections } as HostHandle; + const proxiedCreators = makeProxyCreators(filteredHandle); + + activeChildrenBySession.set(sessionId, host); + + host.child.once('exit', onChildExitOrDisconnect); + host.child.once('disconnect', onChildExitOrDisconnect); + + return [...localCreators, ...proxiedCreators]; + } catch (error) { + log.warn(`Failed to start Collections Host; skipping hosted collections and continuing with built-in/inline collections. ${formatUnknownError(error)}`); + + return localCreators; + } }; export { - composeCollections + composeCollections, + debugChild, + logWarningsErrors, + makeProxyCreators, + sendCollectionsHostShutdown }; diff --git a/src/server.collectionsHost.ts b/src/server.collectionsHost.ts new file mode 100644 index 00000000..b03ce5df --- /dev/null +++ b/src/server.collectionsHost.ts @@ -0,0 +1,343 @@ +import { + type IpcRequest, + type CollectionDescriptor, + makeId +} from './server.collectionsIpc'; +import { serializeError, type SerializedError } from './server.processIpc'; +import { createProcessHost, type HostContext } from './server.processHost'; +// import { resolveExternalCreators } from './server.toolsHostCreator'; +import { DEFAULT_OPTIONS } from './options.defaults'; +// import { type ToolOptions } from './options.tools'; +// import { type McpTool, type McpToolCreator } from './mcpSdk'; +import { type McpCollectionCreator, type McpCollection } from './collections'; +import { resolveCreators } from './server.collectionsHostCreator'; +import { type CollectionOptions } from './options.collections'; + +/** + * SubType of IpcRequest for "load" requests. + */ +type LoadRequest = Extract; + +/** + * SubType of IpcRequest for "invoke" requests. + */ +type InvokeRequest = Extract; + +/** + * State object for the collections host. + */ +type HostState = { + collectionMap: Map; + // descriptors: ToolDescriptor[]; + descriptors: CollectionDescriptor[]; + invokeTimeoutMs: number; +}; + +/** + * Create a new host state object. + * + * @param invokeTimeoutMs + * @returns {HostState} + */ +const createHostState = (invokeTimeoutMs = DEFAULT_OPTIONS.pluginHost.invokeTimeoutMs): HostState => ({ + collectionMap: new Map(), + descriptors: [], + invokeTimeoutMs +}); + +/** + * Check if a value is an error or an error-like object. + * + * Handles cross-realm Error detection via tag checks for `[object Error]`, `[object AggregateError]`, + * and `[object DOMException]`. Does not treat `[object ErrorEvent]` as error-like in the + * Node context; add if your runtime can emit `ErrorEvent`. + * + * @param value + * @returns True if the value is an error-like object, false otherwise. + */ +const isErrorLike = (value: unknown) => { + if (!value || (typeof value !== 'object' && typeof value !== 'function')) { + return false; + } + + if (value instanceof Error || value instanceof AggregateError) { + return true; + } + + const tag = Object.prototype.toString.call(value); + + if (tag === '[object Error]' || tag === '[object AggregateError]' || tag === '[object DOMException]') { + return true; + } + + const val = value as Record; + const has = (key: string) => + Object.hasOwn(val, key) && typeof val[key] === 'string' && val[key].length > 0; + + if (!has('message')) { + return false; + } + + const isNameLike = has('name') && (val.name as string).toLowerCase().endsWith('error'); + const isStackLike = has('stack') && (val.stack as string).includes('\n'); + + return isNameLike || isStackLike; +}; + +/** + * Load external tool creators, realize them, and normalize `inputSchema` in the child. + * + * Stores the real Zod schema in memory for runtime validation and sends a JSON-safe schema in descriptors. + * + * @param {LoadRequest} request - Load request object. + * @returns New state object with updated tools/descriptors and warnings/errors. + */ +const performLoad = async (request: LoadRequest): Promise => { + const nextInvokeTimeout = typeof request?.invokeTimeoutMs === 'number' && Number.isFinite(request.invokeTimeoutMs) && request.invokeTimeoutMs > 0 + ? request.invokeTimeoutMs + : DEFAULT_OPTIONS.pluginHost.invokeTimeoutMs; + + const state = createHostState(nextInvokeTimeout); + const warnings: string[] = []; + const errors: string[] = []; + const options: CollectionOptions | undefined = request.options; + let module: unknown; + + for (const spec of request.specs || []) { + // Import the module. On fail, move to the next module. + try { + const dynamicImport = new Function('spec', 'return import(spec)') as (spec: string) => Promise; + + // let's export a common "collection" function for records/collections + module = await dynamicImport(spec); + } catch (error) { + errors.push(`Failed import: ${spec}: ${String((error as Error)?.message || error)}`); + continue; + } + + // Does the module export a creator function? On fail, move to the next module. + let creators: McpCollectionCreator[] = []; + + try { + creators = resolveCreators(module, options, { throwOnEmpty: true }); + } catch (error) { + warnings.push(`No usable creators in module ${spec}: ${String((error as Error)?.message || error)}`); + continue; + } + + // Finally, convert to JSON for manifest, store, push descriptor + for (const creator of creators) { + try { + const create = creator as (opts?: unknown) => McpCollection; + const collection = create(options); + + const collectionId = makeId(); + + state.collectionMap.set(collectionId, collection); + state.descriptors.push({ + id: collectionId, + name: collection[0], + source: spec + }); + } catch (error) { + warnings.push(`Collection creator threw while realizing: ${spec}: ${String((error as Error)?.message || error)}`); + } + } + + /* + try { + if (module && typeof module === 'object') { + if ('collection' in module && Array.isArray(module.collection)) { + collection = (module as { collection: McpCollection }).collection; + } else { + // Robust search for any exported CollectionCreators and realize them + const exportedFunc = Object.values(module).find(val => typeof val === 'function'); + + if (exportedFunc) { + const potentialSource = (exportedFunc as () => unknown)(); + + if (Array.isArray(potentialSource) && potentialSource.length >= 2) { + collection = potentialSource as McpCollection; + } + } + } + } + + if (!collection) { + throw new Error('collection missing or invalid'); + } + } catch (error) { + warnings.push(`No usable collection in module ${spec}: ${String((error as Error)?.message || error)}`); + continue; + } + + const collectionId = makeId(); + + state.collectionMap.set(collectionId, collection); + state.descriptors.push({ + id: collectionId, + name: collection[0], + source: spec + }); + */ + + /* + // Does the module export a creator function? On fail, move to the next module. + let creators: McpToolCreator[] = []; + + try { + creators = resolveExternalCreators(module, request.toolOptions, { throwOnEmpty: true }); + } catch (error) { + warnings.push(`No usable creators in module ${spec}: ${String((error as Error)?.message || error)}`); + continue; + } + */ + + // Finally, normalize module schema, convert to JSON for manifest, store, push descriptor + /* + for (const creator of creators) { + try { + const { tool, manifestSchema, warnings: creatorWarnings } = normalizeCreatorSchema(creator, toolOptions); + + warnings.push(...creatorWarnings); + + const toolId = makeId(); + + state.recordMap.set(toolId, tool as McpTool); + state.descriptors.push({ + id: toolId, + name: tool[0], + description: tool[1]?.description || '', + inputSchema: manifestSchema, + source: spec + }); + } catch (error) { + warnings.push(`Tool creator threw while realizing: ${spec}: ${String((error as Error)?.message || error)}`); + } + } + */ + } + + return { ...state, warnings, errors }; +}; + +/** + * Invoke a realized tool by id. Validates arguments against the in-memory Zod schema. + * + * @example + * // On validation failure, returns + * { ok: false, error: { code: 'INVALID_ARGS', details } } + * + * @param {HostState} state + * @param {InvokeRequest} request + * @param {HostContext} ctx + */ +const requestInvoke = async (state: HostState, request: InvokeRequest, ctx: HostContext) => { + const collection = state.collectionMap.get(request.collectionId); + + if (!collection) { + ctx.send({ + t: 'invoke:result', + id: request.id, + ok: false, + error: { message: 'Unknown collectionId' } + }); + + return; + } + + let settled = false; + + const timer = setTimeout(() => { + if (settled) { + return; + } + + settled = true; + + ctx.send({ + t: 'invoke:result', + id: request.id, + ok: false, + error: { message: 'Invoke timeout' } + }); + }, state.invokeTimeoutMs); + + timer?.unref?.(); + + const handler = collection[1]; + // const cfg = (tool[1] || {}) as Record; + // const schema = cfg.inputSchema; + + try { + // Child-side validation + const updatedRequestArgs = request.args; + + // Invoke the tool + const result = await Promise.resolve(handler(updatedRequestArgs)); + + // Some handlers may mistakenly return an Error instance instead of throwing. Normalize it to a failure. + if (isErrorLike(result)) { + const err: SerializedError = new Error('Internal error', { cause: { details: result } }); + + err.code = 'INTERNAL_ERROR'; + + throw err; + } + + if (!settled) { + settled = true; + clearTimeout(timer); + ctx.send({ t: 'invoke:result', id: request.id, ok: true, result }); + } + } catch (error) { + if (!settled) { + settled = true; + clearTimeout(timer); + ctx.send({ + t: 'invoke:result', + id: request.id, + ok: false, + error: serializeError(error as Error) + }); + } + } +}; + +/** + * Create the Collections Host: a generic child-process host wired with the record handlers. + * Built-in `hello`/`shutdown` handlers come from `createProcessHost`. + */ +const createCollectionsHost = () => { + let state: HostState = createHostState(); + + return createProcessHost({ + load: async (request, ctx) => { + const loaded = await performLoad(request as LoadRequest); + + state = { + collectionMap: loaded.collectionMap, + descriptors: loaded.descriptors, + invokeTimeoutMs: loaded.invokeTimeoutMs + }; + + ctx.send({ t: 'load:ack', id: request.id, warnings: loaded.warnings, errors: loaded.errors }); + }, + 'manifest:get': (request, ctx) => { + ctx.send({ t: 'manifest:result', id: request.id, collections: state.descriptors }); + }, + invoke: async (request, ctx) => { + await requestInvoke(state, request as InvokeRequest, ctx); + } + }); +}; + +// createProcessHost internally guards on `process.send`, so this is safe at module load. +createCollectionsHost(); + +export { + // normalizeCreatorSchema, + performLoad, + requestInvoke, + createCollectionsHost +}; diff --git a/src/server.collectionsHostCreator.ts b/src/server.collectionsHostCreator.ts new file mode 100644 index 00000000..df6d3670 --- /dev/null +++ b/src/server.collectionsHostCreator.ts @@ -0,0 +1,114 @@ +import { type McpCollectionCreator, type McpCollection } from './collections'; + +/** + * Guard for an array of creators. File-scoped helper. + * + * @private + * @param value + * @returns `true` if value is an array of functions. + */ +const isCreatorsArray = (value: unknown): value is McpCollectionCreator[] => + Array.isArray(value) && value.length > 0 && value.every(fn => typeof fn === 'function'); + +/** + * Guard for tool tuple. File-scoped helper. + * + * @private + * @param value + * @returns `true` if value is a tool tuple. + */ +const isRealizedTuple = (value: unknown): value is McpCollection => + Array.isArray(value) && + value.length >= 2 && + typeof value[0] === 'string' && + typeof (value as unknown[])[1] === 'function'; + +/** + * Wrap a realized tool tuple in a creator function that returns the tuple itself. + * File-scoped helper. + * + * @private + * @param cached + * @returns A normalized creator function that returns the cached tool tuple. + */ +const wrapCachedTuple = (cached: McpCollection): McpCollectionCreator => { + const wrapped: McpCollectionCreator = () => cached; + + return wrapped as McpCollectionCreator; +}; + +/** + * Options for resolveExternalCreators. + */ +type ResolveOptions = { + throwOnEmpty?: boolean; +}; + +/** + * Minimally filter, resolve, then cache tool creators from external module export during the child process. + * + * @param moduleExports - The module exports object from the child process. + * @param options - Options to pass to creators. + * @param settings - Optional settings. + * @param settings.throwOnEmpty - Throw an error if no tool creators are found. Defaults to false. + */ +const resolveCreators = ( + moduleExports: unknown, + options?: Record | undefined, + { throwOnEmpty = false }: ResolveOptions = {} +): McpCollectionCreator[] => { + const mod = moduleExports as any; + const candidates: unknown[] = [mod?.default, mod].filter(Boolean); + + const observed: string[] = []; + + for (const candidate of candidates) { + if (typeof candidate === 'function') { + observed.push('function'); + try { + const result = (candidate as (o?: unknown) => unknown)(options); + + if (isRealizedTuple(result)) { + return [wrapCachedTuple(result)]; + } + + if (isCreatorsArray(result)) { + observed.push('creators[]'); + + return result; + } + + observed.push(Array.isArray(result) ? 'array' : typeof result); + } catch { + // Move to next candidate + } + + continue; + } + + if (isCreatorsArray(candidate)) { + observed.push('creators[]'); + + return candidate as McpCollectionCreator[]; + } + + // Note shape for diagnostics if we end up throwing on empty + observed.push(Array.isArray(candidate) ? 'array' : typeof candidate); + } + + if (throwOnEmpty) { + const shapes = observed.length ? ` Observed candidate shapes: ${observed.join(', ')}` : ''; + + throw new Error([ + `No usable collection creators found from module. ${shapes}`, + 'Expected one of:', + '- default export: a collection creator (function that returns [name, handler])', + '- default export: a function that returns an array of collection creators', + '- default export: an array of collection creators' + ].join('\n')); + } + + return []; +}; + +export { resolveCreators, type ResolveOptions }; diff --git a/src/server.collectionsIpc.ts b/src/server.collectionsIpc.ts new file mode 100644 index 00000000..a42b0efb --- /dev/null +++ b/src/server.collectionsIpc.ts @@ -0,0 +1,90 @@ +import { type CollectionOptions } from './options.collections'; +import { + send, + awaitIpc, + makeId, + matchResponse, + serializeError, + type SerializedError +} from './server.processIpc'; + +/** + * IPC (Inter-Process Communication) request messages. + * + * - `hello` - Sent by the host to the process to acknowledge receipt. + * - `load` - Sent by the host to the process to load tools. + * - `manifest:get` - Sent by the host to the process to request a list of available tools. + * - `invoke` - Sent by the host to the process to invoke a tool. + * - `shutdown` - Sent by the host to the process to shutdown. + * + * @property t - Message type. + * @property id - Message identifier. + * @property specs - List of tool module specifiers to load. + * @property invokeTimeoutMs - Timeout for tool invocations. + * @property {CollectionOptions} options - Options to pass to creators. + * @property session - Session object to pass to creators. + * @property isInternal - Indicates if the request is internal. + */ +type IpcRequest = + | { t: 'hello'; id: string } | + { t: 'load'; id: string; specs: string[]; invokeTimeoutMs?: number; options?: CollectionOptions; session?: unknown; isInternal?: boolean } | + { t: 'manifest:get'; id: string } | + { t: 'invoke'; id: string; collectionId: string; args: unknown; options?: unknown; session?: unknown; isInternal?: boolean } | + { t: 'shutdown'; id: string }; + +/** + * Collection descriptor object for IPC. + * + * @property id - Collection identifier. + * @property name - Collection name. + * @property source - Collection module specifier. + */ +type CollectionDescriptor = { + id: string; + name: string; + source?: string; +}; + +/** + * Inter-Process Communication (IPC) responses. + * + * Types: + * - 'hello:ack': Acknowledgment message for a "hello" operation, including an identifier. + * - 'load:ack': Acknowledgment message for a "load" operation, including an identifier, + * and arrays of warnings and errors. + * - 'manifest:result': Message containing the result of a "manifest" operation, including an + * identifier and a list of tool descriptors. + * - 'invoke:result' (success case): Message containing the result of a successful "invoke" + * operation, including an identifier, a success flag, and the result. + * - 'invoke:result' (failure case): Message containing the result of a failed "invoke" + * operation, including an identifier, a failure flag, and an error descriptor. + * - 'shutdown:ack': Acknowledgment message for a "shutdown" operation, including an identifier. + * + * @property t - Message type. + * @property id - Message identifier. + * @property warnings - List of warnings generated during tool loading. + * @property errors - List of errors generated during tool loading. + * @property {ToolDescriptor[]} tools - List of available tools. + * @property ok - Success flag. + * @property result - Result of the operation. + * @property {SerializedError} error - Error descriptor. + */ +type IpcResponse = + | { t: 'hello:ack'; id: string } | + { t: 'load:ack'; id: string; warnings: string[]; errors: string[] } | + { t: 'manifest:result'; id: string; collections: CollectionDescriptor[] } | + { t: 'invoke:result'; id: string; ok: true; result: unknown } | + { t: 'invoke:result'; id: string; ok: false; error: SerializedError } | + { t: 'shutdown:ack'; id: string }; + +export { + send, + awaitIpc, + makeId, + matchResponse, + serializeError, + type IpcRequest, + type IpcResponse, + type CollectionDescriptor, + type SerializedError +}; diff --git a/src/server.collectionsUser.ts b/src/server.collectionsUser.ts new file mode 100644 index 00000000..161467d0 --- /dev/null +++ b/src/server.collectionsUser.ts @@ -0,0 +1,195 @@ +import { memo } from './server.caching'; +import { sanitizeDataProp } from './server.toolsUser'; +import { type McpCollection } from './collections'; +import { type GlobalOptions } from './options'; +import { type CollectionOptions } from './options.collections'; + +/** + * Inline tool options. + * + * Alias of {@link GlobalOptions}. + * + * @note Author-facing configuration. + */ +type CollectionInternalOptions = GlobalOptions; + +/** + * External tool options. + * + * Alias of {@link ToolOptions}. + * + * @note Author-facing configuration. + */ +type CollectionExternalOptions = CollectionOptions; + +/** + * A normalized tool entry for normalizing values for strings and tool creators. + * + * @property type - Classification of the entry (tuple, invalid) + * @property index - The original input index (for diagnostics) + * @property original - The original input value + * @property value - The final consumer value (string or creator) + * @property collectionName - The collection name for tuple/object/function entries + * @property error - Error message for invalid entries + */ +type NormalizedCollectionEntry = { + type: 'tuple' | 'invalid'; + index: number; + original: unknown; + value: string | CollectionCreator; + collectionName?: string | undefined; + error?: string | undefined; +}; + +/** + * A general tool entry for normalizing values for creators. + */ +type CreatorEntry = Pick; + +/** + * A function that returns a tuple `Collection`. An MCP collection of records "wrapper", or "creator". + * + * - `CollectionExternalOptions` is a limited subset of `CollectionInternalOptions` for external filePackage creators. + * - `CollectionInternalOptions` is available for inline and built-in collection of records creators. + * + * @note Author-facing configuration. + * @example A creator function. The handler may be async or sync. + * () => [ + * 'creatorRecord', + * async (args) => { ... } + * ] + */ +type CollectionCreator = (options?: CollectionExternalOptions | CollectionInternalOptions) => McpCollection; + +/** + * An array of normalized config values. + * + * - `string` - file path or package id + * - `CollectionCreator` - function creator + * + * @note Author-facing multi-collection configuration. + * @example An array/list of normalized config values + * [ + * './a/file/path/collection.mjs', + * () => [ + * 'creatorCollection', + * async (args) => { ... } + * ] + * ]; + */ +type CollectionModule = ReadonlyArray; + +/** + * Normalize a tuple config into a collection of records' creator function. + * + * @param config - The array configuration to normalize. + * @returns A collection of records' creator function, or undefined if the config is invalid. + */ +const normalizeTuple = (config: unknown): CreatorEntry | undefined => { + if (!Array.isArray(config) || config.length < 2) { + return undefined; + } + + const name = sanitizeDataProp(config, '0'); + const handler = sanitizeDataProp(config, '1'); + + if (!name || !handler) { + return undefined; + } + + const updatedName = (name.value as string)?.trim?.() || undefined; + const updatedHandler = typeof handler.value === 'function' ? handler.value : undefined; + + if (!updatedName || !updatedHandler) { + return undefined; + } + + const creator: CollectionCreator = () => [ + updatedName as string, + // updatedHandler as (args: unknown) => unknown | Promise, + updatedHandler, + { + runInChildProcess: true, + isInternal: false + } + ]; + + return { + original: config, + collectionName: updatedName as string, + type: 'tuple', + value: creator + }; +}; + +/** + * Memoize the `normalizeTuple` function. + */ +normalizeTuple.memo = memo(normalizeTuple, { cacheErrors: false, keyHash: args => args[0] }); + +/** + * Normalize the collection of record(s) configuration(s) into a normalized collection entry. + * + * @example Falsy values carried through to retain indexing on messaging + * Input: [ + * () => ['a', { inputSchema: {} }, () => {}], + * undefined, + * { name: 'b', description: 'b', inputSchema: {}, handler: () => {} } + * ] + * Output: ['creator', 'invalid', 'object'] + * + * @param config - The configuration(s) to normalize. + * @returns An array of normalized collection entries. + */ +const normalizeCollections = (config: any): NormalizedCollectionEntry[] => { + const updatedConfigs = (normalizeTuple.memo(config) && [config]) || (Array.isArray(config) && config) || [config]; + const normalizedConfigs: NormalizedCollectionEntry[] = []; + + const flattenedConfigs = updatedConfigs.flatMap((item: unknown) => + (normalizeTuple.memo(item) && [item]) || (Array.isArray(item) && item) || [item]); + + flattenedConfigs.forEach((config: unknown, index: number) => { + if (normalizeTuple.memo(config)) { + normalizedConfigs.push({ + index, + ...normalizeTuple.memo(config) as CreatorEntry + }); + + return; + } + + const err = `createMcpCollection: invalid configuration used at index ${index}: Unsupported type ${typeof config}`; + + normalizedConfigs.push({ + index, + original: config, + type: 'invalid', + value: err, + error: err + }); + }); + + return normalizedConfigs; +}; + +/** + * Memoized version of normalizeCollections. + * + * @note Review the memoization used in server.toolsUser.ts for the final + * implementation. Currently, this is a low-level temporary solution. + */ +normalizeCollections.memo = memo(normalizeCollections, { + cacheErrors: false, + keyHash: args => args[0] +}); + +export { + normalizeCollections, + normalizeTuple, + type CollectionInternalOptions, + type CollectionExternalOptions, + type NormalizedCollectionEntry, + type CollectionCreator, + type CreatorEntry, + type CollectionModule +}; diff --git a/src/server.ts b/src/server.ts index c611821c..50e95088 100644 --- a/src/server.ts +++ b/src/server.ts @@ -545,6 +545,7 @@ runServer.memo = memo( export { runServer, + registerServerCollections, registerServerResources, registerServerTools, type ServerInstance, diff --git a/tests/e2e/__snapshots__/stdioTransport.test.ts.snap b/tests/e2e/__snapshots__/stdioTransport.test.ts.snap index 50256a48..651b7dd7 100644 --- a/tests/e2e/__snapshots__/stdioTransport.test.ts.snap +++ b/tests/e2e/__snapshots__/stdioTransport.test.ts.snap @@ -149,6 +149,8 @@ exports[`Logging should allow setting logging options, stderr 1`] = ` "[INFO]: Server logging enabled. ", "[INFO]: Server stats enabled. +", + "[INFO]: No external collections loaded. ", "[INFO]: No external resources loaded. ",