From ecf84b8f160cac5e6cbea0dd4941294914fb5ab7 Mon Sep 17 00:00:00 2001 From: CD Cabrera Date: Thu, 6 Aug 2026 01:38:08 -0400 Subject: [PATCH 1/2] feat(collections): pf-4401 process spawn, ipc --- package.json | 1 + .../options.defaults.test.ts.snap | 1 + .../__snapshots__/server.test.ts.snap | 33 ++ src/__tests__/server.collections.test.ts | 5 +- src/options.collections.ts | 17 + src/options.defaults.ts | 5 + src/server.collections.ts | 403 +++++++++++++++++- src/server.collectionsHost.ts | 343 +++++++++++++++ src/server.collectionsHostCreator.ts | 114 +++++ src/server.collectionsIpc.ts | 90 ++++ src/server.collectionsUser.ts | 195 +++++++++ src/server.ts | 1 + .../__snapshots__/stdioTransport.test.ts.snap | 2 + 13 files changed, 1200 insertions(+), 10 deletions(-) create mode 100644 src/options.collections.ts create mode 100644 src/server.collectionsHost.ts create mode 100644 src/server.collectionsHostCreator.ts create mode 100644 src/server.collectionsIpc.ts create mode 100644 src/server.collectionsUser.ts diff --git a/package.json b/package.json index 3ca6fbb0..a949d6cb 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ "type": "module", "imports": { "~docsCatalog": "./src/docs.json", + "#collectionsHost": "./dist/server.collectionsHost.js", "#toolsHost": "./dist/server.toolsHost.js" }, "exports": { diff --git a/src/__tests__/__snapshots__/options.defaults.test.ts.snap b/src/__tests__/__snapshots__/options.defaults.test.ts.snap index d784cfbe..200927e3 100644 --- a/src/__tests__/__snapshots__/options.defaults.test.ts.snap +++ b/src/__tests__/__snapshots__/options.defaults.test.ts.snap @@ -2,6 +2,7 @@ exports[`options defaults should return specific properties: defaults 1`] = ` { + "collectionModules": [], "contextManagement": false, "contextPath": "/", "contextUrl": "file:///", diff --git a/src/__tests__/__snapshots__/server.test.ts.snap b/src/__tests__/__snapshots__/server.test.ts.snap index 6ffe41b4..b14a77fe 100644 --- a/src/__tests__/__snapshots__/server.test.ts.snap +++ b/src/__tests__/__snapshots__/server.test.ts.snap @@ -9,6 +9,9 @@ exports[`runServer should allow server to be stopped, http stop server: diagnost [ "Server stats enabled.", ], + [ + "No external collections loaded.", + ], [ "No external resources loaded.", ], @@ -71,6 +74,9 @@ exports[`runServer should allow server to be stopped, stdio stop server: diagnos [ "Server stats enabled.", ], + [ + "No external collections loaded.", + ], [ "No external resources loaded.", ], @@ -133,6 +139,9 @@ exports[`runServer should attempt to run server, create transport, connect, and [ "Server stats enabled.", ], + [ + "No external collections loaded.", + ], [ "No external resources loaded.", ], @@ -206,6 +215,9 @@ exports[`runServer should attempt to run server, disable SIGINT handler: diagnos [ "Server stats enabled.", ], + [ + "No external collections loaded.", + ], [ "No external resources loaded.", ], @@ -274,6 +286,9 @@ exports[`runServer should attempt to run server, enable SIGINT handler explicitl [ "Server stats enabled.", ], + [ + "No external collections loaded.", + ], [ "No external resources loaded.", ], @@ -347,6 +362,9 @@ exports[`runServer should attempt to run server, log warnings for experimental o [ "Server stats enabled.", ], + [ + "No external collections loaded.", + ], [ "No external resources loaded.", ], @@ -435,6 +453,9 @@ exports[`runServer should attempt to run server, register a tool: diagnostics 1` [ "Server stats enabled.", ], + [ + "No external collections loaded.", + ], [ "No external resources loaded.", ], @@ -516,6 +537,9 @@ exports[`runServer should attempt to run server, register multiple tools: diagno [ "Server stats enabled.", ], + [ + "No external collections loaded.", + ], [ "No external resources loaded.", ], @@ -604,6 +628,9 @@ exports[`runServer should attempt to run server, use custom options: diagnostics [ "Server stats enabled.", ], + [ + "No external collections loaded.", + ], [ "No external resources loaded.", ], @@ -677,6 +704,9 @@ exports[`runServer should attempt to run server, use default tools, http: diagno [ "Server stats enabled.", ], + [ + "No external collections loaded.", + ], [ "No external resources loaded.", ], @@ -759,6 +789,9 @@ exports[`runServer should attempt to run server, use default tools, stdio: diagn [ "Server stats enabled.", ], + [ + "No external collections loaded.", + ], [ "No external resources loaded.", ], diff --git a/src/__tests__/server.collections.test.ts b/src/__tests__/server.collections.test.ts index c4fd399f..78f0d5e0 100644 --- a/src/__tests__/server.collections.test.ts +++ b/src/__tests__/server.collections.test.ts @@ -2,8 +2,9 @@ import { composeCollections } from '../server.collections'; import { getOptions, getSessionOptions } from '../options.context'; jest.mock('../options.context', () => ({ - getOptions: jest.fn(), - getSessionOptions: jest.fn() + getOptions: jest.fn(() => ({})), + getSessionOptions: jest.fn(() => ({ sessionId: 'test' })), + getLoggerOptions: jest.fn(() => ({})) })); describe('composeCollections', () => { diff --git a/src/options.collections.ts b/src/options.collections.ts new file mode 100644 index 00000000..ecbb98be --- /dev/null +++ b/src/options.collections.ts @@ -0,0 +1,17 @@ +import { type ToolOptions, setToolOptions } from './options.tools'; + +/** + * Options for records. A limited subset of options. + * + * @alias ToolOptions + */ +type CollectionOptions = ToolOptions; + +/** + * Return a refined set of options from global options for records. + * + * @alias setToolOptions + */ +const setCollectionOptions = setToolOptions; + +export { setCollectionOptions, type CollectionOptions }; diff --git a/src/options.defaults.ts b/src/options.defaults.ts index 3ee44588..2f38c09c 100644 --- a/src/options.defaults.ts +++ b/src/options.defaults.ts @@ -2,6 +2,7 @@ import { basename, resolve } from 'node:path'; import { pathToFileURL } from 'node:url'; import packageJson from '../package.json'; import { type ToolModule } from './server.toolsUser'; +import { type CollectionModule } from './server.collectionsUser'; import { getNodeMajorVersion } from './options.helpers'; /** @@ -47,6 +48,8 @@ import { getNodeMajorVersion } from './options.helpers'; * @property {typeof TOOL_MEMO_OPTIONS} toolMemoOptions - Tool-specific memoization options. * @property {ToolModule|ToolModule[]} toolModules - Array of external tool modules (ESM specs or paths) to be loaded and * registered with the server. + * @property {CollectionModule|CollectionModule[]} collectionModules - Array of external collection modules + * (ESM specs or paths) to be loaded and registered with the server. * @property urlRegex - Regular expression pattern for URL matching. * @property version - Version of the package. * @property whitelist - Central outbound-URL policy options. @@ -83,6 +86,7 @@ interface DefaultOptions { stats: StatsOptions; toolMemoOptions: Partial; toolModules: ToolModule | ToolModule[]; + collectionModules: CollectionModule | CollectionModule[]; urlRegex: RegExp; version: string; whitelist: WhitelistOptions; @@ -560,6 +564,7 @@ const DEFAULT_OPTIONS: DefaultOptions = { resourceModules: [], toolMemoOptions: TOOL_MEMO_OPTIONS, toolModules: [], + collectionModules: [], separator: DEFAULT_SEPARATOR, urlRegex: URL_REGEX, version: (process.env.NODE_ENV === 'local' && '0.0.0') || packageJson.version, diff --git a/src/server.collections.ts b/src/server.collections.ts index 32540522..09608c1a 100644 --- a/src/server.collections.ts +++ b/src/server.collections.ts @@ -1,21 +1,274 @@ +import { type ChildProcess } from 'node:child_process'; import { type AppSession, type GlobalOptions } from './options'; +import { formatUnknownError, log } from './logger'; +import { + spawnChildProcess, + shutdownChildProcess, + activeChildrenBySession, + type ChildHandle +} from './server.process'; import { getOptions, getSessionOptions } from './options.context'; -import { type McpCollectionCreator } from './collections'; +import { type McpCollectionCreator, type McpCollectionResult } from './collections'; +import { setCollectionOptions } from './options.collections'; +import { type CollectionDescriptor, type IpcResponse } from './server.collectionsIpc'; +import { normalizeCollections, type NormalizedCollectionEntry } from './server.collectionsUser'; + +/** + * Handle for a spawned Host process. + * + * @property manifest - Array of collection descriptors. + */ +type HostHandle = ChildHandle & { + collections: CollectionDescriptor[]; +}; + +/** + * Compute the allowlist for the Tools Host. + * + * @param {GlobalOptions} options - Global options. + * @returns Array of absolute directories to allow read access. + */ +const computeFsReadAllowlist = ({ contextPath }: GlobalOptions = getOptions()): string[] => { + const directories = new Set(); + + if (contextPath) { + directories.add(contextPath); + } + + return [...directories]; +}; + +/** + * Extract the names of built-in collections. + * + * @param builtinCreators - Array of built-in collection creators. + * @returns Set of collection names. + */ +const getBuiltInCollectionNames = (builtinCreators: McpCollectionCreator[]) => + new Set(builtinCreators.map((creator, index) => { + const [name] = creator() || []; + + if (!name) { + log.warn(`Built-in collection at index ${index} is missing the name property`); + } + + return name; + }).filter(Boolean)); + +/** + * Log warnings and errors from Tools' load. + * + * @param warningsErrors - Object containing warnings and errors + * @param warningsErrors.warnings - Log warnings + * @param warningsErrors.errors - Log errors + */ +const logWarningsErrors = ({ warnings = [], errors = [] }: { warnings?: string[], errors?: string[] } = {}) => { + if (Array.isArray(warnings) && warnings.length > 0) { + const lines = warnings.map(warning => ` - ${String(warning)}`); + + log.warn(`Collections load warnings (${warnings.length})\n${lines.join('\n')}`); + } + + if (Array.isArray(errors) && errors.length > 0) { + const lines = errors.map(error => ` - ${String(error)}`); + + log.error(`Collections load errors (${errors.length})\n${lines.join('\n')}`); + } +}; + +/** + * Get normalized "inline" modules. Inline modules can be internal or embedded and are explicitly trusted. + * + * @param {GlobalOptions} options - Global options. + * @param options.collectionModules - Array of modules to normalize + * @returns - Filtered array of normalized "inline" tool modules + */ +const getInlineCollections = ({ collectionModules }: GlobalOptions = getOptions()): NormalizedCollectionEntry[] => + normalizeCollections.memo(collectionModules).filter(module => module.type === 'tuple'); + +/** + * Get normalized "inline" modules. + * + * @param {GlobalOptions} options - Global options. + * @param options.collectionModules - Array of modules to normalize + * @returns - Filtered array of normalized "inline" tool modules + */ +const getInvalidCollections = ({ collectionModules }: GlobalOptions = getOptions()): NormalizedCollectionEntry[] => + normalizeCollections.memo(collectionModules).filter(module => module.type === 'invalid'); + +/** + * Debug a child process' stderr output. + * + * @param child - Child process to debug + * @param {AppSession} sessionOptions - Session options + */ +const debugChild = (child: ChildProcess, { sessionId } = getSessionOptions()) => { + const childPid = child.pid; + + const debugHandler = (chunk: Buffer | string) => { + const raw = String(chunk); + + if (!raw || !raw.trim()) { + return; + } + + // Split multi-line chunks so each line is tagged + const lines = raw.split(/\r?\n/).map(line => line.trim()).filter(Boolean); + + for (const line of lines) { + const tagged = `[collections-host pid=${childPid} sid=${sessionId}] ${line}`; + + // Default: debug-level passthrough + log.debug(tagged); + } + }; + + child.stderr?.on?.('data', debugHandler); + + return () => { + child.stderr?.off?.('data', debugHandler); + }; +}; + +const spawnCollectionHost = async ( + options: GlobalOptions = getOptions() +): Promise => { + const { pluginIsolation, pluginHost, nodeVersion } = options || {}; + const { loadTimeoutMs, invokeTimeoutMs } = pluginHost || {}; + + // const filePackageCollectionModules = []; + // const internalCollectionOptions = options; + const collectionOptions = setCollectionOptions(options); + + const handle = spawnChildProcess({ + importSpecifier: '#collectionsHost', + label: 'Collections Host', + isolation: { + mode: pluginIsolation === 'strict' ? 'strict' : 'none', + nodeVersion, + fsReadAllowlist: computeFsReadAllowlist() + }, + enableStderrDebug: child => debugChild(child) + }); + + // hello + await handle.request({ t: 'hello' }, 'hello:ack', loadTimeoutMs); + + // load + const loadAck = await handle.request>( + { t: 'load', specs: [], invokeTimeoutMs, collectionOptions }, + 'load:ack', + loadTimeoutMs + ); + + logWarningsErrors(loadAck); + + // manifest + const manifest = await handle.request>( + { t: 'manifest:get' }, + 'manifest:result', + loadTimeoutMs + ); + + return { ...handle, collections: manifest.collections as CollectionDescriptor[] }; +}; + +/** + * Dynamically proxies a remote child-process record callback across the IPC boundary. + * + * @param sourceName + * @param handle + * @param globalOpts + * @param handle.pluginHost + */ +const makeProxyCreators = ( + handle: HostHandle, + { pluginHost }: GlobalOptions = getOptions() +): McpCollectionCreator[] => handle.collections.map((collection): McpCollectionCreator => () => { + const name = collection.name; + const invokeTimeoutMs = Math.max(0, Number(pluginHost?.invokeTimeoutMs) || 0); + + const handler = async (args?: unknown): Promise => { + const response = await handle.request>( + { t: 'invoke', collectionId: collection.id, args }, + 'invoke:result', + invokeTimeoutMs + ); + + if ('ok' in response && response.ok === false) { + const invocationError = new Error(response.error?.message || 'Collection invocation failed', { cause: response.error?.cause }) as Error & { + code?: string; + details?: unknown; + }; + + if (response.error?.stack) { + invocationError.stack = response.error.stack; + } + + if (response.error?.code) { + invocationError.code = response.error?.code; + } + + const errorCause = response.error?.cause as { details?: unknown } | undefined; + + invocationError.details = response.error?.details || errorCause?.details; + throw invocationError; + } + + return response.result as McpCollectionResult; + }; + + return [name, handler]; +}); + +/** + * Best-effort Tools Host shutdown for the current session. + * + * Policy: + * - Primary grace defaults to 0 ms (internal-only, from DEFAULT_OPTIONS.pluginHost.gracePeriodMs) + * - Single fallback kill at grace + 200 ms to avoid racing simultaneous kills + * - Close logging for child(ren) stderr + * + * @param {GlobalOptions} options - Global options. + * @param {AppSession} sessionOptions - Session options. + */ +const sendCollectionsHostShutdown = async ( + { pluginHost }: GlobalOptions = getOptions(), + { sessionId }: AppSession = getSessionOptions() +): Promise => { + const handle = activeChildrenBySession.get(sessionId) as HostHandle | undefined; + + await shutdownChildProcess(handle, { + gracePeriodMs: Math.max(0, Number(pluginHost?.gracePeriodMs) || 0), + sessionId, + label: 'Collections Host' + }); +}; /** * Composes multi-source record collections across process boundaries. * * @param builtinCreators - * @param {GlobalOptions} _options - Global options. - * @param {AppSession} _session - Session options. + * @param {GlobalOptions} options - Global options. + * @param {AppSession} session - Session options. * @returns Promise array of collection creators. */ const composeCollections = async ( builtinCreators: McpCollectionCreator[], - _options: GlobalOptions = getOptions(), - _session: AppSession = getSessionOptions() + options: GlobalOptions = getOptions(), + session: AppSession = getSessionOptions() ): Promise => { // Wrap built-in creators to enforce trusted _isInternal. Ties into what options, session values are available. + const { collectionModules, nodeVersion, contextUrl, contextPath } = options; + const { sessionId } = session; + const existingSession = activeChildrenBySession.get(sessionId); + + if (existingSession) { + log.warn(`Existing Collections Host session detected ${sessionId}. Shutting down the existing host before creating a new one.`); + await sendCollectionsHostShutdown(); + } + + // Intercept and wrap built-in creators to enforce trusted isInternal: true status const securedBuiltinCreators = builtinCreators.map((creator): McpCollectionCreator => opt => { const [name, callback, config] = creator(opt); @@ -29,13 +282,147 @@ const composeCollections = async ( ]; }); - if (securedBuiltinCreators.length === 0) { + const updatedCollectionModules = Array.isArray(collectionModules) ? collectionModules : []; + const usedNames = getBuiltInCollectionNames(securedBuiltinCreators); + + if (updatedCollectionModules.length === 0) { + log.info('No external collections loaded.'); + } + + if (updatedCollectionModules.length === 0 && securedBuiltinCreators.length === 0) { return []; } - return securedBuiltinCreators; + const filePackageCreators: NormalizedCollectionEntry[] = []; + const invalidCreators = getInvalidCollections({ collectionModules, contextUrl, contextPath } as GlobalOptions); + const inlineCreators: NormalizedCollectionEntry[] = getInlineCollections({ collectionModules, contextUrl, contextPath } as GlobalOptions); + + const normalizeCollectionName = (collectionName?: string) => collectionName?.trim?.()?.toLowerCase?.(); + + invalidCreators.forEach(({ error }) => { + log.warn(error); + }); + + // collectionCreators.push(...filteredInlineCreators); + + const localCreators: McpCollectionCreator[] = []; + const hostedCreators: McpCollectionCreator[] = []; + + for (const creator of securedBuiltinCreators) { + const [, , config] = creator(options); + const runHost = typeof config?.runInChildProcess === 'function' + ? await config.runInChildProcess(options) + : Boolean(config?.runInChildProcess); + + if (runHost) { + hostedCreators.push(creator); + } else { + localCreators.push(creator); + } + } + + const filteredInlineCreators = inlineCreators.map(collection => + collection.value as McpCollectionCreator).filter(Boolean); + + /* + This is already taken care of as part of the getInlineCollections and normalizeCollections chain + const filteredInlineCreators = inlineCreators.map(collection => { + const creator = collection.value as McpCollectionCreator; + + if (!creator) { + return null; + } + + return (opts?: GlobalOptions) => { + const [name, callback, config] = creator(opts); + + return [ + name, + callback, + { + ...config, + isInternal: false // Override/strip to ensure untrusted collections remain sandboxed + } + ]; + }; + }).filter(Boolean) as McpCollectionCreator[]; + */ + + hostedCreators.push(...filteredInlineCreators); + + if (filePackageCreators.length && (!nodeVersion || nodeVersion < 22)) { + log.warn('External collection plugins require Node >= 22; skipping file-based collections.'); + } + + if (hostedCreators.length === 0) { + return localCreators; + } + + let host: HostHandle | undefined; + + // Clean up on exit or disconnect + const onChildExitOrDisconnect = () => { + if (!host) { + return; + } + + const current = activeChildrenBySession.get(sessionId); + + if (current && current.child === host.child) { + try { + host.closeStderr(); + log.info('Collections Host stderr reader closed.'); + } catch (error) { + log.error(`Failed to close Collections Host stderr reader: ${formatUnknownError(error)}`); + } + + activeChildrenBySession.delete(sessionId); + } + + host.child.off('exit', onChildExitOrDisconnect); + host.child.off('disconnect', onChildExitOrDisconnect); + }; + + try { + host = await spawnCollectionHost(options); + + // Filter manifest by reserved names BEFORE proxying + const filteredCollections = host.collections.filter(collection => { + const collectionName = normalizeCollectionName(collection.name); + + if (collectionName && usedNames.has(collectionName)) { + log.warn(`Skipping collection plugin "${collection.name}" – name already used by built-in/inline collection.`); + + return false; + } + + if (collectionName) { + usedNames.add(collectionName); + } + + return true; + }); + + const filteredHandle = { ...host, collections: filteredCollections } as HostHandle; + const proxiedCreators = makeProxyCreators(filteredHandle); + + activeChildrenBySession.set(sessionId, host); + + host.child.once('exit', onChildExitOrDisconnect); + host.child.once('disconnect', onChildExitOrDisconnect); + + return [...localCreators, ...proxiedCreators]; + } catch (error) { + log.warn(`Failed to start Collections Host; skipping hosted collections and continuing with built-in/inline collections. ${formatUnknownError(error)}`); + + return localCreators; + } }; export { - composeCollections + composeCollections, + debugChild, + logWarningsErrors, + makeProxyCreators, + sendCollectionsHostShutdown }; diff --git a/src/server.collectionsHost.ts b/src/server.collectionsHost.ts new file mode 100644 index 00000000..b03ce5df --- /dev/null +++ b/src/server.collectionsHost.ts @@ -0,0 +1,343 @@ +import { + type IpcRequest, + type CollectionDescriptor, + makeId +} from './server.collectionsIpc'; +import { serializeError, type SerializedError } from './server.processIpc'; +import { createProcessHost, type HostContext } from './server.processHost'; +// import { resolveExternalCreators } from './server.toolsHostCreator'; +import { DEFAULT_OPTIONS } from './options.defaults'; +// import { type ToolOptions } from './options.tools'; +// import { type McpTool, type McpToolCreator } from './mcpSdk'; +import { type McpCollectionCreator, type McpCollection } from './collections'; +import { resolveCreators } from './server.collectionsHostCreator'; +import { type CollectionOptions } from './options.collections'; + +/** + * SubType of IpcRequest for "load" requests. + */ +type LoadRequest = Extract; + +/** + * SubType of IpcRequest for "invoke" requests. + */ +type InvokeRequest = Extract; + +/** + * State object for the collections host. + */ +type HostState = { + collectionMap: Map; + // descriptors: ToolDescriptor[]; + descriptors: CollectionDescriptor[]; + invokeTimeoutMs: number; +}; + +/** + * Create a new host state object. + * + * @param invokeTimeoutMs + * @returns {HostState} + */ +const createHostState = (invokeTimeoutMs = DEFAULT_OPTIONS.pluginHost.invokeTimeoutMs): HostState => ({ + collectionMap: new Map(), + descriptors: [], + invokeTimeoutMs +}); + +/** + * Check if a value is an error or an error-like object. + * + * Handles cross-realm Error detection via tag checks for `[object Error]`, `[object AggregateError]`, + * and `[object DOMException]`. Does not treat `[object ErrorEvent]` as error-like in the + * Node context; add if your runtime can emit `ErrorEvent`. + * + * @param value + * @returns True if the value is an error-like object, false otherwise. + */ +const isErrorLike = (value: unknown) => { + if (!value || (typeof value !== 'object' && typeof value !== 'function')) { + return false; + } + + if (value instanceof Error || value instanceof AggregateError) { + return true; + } + + const tag = Object.prototype.toString.call(value); + + if (tag === '[object Error]' || tag === '[object AggregateError]' || tag === '[object DOMException]') { + return true; + } + + const val = value as Record; + const has = (key: string) => + Object.hasOwn(val, key) && typeof val[key] === 'string' && val[key].length > 0; + + if (!has('message')) { + return false; + } + + const isNameLike = has('name') && (val.name as string).toLowerCase().endsWith('error'); + const isStackLike = has('stack') && (val.stack as string).includes('\n'); + + return isNameLike || isStackLike; +}; + +/** + * Load external tool creators, realize them, and normalize `inputSchema` in the child. + * + * Stores the real Zod schema in memory for runtime validation and sends a JSON-safe schema in descriptors. + * + * @param {LoadRequest} request - Load request object. + * @returns New state object with updated tools/descriptors and warnings/errors. + */ +const performLoad = async (request: LoadRequest): Promise => { + const nextInvokeTimeout = typeof request?.invokeTimeoutMs === 'number' && Number.isFinite(request.invokeTimeoutMs) && request.invokeTimeoutMs > 0 + ? request.invokeTimeoutMs + : DEFAULT_OPTIONS.pluginHost.invokeTimeoutMs; + + const state = createHostState(nextInvokeTimeout); + const warnings: string[] = []; + const errors: string[] = []; + const options: CollectionOptions | undefined = request.options; + let module: unknown; + + for (const spec of request.specs || []) { + // Import the module. On fail, move to the next module. + try { + const dynamicImport = new Function('spec', 'return import(spec)') as (spec: string) => Promise; + + // let's export a common "collection" function for records/collections + module = await dynamicImport(spec); + } catch (error) { + errors.push(`Failed import: ${spec}: ${String((error as Error)?.message || error)}`); + continue; + } + + // Does the module export a creator function? On fail, move to the next module. + let creators: McpCollectionCreator[] = []; + + try { + creators = resolveCreators(module, options, { throwOnEmpty: true }); + } catch (error) { + warnings.push(`No usable creators in module ${spec}: ${String((error as Error)?.message || error)}`); + continue; + } + + // Finally, convert to JSON for manifest, store, push descriptor + for (const creator of creators) { + try { + const create = creator as (opts?: unknown) => McpCollection; + const collection = create(options); + + const collectionId = makeId(); + + state.collectionMap.set(collectionId, collection); + state.descriptors.push({ + id: collectionId, + name: collection[0], + source: spec + }); + } catch (error) { + warnings.push(`Collection creator threw while realizing: ${spec}: ${String((error as Error)?.message || error)}`); + } + } + + /* + try { + if (module && typeof module === 'object') { + if ('collection' in module && Array.isArray(module.collection)) { + collection = (module as { collection: McpCollection }).collection; + } else { + // Robust search for any exported CollectionCreators and realize them + const exportedFunc = Object.values(module).find(val => typeof val === 'function'); + + if (exportedFunc) { + const potentialSource = (exportedFunc as () => unknown)(); + + if (Array.isArray(potentialSource) && potentialSource.length >= 2) { + collection = potentialSource as McpCollection; + } + } + } + } + + if (!collection) { + throw new Error('collection missing or invalid'); + } + } catch (error) { + warnings.push(`No usable collection in module ${spec}: ${String((error as Error)?.message || error)}`); + continue; + } + + const collectionId = makeId(); + + state.collectionMap.set(collectionId, collection); + state.descriptors.push({ + id: collectionId, + name: collection[0], + source: spec + }); + */ + + /* + // Does the module export a creator function? On fail, move to the next module. + let creators: McpToolCreator[] = []; + + try { + creators = resolveExternalCreators(module, request.toolOptions, { throwOnEmpty: true }); + } catch (error) { + warnings.push(`No usable creators in module ${spec}: ${String((error as Error)?.message || error)}`); + continue; + } + */ + + // Finally, normalize module schema, convert to JSON for manifest, store, push descriptor + /* + for (const creator of creators) { + try { + const { tool, manifestSchema, warnings: creatorWarnings } = normalizeCreatorSchema(creator, toolOptions); + + warnings.push(...creatorWarnings); + + const toolId = makeId(); + + state.recordMap.set(toolId, tool as McpTool); + state.descriptors.push({ + id: toolId, + name: tool[0], + description: tool[1]?.description || '', + inputSchema: manifestSchema, + source: spec + }); + } catch (error) { + warnings.push(`Tool creator threw while realizing: ${spec}: ${String((error as Error)?.message || error)}`); + } + } + */ + } + + return { ...state, warnings, errors }; +}; + +/** + * Invoke a realized tool by id. Validates arguments against the in-memory Zod schema. + * + * @example + * // On validation failure, returns + * { ok: false, error: { code: 'INVALID_ARGS', details } } + * + * @param {HostState} state + * @param {InvokeRequest} request + * @param {HostContext} ctx + */ +const requestInvoke = async (state: HostState, request: InvokeRequest, ctx: HostContext) => { + const collection = state.collectionMap.get(request.collectionId); + + if (!collection) { + ctx.send({ + t: 'invoke:result', + id: request.id, + ok: false, + error: { message: 'Unknown collectionId' } + }); + + return; + } + + let settled = false; + + const timer = setTimeout(() => { + if (settled) { + return; + } + + settled = true; + + ctx.send({ + t: 'invoke:result', + id: request.id, + ok: false, + error: { message: 'Invoke timeout' } + }); + }, state.invokeTimeoutMs); + + timer?.unref?.(); + + const handler = collection[1]; + // const cfg = (tool[1] || {}) as Record; + // const schema = cfg.inputSchema; + + try { + // Child-side validation + const updatedRequestArgs = request.args; + + // Invoke the tool + const result = await Promise.resolve(handler(updatedRequestArgs)); + + // Some handlers may mistakenly return an Error instance instead of throwing. Normalize it to a failure. + if (isErrorLike(result)) { + const err: SerializedError = new Error('Internal error', { cause: { details: result } }); + + err.code = 'INTERNAL_ERROR'; + + throw err; + } + + if (!settled) { + settled = true; + clearTimeout(timer); + ctx.send({ t: 'invoke:result', id: request.id, ok: true, result }); + } + } catch (error) { + if (!settled) { + settled = true; + clearTimeout(timer); + ctx.send({ + t: 'invoke:result', + id: request.id, + ok: false, + error: serializeError(error as Error) + }); + } + } +}; + +/** + * Create the Collections Host: a generic child-process host wired with the record handlers. + * Built-in `hello`/`shutdown` handlers come from `createProcessHost`. + */ +const createCollectionsHost = () => { + let state: HostState = createHostState(); + + return createProcessHost({ + load: async (request, ctx) => { + const loaded = await performLoad(request as LoadRequest); + + state = { + collectionMap: loaded.collectionMap, + descriptors: loaded.descriptors, + invokeTimeoutMs: loaded.invokeTimeoutMs + }; + + ctx.send({ t: 'load:ack', id: request.id, warnings: loaded.warnings, errors: loaded.errors }); + }, + 'manifest:get': (request, ctx) => { + ctx.send({ t: 'manifest:result', id: request.id, collections: state.descriptors }); + }, + invoke: async (request, ctx) => { + await requestInvoke(state, request as InvokeRequest, ctx); + } + }); +}; + +// createProcessHost internally guards on `process.send`, so this is safe at module load. +createCollectionsHost(); + +export { + // normalizeCreatorSchema, + performLoad, + requestInvoke, + createCollectionsHost +}; diff --git a/src/server.collectionsHostCreator.ts b/src/server.collectionsHostCreator.ts new file mode 100644 index 00000000..df6d3670 --- /dev/null +++ b/src/server.collectionsHostCreator.ts @@ -0,0 +1,114 @@ +import { type McpCollectionCreator, type McpCollection } from './collections'; + +/** + * Guard for an array of creators. File-scoped helper. + * + * @private + * @param value + * @returns `true` if value is an array of functions. + */ +const isCreatorsArray = (value: unknown): value is McpCollectionCreator[] => + Array.isArray(value) && value.length > 0 && value.every(fn => typeof fn === 'function'); + +/** + * Guard for tool tuple. File-scoped helper. + * + * @private + * @param value + * @returns `true` if value is a tool tuple. + */ +const isRealizedTuple = (value: unknown): value is McpCollection => + Array.isArray(value) && + value.length >= 2 && + typeof value[0] === 'string' && + typeof (value as unknown[])[1] === 'function'; + +/** + * Wrap a realized tool tuple in a creator function that returns the tuple itself. + * File-scoped helper. + * + * @private + * @param cached + * @returns A normalized creator function that returns the cached tool tuple. + */ +const wrapCachedTuple = (cached: McpCollection): McpCollectionCreator => { + const wrapped: McpCollectionCreator = () => cached; + + return wrapped as McpCollectionCreator; +}; + +/** + * Options for resolveExternalCreators. + */ +type ResolveOptions = { + throwOnEmpty?: boolean; +}; + +/** + * Minimally filter, resolve, then cache tool creators from external module export during the child process. + * + * @param moduleExports - The module exports object from the child process. + * @param options - Options to pass to creators. + * @param settings - Optional settings. + * @param settings.throwOnEmpty - Throw an error if no tool creators are found. Defaults to false. + */ +const resolveCreators = ( + moduleExports: unknown, + options?: Record | undefined, + { throwOnEmpty = false }: ResolveOptions = {} +): McpCollectionCreator[] => { + const mod = moduleExports as any; + const candidates: unknown[] = [mod?.default, mod].filter(Boolean); + + const observed: string[] = []; + + for (const candidate of candidates) { + if (typeof candidate === 'function') { + observed.push('function'); + try { + const result = (candidate as (o?: unknown) => unknown)(options); + + if (isRealizedTuple(result)) { + return [wrapCachedTuple(result)]; + } + + if (isCreatorsArray(result)) { + observed.push('creators[]'); + + return result; + } + + observed.push(Array.isArray(result) ? 'array' : typeof result); + } catch { + // Move to next candidate + } + + continue; + } + + if (isCreatorsArray(candidate)) { + observed.push('creators[]'); + + return candidate as McpCollectionCreator[]; + } + + // Note shape for diagnostics if we end up throwing on empty + observed.push(Array.isArray(candidate) ? 'array' : typeof candidate); + } + + if (throwOnEmpty) { + const shapes = observed.length ? ` Observed candidate shapes: ${observed.join(', ')}` : ''; + + throw new Error([ + `No usable collection creators found from module. ${shapes}`, + 'Expected one of:', + '- default export: a collection creator (function that returns [name, handler])', + '- default export: a function that returns an array of collection creators', + '- default export: an array of collection creators' + ].join('\n')); + } + + return []; +}; + +export { resolveCreators, type ResolveOptions }; diff --git a/src/server.collectionsIpc.ts b/src/server.collectionsIpc.ts new file mode 100644 index 00000000..a42b0efb --- /dev/null +++ b/src/server.collectionsIpc.ts @@ -0,0 +1,90 @@ +import { type CollectionOptions } from './options.collections'; +import { + send, + awaitIpc, + makeId, + matchResponse, + serializeError, + type SerializedError +} from './server.processIpc'; + +/** + * IPC (Inter-Process Communication) request messages. + * + * - `hello` - Sent by the host to the process to acknowledge receipt. + * - `load` - Sent by the host to the process to load tools. + * - `manifest:get` - Sent by the host to the process to request a list of available tools. + * - `invoke` - Sent by the host to the process to invoke a tool. + * - `shutdown` - Sent by the host to the process to shutdown. + * + * @property t - Message type. + * @property id - Message identifier. + * @property specs - List of tool module specifiers to load. + * @property invokeTimeoutMs - Timeout for tool invocations. + * @property {CollectionOptions} options - Options to pass to creators. + * @property session - Session object to pass to creators. + * @property isInternal - Indicates if the request is internal. + */ +type IpcRequest = + | { t: 'hello'; id: string } | + { t: 'load'; id: string; specs: string[]; invokeTimeoutMs?: number; options?: CollectionOptions; session?: unknown; isInternal?: boolean } | + { t: 'manifest:get'; id: string } | + { t: 'invoke'; id: string; collectionId: string; args: unknown; options?: unknown; session?: unknown; isInternal?: boolean } | + { t: 'shutdown'; id: string }; + +/** + * Collection descriptor object for IPC. + * + * @property id - Collection identifier. + * @property name - Collection name. + * @property source - Collection module specifier. + */ +type CollectionDescriptor = { + id: string; + name: string; + source?: string; +}; + +/** + * Inter-Process Communication (IPC) responses. + * + * Types: + * - 'hello:ack': Acknowledgment message for a "hello" operation, including an identifier. + * - 'load:ack': Acknowledgment message for a "load" operation, including an identifier, + * and arrays of warnings and errors. + * - 'manifest:result': Message containing the result of a "manifest" operation, including an + * identifier and a list of tool descriptors. + * - 'invoke:result' (success case): Message containing the result of a successful "invoke" + * operation, including an identifier, a success flag, and the result. + * - 'invoke:result' (failure case): Message containing the result of a failed "invoke" + * operation, including an identifier, a failure flag, and an error descriptor. + * - 'shutdown:ack': Acknowledgment message for a "shutdown" operation, including an identifier. + * + * @property t - Message type. + * @property id - Message identifier. + * @property warnings - List of warnings generated during tool loading. + * @property errors - List of errors generated during tool loading. + * @property {ToolDescriptor[]} tools - List of available tools. + * @property ok - Success flag. + * @property result - Result of the operation. + * @property {SerializedError} error - Error descriptor. + */ +type IpcResponse = + | { t: 'hello:ack'; id: string } | + { t: 'load:ack'; id: string; warnings: string[]; errors: string[] } | + { t: 'manifest:result'; id: string; collections: CollectionDescriptor[] } | + { t: 'invoke:result'; id: string; ok: true; result: unknown } | + { t: 'invoke:result'; id: string; ok: false; error: SerializedError } | + { t: 'shutdown:ack'; id: string }; + +export { + send, + awaitIpc, + makeId, + matchResponse, + serializeError, + type IpcRequest, + type IpcResponse, + type CollectionDescriptor, + type SerializedError +}; diff --git a/src/server.collectionsUser.ts b/src/server.collectionsUser.ts new file mode 100644 index 00000000..161467d0 --- /dev/null +++ b/src/server.collectionsUser.ts @@ -0,0 +1,195 @@ +import { memo } from './server.caching'; +import { sanitizeDataProp } from './server.toolsUser'; +import { type McpCollection } from './collections'; +import { type GlobalOptions } from './options'; +import { type CollectionOptions } from './options.collections'; + +/** + * Inline tool options. + * + * Alias of {@link GlobalOptions}. + * + * @note Author-facing configuration. + */ +type CollectionInternalOptions = GlobalOptions; + +/** + * External tool options. + * + * Alias of {@link ToolOptions}. + * + * @note Author-facing configuration. + */ +type CollectionExternalOptions = CollectionOptions; + +/** + * A normalized tool entry for normalizing values for strings and tool creators. + * + * @property type - Classification of the entry (tuple, invalid) + * @property index - The original input index (for diagnostics) + * @property original - The original input value + * @property value - The final consumer value (string or creator) + * @property collectionName - The collection name for tuple/object/function entries + * @property error - Error message for invalid entries + */ +type NormalizedCollectionEntry = { + type: 'tuple' | 'invalid'; + index: number; + original: unknown; + value: string | CollectionCreator; + collectionName?: string | undefined; + error?: string | undefined; +}; + +/** + * A general tool entry for normalizing values for creators. + */ +type CreatorEntry = Pick; + +/** + * A function that returns a tuple `Collection`. An MCP collection of records "wrapper", or "creator". + * + * - `CollectionExternalOptions` is a limited subset of `CollectionInternalOptions` for external filePackage creators. + * - `CollectionInternalOptions` is available for inline and built-in collection of records creators. + * + * @note Author-facing configuration. + * @example A creator function. The handler may be async or sync. + * () => [ + * 'creatorRecord', + * async (args) => { ... } + * ] + */ +type CollectionCreator = (options?: CollectionExternalOptions | CollectionInternalOptions) => McpCollection; + +/** + * An array of normalized config values. + * + * - `string` - file path or package id + * - `CollectionCreator` - function creator + * + * @note Author-facing multi-collection configuration. + * @example An array/list of normalized config values + * [ + * './a/file/path/collection.mjs', + * () => [ + * 'creatorCollection', + * async (args) => { ... } + * ] + * ]; + */ +type CollectionModule = ReadonlyArray; + +/** + * Normalize a tuple config into a collection of records' creator function. + * + * @param config - The array configuration to normalize. + * @returns A collection of records' creator function, or undefined if the config is invalid. + */ +const normalizeTuple = (config: unknown): CreatorEntry | undefined => { + if (!Array.isArray(config) || config.length < 2) { + return undefined; + } + + const name = sanitizeDataProp(config, '0'); + const handler = sanitizeDataProp(config, '1'); + + if (!name || !handler) { + return undefined; + } + + const updatedName = (name.value as string)?.trim?.() || undefined; + const updatedHandler = typeof handler.value === 'function' ? handler.value : undefined; + + if (!updatedName || !updatedHandler) { + return undefined; + } + + const creator: CollectionCreator = () => [ + updatedName as string, + // updatedHandler as (args: unknown) => unknown | Promise, + updatedHandler, + { + runInChildProcess: true, + isInternal: false + } + ]; + + return { + original: config, + collectionName: updatedName as string, + type: 'tuple', + value: creator + }; +}; + +/** + * Memoize the `normalizeTuple` function. + */ +normalizeTuple.memo = memo(normalizeTuple, { cacheErrors: false, keyHash: args => args[0] }); + +/** + * Normalize the collection of record(s) configuration(s) into a normalized collection entry. + * + * @example Falsy values carried through to retain indexing on messaging + * Input: [ + * () => ['a', { inputSchema: {} }, () => {}], + * undefined, + * { name: 'b', description: 'b', inputSchema: {}, handler: () => {} } + * ] + * Output: ['creator', 'invalid', 'object'] + * + * @param config - The configuration(s) to normalize. + * @returns An array of normalized collection entries. + */ +const normalizeCollections = (config: any): NormalizedCollectionEntry[] => { + const updatedConfigs = (normalizeTuple.memo(config) && [config]) || (Array.isArray(config) && config) || [config]; + const normalizedConfigs: NormalizedCollectionEntry[] = []; + + const flattenedConfigs = updatedConfigs.flatMap((item: unknown) => + (normalizeTuple.memo(item) && [item]) || (Array.isArray(item) && item) || [item]); + + flattenedConfigs.forEach((config: unknown, index: number) => { + if (normalizeTuple.memo(config)) { + normalizedConfigs.push({ + index, + ...normalizeTuple.memo(config) as CreatorEntry + }); + + return; + } + + const err = `createMcpCollection: invalid configuration used at index ${index}: Unsupported type ${typeof config}`; + + normalizedConfigs.push({ + index, + original: config, + type: 'invalid', + value: err, + error: err + }); + }); + + return normalizedConfigs; +}; + +/** + * Memoized version of normalizeCollections. + * + * @note Review the memoization used in server.toolsUser.ts for the final + * implementation. Currently, this is a low-level temporary solution. + */ +normalizeCollections.memo = memo(normalizeCollections, { + cacheErrors: false, + keyHash: args => args[0] +}); + +export { + normalizeCollections, + normalizeTuple, + type CollectionInternalOptions, + type CollectionExternalOptions, + type NormalizedCollectionEntry, + type CollectionCreator, + type CreatorEntry, + type CollectionModule +}; diff --git a/src/server.ts b/src/server.ts index c611821c..50e95088 100644 --- a/src/server.ts +++ b/src/server.ts @@ -545,6 +545,7 @@ runServer.memo = memo( export { runServer, + registerServerCollections, registerServerResources, registerServerTools, type ServerInstance, diff --git a/tests/e2e/__snapshots__/stdioTransport.test.ts.snap b/tests/e2e/__snapshots__/stdioTransport.test.ts.snap index 50256a48..651b7dd7 100644 --- a/tests/e2e/__snapshots__/stdioTransport.test.ts.snap +++ b/tests/e2e/__snapshots__/stdioTransport.test.ts.snap @@ -149,6 +149,8 @@ exports[`Logging should allow setting logging options, stderr 1`] = ` "[INFO]: Server logging enabled. ", "[INFO]: Server stats enabled. +", + "[INFO]: No external collections loaded. ", "[INFO]: No external resources loaded. ", From 52082ee026c5c12af0aa928f710974accedd44ec Mon Sep 17 00:00:00 2001 From: CD Cabrera Date: Thu, 6 Aug 2026 01:39:16 -0400 Subject: [PATCH 2/2] feat(collections): pf-4401 patternfly api --- .../options.defaults.test.ts.snap | 19 + .../__snapshots__/server.test.ts.snap | 27 ++ .../collection.patternFlyApi.test.ts | 163 +++++++++ src/collection.patternFlyApi.ts | 330 ++++++++++++++++++ src/options.defaults.ts | 34 +- src/options.registry.ts | 4 +- src/patternFly.getResources.ts | 5 + src/server.helpers.ts | 25 ++ 8 files changed, 605 insertions(+), 2 deletions(-) create mode 100644 src/__tests__/collection.patternFlyApi.test.ts create mode 100644 src/collection.patternFlyApi.ts diff --git a/src/__tests__/__snapshots__/options.defaults.test.ts.snap b/src/__tests__/__snapshots__/options.defaults.test.ts.snap index 200927e3..38b2d147 100644 --- a/src/__tests__/__snapshots__/options.defaults.test.ts.snap +++ b/src/__tests__/__snapshots__/options.defaults.test.ts.snap @@ -52,6 +52,16 @@ exports[`options defaults should return specific properties: defaults 1`] = ` "nodeVersion": 22, "nodeVersionPreferred": 22, "patternflyOptions": { + "api": { + "base": "https://main.patternfly-org.pages.dev/api", + "componentPaths": [ + "props", + "css", + ], + "crawlTimeoutMs": 180000, + "enabled": false, + "versions": "https://main.patternfly-org.pages.dev/api/versions", + }, "availableResourceVersions": [ "6.0.0", ], @@ -97,6 +107,12 @@ exports[`options defaults should return specific properties: defaults 1`] = ` "cacheLimit": 100, "expire": 180000, }, + "high": { + "cacheLimit": 50, + }, + "medium": { + "cacheLimit": 25, + }, "readFile": { "cacheErrors": false, "cacheLimit": 50, @@ -140,7 +156,10 @@ exports[`options defaults should return specific properties: defaults 1`] = ` ], "urls": [ "https://patternfly.org", + "https://www.patternfly.org", "https://github.com/patternfly", + "https://www.github.com/patternfly", + "https://main.patternfly-org.pages.dev", "https://raw.githubusercontent.com/patternfly", ], }, diff --git a/src/__tests__/__snapshots__/server.test.ts.snap b/src/__tests__/__snapshots__/server.test.ts.snap index b14a77fe..59aff925 100644 --- a/src/__tests__/__snapshots__/server.test.ts.snap +++ b/src/__tests__/__snapshots__/server.test.ts.snap @@ -178,6 +178,9 @@ exports[`runServer should attempt to run server, create transport, connect, and [ "test-server-4 server running on stdio transport", ], + [ + "Failed to start Collections Host; skipping hosted collections and continuing with built-in/inline collections. Error: Child process exited before response (code=undefined, signal=none)", + ], ], "hasDebugLogs": true, "mcpServer": [ @@ -254,6 +257,9 @@ exports[`runServer should attempt to run server, disable SIGINT handler: diagnos [ "test-server-7 server running on stdio transport", ], + [ + "Failed to start Collections Host; skipping hosted collections and continuing with built-in/inline collections. Error: Child process exited before response (code=undefined, signal=none)", + ], ], "hasDebugLogs": true, "mcpServer": [ @@ -325,6 +331,9 @@ exports[`runServer should attempt to run server, enable SIGINT handler explicitl [ "test-server-8 server running on stdio transport", ], + [ + "Failed to start Collections Host; skipping hosted collections and continuing with built-in/inline collections. Error: Child process exited before response (code=undefined, signal=none)", + ], ], "hasDebugLogs": true, "mcpServer": [ @@ -413,6 +422,9 @@ exports[`runServer should attempt to run server, log warnings for experimental o [ "Enabled experimental option: loremIpsum", ], + [ + "Failed to start Collections Host; skipping hosted collections and continuing with built-in/inline collections. Error: Child process exited before response (code=undefined, signal=none)", + ], ], "hasDebugLogs": true, "mcpServer": [ @@ -495,6 +507,9 @@ exports[`runServer should attempt to run server, register a tool: diagnostics 1` [ "test-server-5 server running on stdio transport", ], + [ + "Failed to start Collections Host; skipping hosted collections and continuing with built-in/inline collections. Error: Child process exited before response (code=undefined, signal=none)", + ], [ "Built-in tool at index 0 is missing the static name property, "toolName"", ], @@ -582,6 +597,9 @@ exports[`runServer should attempt to run server, register multiple tools: diagno [ "test-server-6 server running on stdio transport", ], + [ + "Failed to start Collections Host; skipping hosted collections and continuing with built-in/inline collections. Error: Child process exited before response (code=undefined, signal=none)", + ], [ "Built-in tool at index 0 is missing the static name property, "toolName"", ], @@ -667,6 +685,9 @@ exports[`runServer should attempt to run server, use custom options: diagnostics [ "test-server-3 server running on stdio transport", ], + [ + "Failed to start Collections Host; skipping hosted collections and continuing with built-in/inline collections. Error: Child process exited before response (code=undefined, signal=none)", + ], ], "hasDebugLogs": true, "mcpServer": [ @@ -749,6 +770,9 @@ exports[`runServer should attempt to run server, use default tools, http: diagno [ "test-server-2 server running on HTTP transport", ], + [ + "Failed to start Collections Host; skipping hosted collections and continuing with built-in/inline collections. Error: Child process exited before response (code=undefined, signal=none)", + ], ], "hasDebugLogs": true, "mcpServer": [ @@ -834,6 +858,9 @@ exports[`runServer should attempt to run server, use default tools, stdio: diagn [ "test-server-1 server running on stdio transport", ], + [ + "Failed to start Collections Host; skipping hosted collections and continuing with built-in/inline collections. Error: Child process exited before response (code=undefined, signal=none)", + ], ], "hasDebugLogs": true, "mcpServer": [ diff --git a/src/__tests__/collection.patternFlyApi.test.ts b/src/__tests__/collection.patternFlyApi.test.ts new file mode 100644 index 00000000..1149b481 --- /dev/null +++ b/src/__tests__/collection.patternFlyApi.test.ts @@ -0,0 +1,163 @@ +import { apiSpider, parsePayload, isEmptyPayload, crawler } from '../collection.patternFlyApi'; +import { processDocsFunction } from '../server.getResources'; + +jest.mock('../server.getResources'); + +const mockedProcessDocsFunction = processDocsFunction as jest.MockedFunction; + +describe('collections.api', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + describe('parsePayload / isEmptyPayload', () => { + it('treats {}, [], null, "" as empty (soft-404)', () => { + expect(isEmptyPayload('{}')).toBe(true); + expect(isEmptyPayload('[]')).toBe(true); + expect(isEmptyPayload('null')).toBe(true); + expect(isEmptyPayload('""')).toBe(true); + expect(isEmptyPayload('')).toBe(true); + }); + it('parses numeric payloads as non-empty', () => { + expect(parsePayload('42').isEmpty).toBe(false); + }); + }); + + describe('crawler', () => { + it('recursively crawls and returns content', async () => { + mockedProcessDocsFunction + .mockResolvedValueOnce([ + { + content: JSON.stringify(['v1']), + path: 'https://api.com/versions', + resolvedPath: 'https://api.com/versions', + isSuccess: true + } + ]) + .mockResolvedValueOnce([ + { + content: 'some content', + path: 'https://api.com/v1', + resolvedPath: 'https://api.com/v1', + isSuccess: true + } + ]); + + const res = await crawler(['https://api.com/versions']); + + expect(res).toHaveLength(1); + expect(res[0]?.content).toBe('some content'); + expect(mockedProcessDocsFunction).toHaveBeenCalledTimes(2); + }); + + it('handles component paths and terminates recursion', async () => { + mockedProcessDocsFunction.mockResolvedValueOnce([ + { + content: JSON.stringify(['item1']), + path: 'https://api.com/v1/props', + resolvedPath: 'https://api.com/v1/props', + isSuccess: true + } + ]); + + const res = await crawler(['https://api.com/v1/props']); + + expect(res).toHaveLength(1); + expect(res[0]?.path).toBe('https://api.com/v1/props'); + expect(mockedProcessDocsFunction).toHaveBeenCalledTimes(1); + }); + + it('filters out empty payloads', async () => { + mockedProcessDocsFunction.mockResolvedValueOnce([ + { + content: '{}', + path: 'https://api.com/v1/leaf', + resolvedPath: 'https://api.com/v1/leaf', + isSuccess: true + } + ]); + + const res = await crawler(['https://api.com/v1/leaf']); + + expect(res).toHaveLength(0); + }); + + it('handles recursive arrays and joins URLs correctly', async () => { + mockedProcessDocsFunction + .mockResolvedValueOnce([ + { + content: JSON.stringify(['sub-item']), + path: 'https://api.com/v1', + resolvedPath: 'https://api.com/v1', + isSuccess: true + } + ]) + .mockResolvedValue([ + { + content: 'leaf', + path: 'https://api.com/v1/sub-item', + resolvedPath: 'https://api.com/v1/sub-item', + isSuccess: true + } + ]); + + const res = await crawler(['https://api.com/v1']); + + // It should have called for sub-item AND default componentPaths (props, css) + // but my mock returns 'leaf' for everything else + expect(res.length).toBeGreaterThanOrEqual(1); + expect(mockedProcessDocsFunction).toHaveBeenCalledWith(['https://api.com/v1']); + }); + }); + + describe('apiSpider', () => { + it('returns [] when getVersions rejects', async () => { + mockedProcessDocsFunction.mockResolvedValueOnce([ + { + content: '❌ Failed to load', + path: 'https://main.patternfly-org.pages.dev/api/versions', + resolvedPath: 'https://main.patternfly-org.pages.dev/api/versions', + isSuccess: false + } + ]); + + const res = await apiSpider(); + + expect(res).toEqual([]); + }); + + it('returns ApiContent[] with metadata shape', async () => { + mockedProcessDocsFunction + .mockResolvedValueOnce([ + { + content: JSON.stringify(['v1']), + path: 'https://main.patternfly-org.pages.dev/api/versions', + resolvedPath: 'https://main.patternfly-org.pages.dev/api/versions', + isSuccess: true + } + ]) + .mockResolvedValueOnce([ + { + content: 'leaf content', + path: 'https://main.patternfly-org.pages.dev/api/v1', + resolvedPath: 'https://main.patternfly-org.pages.dev/api/v1/section/item/facet', + isSuccess: true + } + ]); + + const res = await apiSpider(); + + expect(res.length).toBeGreaterThan(0); + expect(res[0]).toMatchObject({ + url: 'https://main.patternfly-org.pages.dev/api/v1/section/item/facet', + content: 'leaf content', + semanticContext: { + version: 'v1', + section: 'section', + item: 'item', + facet: 'facet' + } + }); + }); + }); +}); diff --git a/src/collection.patternFlyApi.ts b/src/collection.patternFlyApi.ts new file mode 100644 index 00000000..e37c5390 --- /dev/null +++ b/src/collection.patternFlyApi.ts @@ -0,0 +1,330 @@ +import { log } from './logger'; +import { processDocsFunction } from './server.getResources'; +import { memo } from './server.caching'; +import { isPlainObject, joinUrl } from './server.helpers'; +import { + getOptions + // getSessionOptions, + // runWithOptions, + // runWithSession +} from './options.context'; +import { DEFAULT_OPTIONS } from './options.defaults'; +import { deferTask } from './server.task'; +import { type McpCollection, type McpCollectionRecord } from './collections'; + +/** + * Processed content for API responses. + * + * @property url - The URL of the content. + * @property content - The content itself. + * @property semanticContext - Semantic context of the content. + * @property semanticContext.version - PatternFly version of the content. + * @property semanticContext.section - Section of the content. + * @property semanticContext.item - Item of the content. + * @property semanticContext.facet - Facet of the content. + * @property semanticContext.kind - Kind of the content. + * @property semanticContext.metadata - Remaining metadata, if any, of the content. + */ +interface ApiContent { + url: string; + content: string; + semanticContext: { + version?: string | undefined; + section?: string | undefined; + item?: string | undefined; + facet?: string | undefined; + kind?: string | undefined; + metadata?: string[] | undefined; + } +} + +// type ApiProcessedDoc = NonNullable; +interface ApiCrawler { + content: string; + path: string; + resolvedPath: string; +} + +/** + * Parses the given payload and determines its state and structure. + * + * @param payload - Input payload to be parsed. + * @returns An object containing: + * - `isEmpty`: A boolean indicating whether the parsed payload is considered empty. + * - `payload`: The parsed version of the input payload. If the input is a string + * and can be parsed as JSON without error, the parsed result is returned. + * Otherwise, the trimmed string or original value is provided. + */ +const parsePayload = (payload: unknown) => { + const updatedPayload = typeof payload === 'string' ? payload.trim() : ''; + let isEmpty; + let parsedPayload; + + try { + parsedPayload = JSON.parse(updatedPayload); + + if (typeof parsedPayload === 'number') { + isEmpty = false; + } else { + isEmpty = (Array.isArray(parsedPayload) && parsedPayload.length === 0) || + (isPlainObject(parsedPayload) && Object.keys(parsedPayload).length === 0) || + parsedPayload === null; + } + } catch { + parsedPayload = updatedPayload; + isEmpty = updatedPayload.length === 0; + } + + return { isEmpty, payload: parsedPayload }; +}; + +/** + * Memoized version of parsePayload. + */ +parsePayload.memo = memo(parsePayload, DEFAULT_OPTIONS.resourceMemoOptions.default); + +/** + * Determines if the payload is empty. + * + * @param payload - Data to be evaluated for emptiness. + * @returns Returns `true` if the payload is empty, otherwise `false`. + */ +const isEmptyPayload = (payload: unknown) => { + if (typeof payload === 'string') { + const trimmedPayload = payload.trim(); + + return trimmedPayload === '' || trimmedPayload === '{}' || trimmedPayload === '[]' || trimmedPayload === 'null' || trimmedPayload === '""'; + } + + return payload === null || payload === undefined || parsePayload.memo(payload).isEmpty; +}; + +/** + * Memoized version of isEmptyPayload. + */ +isEmptyPayload.memo = memo(isEmptyPayload, DEFAULT_OPTIONS.resourceMemoOptions.default); + +/** + * Recursively crawls a list of URLs. + * + * Resolves paths and fetches content; built specifically around the PatternFly API response structure. + * + * @param urls - The list of URLs to crawl. + * @param [options] - An optional configuration object. + * @returns {Promise} A promise that resolves to an array of processed documents, + * each containing information about the crawling result, status, and content. + */ +const crawler = async (urls: string[], options = getOptions()): Promise => { + const componentPaths = options.patternflyOptions.api.componentPaths; + const settled = await processDocsFunction(urls); + const content: ApiCrawler[] = []; + + for (const res of settled) { + const { isEmpty, payload } = parsePayload.memo(res.content); + + if (res.isSuccess) { + if (Array.isArray(payload)) { + if (componentPaths.some(componentPath => res?.path?.includes(componentPath))) { + if (!isEmpty) { + content.push({ ...res }); + } + continue; + } + + const updatedPayload = [...payload, ...componentPaths].map(path => joinUrl(res.path, path)); + const crawledContent = await crawler(updatedPayload); + + content.push(...crawledContent); + continue; + } + + if (!isEmpty) { + content.push({ ...res }); + } + } + } + + return content; +}; + +/** + * Get and process available API versions. + * + * @param [options=getOptions()] - Configuration options. + * @returns A promise that resolves to an array of processed version URLs. + * + * @throws + */ +const getVersions = async (options = getOptions()) => { + const versionUrl = options.patternflyOptions.api.versions; + const processedVersions = await processDocsFunction([versionUrl]); + const versions: string[] = []; + + if (processedVersions[0]) { + const response = processedVersions[0]; + + if (response.isSuccess) { + const { payload } = parsePayload.memo(response.content); + + if (Array.isArray(payload)) { + versions.push(...payload.map(version => joinUrl(options.patternflyOptions.api.base, version))); + } + } + } + + if (versions.length === 0) { + throw new Error(`No API versions available ${versionUrl}.`); + } + + return versions; +}; + +/** + * Process content metadata from response paths. + * + * @param apiResponses - The list of pre-metadata content. + * @param [options=getOptions()] - Configuration options. + * @returns The list of processed API content with metadata. + */ +const contentMetadata = (apiResponses: ApiCrawler[], options = getOptions()): ApiContent[] => { + const base = options.patternflyOptions.api.base; + const componentPaths = options.patternflyOptions.api.componentPaths; + + return apiResponses.map(({ content, resolvedPath }) => { + const [version, section, item, facet, ...remaining] = resolvedPath.replace(base, '').split('/').filter(Boolean) || []; + const kind = facet && (componentPaths.includes(facet) || remaining.includes(facet)) ? facet : 'doc'; + + return { + url: resolvedPath, + content, + semanticContext: { + version, + section, + item, + facet, + kind, + metadata: (remaining.length && remaining) || undefined + } + }; + }); +}; + +/** + * Memoized version of contentMetadata. + */ +contentMetadata.memo = memo(contentMetadata); + +/** + * Initiate API crawl. + * + * @returns A promise resolving to an array of processed API content entries. + */ +const apiSpider = async (): Promise => { + log.info(`API spider crawl started`); + let seedVersions: string[] = []; + let content: ApiCrawler[] = []; + + try { + seedVersions = await getVersions(); + } catch (err) { + log.warn(`API spider: getVersions failed`, err); + + return []; + } + + if (seedVersions.length) { + try { + content = await crawler(seedVersions); + } catch (err) { + log.warn(`API spider: crawler failed`, err); + + return []; + } + } + + // Review the memo here. It may be better served to tie into crawler, + // like `crawler.memo` as part of the countdown to refresh + const updatedContent = contentMetadata.memo(content); + + log.info( + `API spider crawl completed. ${updatedContent.length} content ${ + (updatedContent.length === 1 && 'entry') || 'entries' + } retrieved.` + ); + + return updatedContent; +}; + +/** + * Deferred task for PatternFly API spider. + */ +apiSpider.deferTask = deferTask(apiSpider, { + cancelMs: DEFAULT_OPTIONS.patternflyOptions.api.crawlTimeoutMs +}); + +/** + * Create a PatternFly API collection. + */ +const patternFlyApiCollection = (): McpCollection => { + const callback = async () => { + const taskHandle = apiSpider.deferTask(); + const entries = await taskHandle.start(); + const recordsMap: Map = new Map(); + + entries?.forEach((entry, index) => { + const semanticContext = entry.semanticContext || {}; + const name = (semanticContext.item || 'api-entry').toLowerCase(); + const version = (semanticContext.version || 'unknown').toLowerCase(); + const displayName = semanticContext.item || name; + + const id = `api::${version}::${semanticContext.section || ''}::${name}::${semanticContext.kind || ''}::${index}`; + + if (recordsMap.has(id)) { + return; + } + + const adaptedEntry = { + displayName, + description: entry.content || `PatternFly API documentation for ${displayName}`, + pathSlug: name, + category: semanticContext.kind, + section: semanticContext.section || 'components', + source: 'api' as const, + version, + id, + path: entry.url + }; + + const record = { + id, + sourceId: entry.url, + sourceType: 'api' as const, + data: { + [name]: adaptedEntry + } + }; + + recordsMap.set(record.id, record); + }); + + return { records: [...recordsMap.values()] }; + }; + + return [ + 'patternfly-api', + callback, + { + runInChildProcess: true + } + ]; +}; + +export { + patternFlyApiCollection, + apiSpider, + crawler, + isEmptyPayload, + parsePayload, + type ApiContent, + type ApiCrawler +}; diff --git a/src/options.defaults.ts b/src/options.defaults.ts index 2f38c09c..49c15d2e 100644 --- a/src/options.defaults.ts +++ b/src/options.defaults.ts @@ -182,6 +182,11 @@ interface ModeOptions { /** * PatternFly-specific options. * + * @property api PatternFly API. + * @property api.base URL starting base for crawling the PatternFly API. + * @property api.versions URL Get the available PatternFly API versions. Versions are required to crawl. + * @property api.componentPaths List of additional PatternFly API component paths to try. + * @property api.crawlTimeoutMs Timeout in milliseconds for crawling the PatternFly API. * @property availableResourceVersions List of available PatternFly resource versions to the MCP server. * @property availableSearchVersions List of available PatternFly search versions to the MCP server. * @property availableSchemasVersions List of available PatternFly schema versions to the MCP server. @@ -195,6 +200,14 @@ interface ModeOptions { * - 'lowest': Use the lowest major version found. */ interface PatternFlyOptions { + api: { + base: string; + versions: string; + componentPaths: string[]; + crawlTimeoutMs: number; + enabled: boolean; + // concurrency: number; + }, availableResourceVersions: ('6.0.0')[]; availableSearchVersions: ('current' | 'latest' | 'v6')[]; availableSchemasVersions: ('v6')[]; @@ -414,6 +427,12 @@ const RESOURCE_MEMO_OPTIONS = { default: { cacheLimit: 3 }, + medium: { + cacheLimit: 25 + }, + high: { + cacheLimit: 50 + }, fetchUrl: { cacheLimit: 100, expire: 3 * 60 * 1000, // 3 minute sliding cache @@ -466,8 +485,10 @@ const STATS_OPTIONS: StatsOptions = { const WHITELIST_OPTIONS: WhitelistOptions = { urls: [ 'https://patternfly.org', - // 'https://www.patternfly.org', + 'https://www.patternfly.org', 'https://github.com/patternfly', + 'https://www.github.com/patternfly', + 'https://main.patternfly-org.pages.dev', 'https://raw.githubusercontent.com/patternfly' ], protocols: ['http', 'https'] @@ -492,6 +513,17 @@ const CHANNEL_BASENAME = 'pf-mcp'; * Default PatternFly-specific options. */ const PATTERNFLY_OPTIONS: PatternFlyOptions = { + api: { + base: 'https://main.patternfly-org.pages.dev/api', + versions: 'https://main.patternfly-org.pages.dev/api/versions', + componentPaths: [ + 'props', + 'css' + ], + crawlTimeoutMs: 180_000, + enabled: false + // concurrency: 4 + }, availableResourceVersions: ['6.0.0'], availableSearchVersions: ['current', 'latest', 'v6'], availableSchemasVersions: ['v6'], diff --git a/src/options.registry.ts b/src/options.registry.ts index cb87218e..de5b7d1b 100644 --- a/src/options.registry.ts +++ b/src/options.registry.ts @@ -9,6 +9,7 @@ import { patternFlyDocsIndexResource } from './resource.patternFlyDocsIndex'; import { patternFlyDocsTemplateResource } from './resource.patternFlyDocsTemplate'; import { patternFlySchemasIndexResource } from './resource.patternFlySchemasIndex'; import { patternFlySchemasTemplateResource } from './resource.patternFlySchemasTemplate'; +import { patternFlyApiCollection } from './collection.patternFlyApi'; import { patternFlyDocsCollection } from './collection.patternFlyDocs'; import { patternFlySchemasCollection } from './collection.patternFlySchemas'; @@ -44,7 +45,8 @@ const builtinResources: McpResourceCreator[] = [ */ const builtinCollections: McpCollectionCreator[] = [ patternFlyDocsCollection, - patternFlySchemasCollection + patternFlySchemasCollection, + patternFlyApiCollection ]; export { builtinCollections, builtinResources, builtinTools }; diff --git a/src/patternFly.getResources.ts b/src/patternFly.getResources.ts index 9963f291..f2c3e862 100644 --- a/src/patternFly.getResources.ts +++ b/src/patternFly.getResources.ts @@ -213,6 +213,9 @@ const setCategoryDisplayLabel = (entry?: PatternFlyMcpDocsCatalogDoc) => { } switch (categoryLabel) { + // case 'api': + // categoryLabel = 'API Reference'; + // break; case 'grammar': categoryLabel = 'Grammar'; break; @@ -474,10 +477,12 @@ const getPatternFlyMcpResources = async (contextPathOverride?: string): Promise< const { componentNamesIndex, byVersion: componentNamesByVersion, byDocs: componentNamesByDocs } = componentNames; const originalDocs = patternFlyRecordsRegistry.get('patternfly-docs'); + // const apiCollection = patternFlyRecordsRegistry.get('patternfly-api'); const catalog = [ ...originalDocs?.records?.flatMap(({ data }) => Object.entries(data as Record)) || [], ...Array.from(componentNamesByDocs) + // ...apiCollection?.records?.flatMap(({ data }) => Object.entries(data as Record)) || [] ]; const resources = new Map(); diff --git a/src/server.helpers.ts b/src/server.helpers.ts index fd3cd775..2a8a5450 100644 --- a/src/server.helpers.ts +++ b/src/server.helpers.ts @@ -556,6 +556,30 @@ const parseUrl = (url: string, { prefix, normalizeSearchParamKeys = true, isStri return undefined; }; +/** + * Joins multiple URL segments into a single URL string, ensuring no double slashes. + * If `base` is not a valid URL, it's returned as-is. + * + * @param base - The base URL string + * @param parts - Additional path segments to join + * @returns The joined URL string + */ +const joinUrl = (base: string, ...parts: string[]): string => { + if (!isUrl(base)) { + return base; + } + + const url = new URL(base); + + parts.join('/').split('/').filter(Boolean).forEach(part => { + const updatedPathname = url.pathname.endsWith('/') ? url.pathname : `${url.pathname}/`; + + url.pathname = `${updatedPathname}${part}`; + }); + + return url.toString(); +}; + /** * Basic split for URIs to find base and search. * @@ -767,6 +791,7 @@ export { isUrl, isUrlObject, isWhitelistedUrl, + joinUrl, listAllCombinations, listIncrementalCombinations, mergeObjects,