diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index d3556558a..97dd6fb62 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -171,6 +171,19 @@ ObjectStack is organized as a **monorepo** with distinct package layers: **Dependencies**: `@objectstack/core`, `@objectstack/spec`, `@objectstack/types` +#### `@objectstack/metadata` +**Location**: `packages/metadata/` +**Role**: Metadata Loading & Persistence + +- Metadata serialization/deserialization (JSON, YAML, TypeScript) +- File system operations (load, save, watch) +- Validation using Zod schemas +- ETag-based caching +- Import/export tools +- SchemaRegistry persistence bridge + +**Dependencies**: `@objectstack/spec`, `@objectstack/core`, `@objectstack/types`, `glob`, `js-yaml`, `chokidar` + #### `@objectstack/runtime` **Location**: `packages/runtime/` **Role**: Runtime Utilities & Plugins @@ -409,6 +422,12 @@ interface PluginCapabilityManifest { │ │ ↑ │ │ └── @objectstack/plugin-msw │ │ + │ ├── @objectstack/metadata (Metadata I/O) + │ │ ↑ + │ │ ├── @objectstack/runtime (Uses for manifest loading) + │ │ ├── @objectstack/cli (Uses for code generation) + │ │ └── @objectstack/objectql (Uses for registry persistence) + │ │ │ ├── @objectstack/runtime (Plugins & HTTP) │ │ ↑ │ │ └── (Used by server plugins) diff --git a/PACKAGE-DEPENDENCIES.md b/PACKAGE-DEPENDENCIES.md index 8f98bbe3c..f91349660 100644 --- a/PACKAGE-DEPENDENCIES.md +++ b/PACKAGE-DEPENDENCIES.md @@ -45,6 +45,16 @@ This is the foundation layer. All other packages depend on `@objectstack/spec`. │ └── @objectstack/types └── Role: ObjectQL query engine +@objectstack/metadata +├── Dependencies: +│ ├── @objectstack/core +│ ├── @objectstack/spec +│ ├── @objectstack/types +│ ├── glob +│ ├── js-yaml +│ └── chokidar (optional) +└── Role: Metadata loading, saving, and persistence + @objectstack/runtime ├── Dependencies: │ ├── @objectstack/core @@ -131,6 +141,12 @@ This is the foundation layer. All other packages depend on `@objectstack/spec`. │ │ ↑ │ │ └── @objectstack/plugin-msw (Layer 5) │ │ + │ ├── @objectstack/metadata (Layer 3) + │ │ ↑ + │ │ ├── @objectstack/runtime (Layer 3) + │ │ ├── @objectstack/cli (Layer 6) + │ │ └── (Future integrations) + │ │ │ ├── @objectstack/runtime (Layer 3) │ │ ↑ │ │ └── (Used by server plugins) @@ -156,6 +172,7 @@ This is the foundation layer. All other packages depend on `@objectstack/spec`. | `@objectstack/types` | `@objectstack/spec` | | `@objectstack/core` | `@objectstack/spec`, `pino`, `zod` | | `@objectstack/objectql` | `@objectstack/core`, `@objectstack/spec`, `@objectstack/types` | +| `@objectstack/metadata` | `@objectstack/core`, `@objectstack/spec`, `@objectstack/types`, `glob`, `js-yaml`, `chokidar` | | `@objectstack/runtime` | `@objectstack/core`, `@objectstack/spec`, `@objectstack/types` | | `@objectstack/client` | `@objectstack/core`, `@objectstack/spec` | | `@objectstack/client-react` | `@objectstack/client`, `@objectstack/core`, `@objectstack/spec`, `react` (peer) | @@ -181,6 +198,7 @@ pnpm --filter @objectstack/core build # Level 3 pnpm --filter @objectstack/objectql build +pnpm --filter @objectstack/metadata build pnpm --filter @objectstack/runtime build # Level 4 @@ -253,7 +271,8 @@ Use peer dependencies for: |---------|---------------|-------------------| | `@objectstack/spec` | Define protocols | Runtime behavior | | `@objectstack/core` | Manage plugin lifecycle | Query execution, HTTP handling | -| `@objectstack/objectql` | Execute queries | HTTP routing, UI rendering | +| `@objectstack/objectql` | Execute queries | HTTP routing, UI rendering, Metadata persistence | +| `@objectstack/metadata` | Load/save metadata | Query execution, Plugin lifecycle | | `@objectstack/runtime` | Provide plugin utilities | Execute queries directly | ## External Dependencies diff --git a/content/docs/references/meta.json b/content/docs/references/meta.json index 154d7fd6a..577f212af 100644 --- a/content/docs/references/meta.json +++ b/content/docs/references/meta.json @@ -1,7 +1,6 @@ { "title": "Protocol Reference", "pages": [ - "packages", "data", "ui", "automation", @@ -9,10 +8,6 @@ "permission", "ai", "api", - "driver", - "hub", - "integration", - "auth", - "shared" + "driver" ] } \ No newline at end of file diff --git a/content/docs/references/system/index.mdx b/content/docs/references/system/index.mdx index ce719f6ac..a82ccf6b1 100644 --- a/content/docs/references/system/index.mdx +++ b/content/docs/references/system/index.mdx @@ -24,6 +24,7 @@ This section contains all protocol schemas for the system layer of ObjectStack. + diff --git a/content/docs/references/system/meta.json b/content/docs/references/system/meta.json index 3c2db1762..e70c88158 100644 --- a/content/docs/references/system/meta.json +++ b/content/docs/references/system/meta.json @@ -17,6 +17,7 @@ "manifest", "masking", "message-queue", + "metadata-loader", "metrics", "notification", "object-storage", diff --git a/examples/msw-react-crud/src/components/TaskList.tsx b/examples/msw-react-crud/src/components/TaskList.tsx index eef7b9d5a..6a76b8a0e 100644 --- a/examples/msw-react-crud/src/components/TaskList.tsx +++ b/examples/msw-react-crud/src/components/TaskList.tsx @@ -36,8 +36,8 @@ export function TaskList({ client, onEdit, refreshTrigger }: TaskListProps) { sort: ['priority', '-created_at'] }); - // Handle { value: [] } (OData), { records: [] } (Protocol) and [] (Raw) formats - const rawValues = Array.isArray(result) ? result : (result.value || result.records || []); + // Handle { value: [] } (PaginatedResult) and [] (Raw) formats + const rawValues = Array.isArray(result) ? result : (result.value || []); const fetchedTasks = [...rawValues] as Task[]; // Client-side sort fallback (since InMemoryDriver has limited sort support) diff --git a/packages/metadata/README.md b/packages/metadata/README.md new file mode 100644 index 000000000..da4366b7d --- /dev/null +++ b/packages/metadata/README.md @@ -0,0 +1,201 @@ +# @objectstack/metadata + +Metadata loading, saving, and persistence for ObjectStack. + +## Overview + +The `@objectstack/metadata` package provides a unified interface for managing metadata across the ObjectStack ecosystem. It handles: + +- **Metadata Serialization/Deserialization** - Support for JSON, YAML, TypeScript, and JavaScript formats +- **File System Operations** - Load, save, and watch metadata files +- **Validation** - Integration with Zod schemas from `@objectstack/spec` +- **Caching** - ETag-based caching for performance optimization +- **File Watching** - Development mode with automatic reload on file changes + +## Installation + +```bash +pnpm add @objectstack/metadata +``` + +## Quick Start + +```typescript +import { MetadataManager } from '@objectstack/metadata'; +import type { ServiceObject } from '@objectstack/spec/data'; + +// Create manager +const manager = new MetadataManager({ + rootDir: './metadata', + formats: ['typescript', 'json', 'yaml'], + cache: { enabled: true, ttl: 3600 }, + watch: process.env.NODE_ENV === 'development', +}); + +// Load metadata +const customer = await manager.load('object', 'customer'); + +// Save metadata +await manager.save('object', 'project', projectObject, { + format: 'typescript', + prettify: true, +}); + +// Load multiple items +const objects = await manager.loadMany('object', { + patterns: ['**/*.object.ts', '**/*.object.json'], +}); + +// Watch for changes +manager.watch('object', (event) => { + console.log(`Object ${event.type}:`, event.name); +}); +``` + +## API + +### MetadataManager + +Main class for metadata operations. + +#### Constructor + +```typescript +new MetadataManager(config: MetadataManagerConfig) +``` + +**Config options:** +- `rootDir` - Root directory for metadata files +- `formats` - Enabled serialization formats (default: `['typescript', 'json', 'yaml']`) +- `cache` - Cache configuration with `enabled` and `ttl` options +- `watch` - Enable file watching (default: `false`) +- `watchOptions` - File watcher options (`ignored`, `persistent`, `ignoreInitial`) + +#### Methods + +**load(type: string, name: string, options?: MetadataLoadOptions): Promise** + +Load a single metadata item. + +```typescript +const customer = await manager.load('object', 'customer'); +``` + +**loadMany(type: string, options?: MetadataLoadOptions): Promise** + +Load multiple metadata items matching patterns. + +```typescript +const objects = await manager.loadMany('object', { + patterns: ['**/*.object.ts'], + limit: 100, +}); +``` + +**save(type: string, name: string, data: T, options?: MetadataSaveOptions): Promise** + +Save metadata to disk. + +```typescript +await manager.save('object', 'customer', customerObject, { + format: 'typescript', + prettify: true, + backup: true, +}); +``` + +**exists(type: string, name: string): Promise** + +Check if metadata item exists. + +```typescript +const exists = await manager.exists('object', 'customer'); +``` + +**list(type: string): Promise** + +List all items of a type. + +```typescript +const objectNames = await manager.list('object'); +``` + +**watch(type: string, callback: WatchCallback): void** + +Watch for metadata changes. + +```typescript +manager.watch('object', (event) => { + if (event.type === 'added') { + console.log('New object:', event.name); + } +}); +``` + +**stopWatching(): Promise** + +Stop all file watching. + +## Serialization Formats + +### JSON + +```json +{ + "name": "customer", + "label": "Customer", + "fields": { + "name": { "type": "text", "label": "Name" } + } +} +``` + +### YAML + +```yaml +name: customer +label: Customer +fields: + name: + type: text + label: Name +``` + +### TypeScript + +```typescript +import type { ServiceObject } from '@objectstack/spec/data'; + +export const metadata: ServiceObject = { + name: 'customer', + label: 'Customer', + fields: { + name: { type: 'text', label: 'Name' }, + }, +}; + +export default metadata; +``` + +## Architecture + +The metadata package is designed as a Layer 3 package in the ObjectStack architecture: + +``` +@objectstack/metadata (Layer 3) +├── Dependencies: +│ ├── @objectstack/spec (validation) +│ ├── @objectstack/core (logging, DI) +│ ├── @objectstack/types (shared types) +│ ├── glob (file pattern matching) +│ ├── js-yaml (YAML support) +│ └── chokidar (file watching) +└── Used By: + ├── @objectstack/cli (code generation) + ├── @objectstack/runtime (manifest loading) + └── @objectstack/objectql (registry persistence) +``` + +## License + +MIT diff --git a/packages/metadata/package.json b/packages/metadata/package.json new file mode 100644 index 000000000..16c428c7a --- /dev/null +++ b/packages/metadata/package.json @@ -0,0 +1,37 @@ +{ + "name": "@objectstack/metadata", + "version": "0.6.1", + "description": "Metadata loading, saving, and persistence for ObjectStack", + "main": "src/index.ts", + "types": "src/index.ts", + "scripts": { + "build": "tsc", + "dev": "tsc --watch", + "clean": "rm -rf dist", + "test": "vitest run", + "test:watch": "vitest", + "test:coverage": "vitest run --coverage" + }, + "keywords": [ + "objectstack", + "metadata", + "persistence", + "serialization", + "loader" + ], + "dependencies": { + "@objectstack/core": "workspace:*", + "@objectstack/spec": "workspace:*", + "@objectstack/types": "workspace:*", + "glob": "^10.3.10", + "js-yaml": "^4.1.0", + "chokidar": "^3.5.3", + "zod": "^3.22.4" + }, + "devDependencies": { + "@types/js-yaml": "^4.0.9", + "@types/node": "^20.0.0", + "typescript": "^5.0.0", + "vitest": "^1.0.0" + } +} diff --git a/packages/metadata/src/index.ts b/packages/metadata/src/index.ts new file mode 100644 index 000000000..21b0ecc82 --- /dev/null +++ b/packages/metadata/src/index.ts @@ -0,0 +1,34 @@ +/** + * @objectstack/metadata + * + * Metadata loading, saving, and persistence for ObjectStack + */ + +// Main Manager +export { MetadataManager, type WatchCallback } from './metadata-manager.js'; + +// Loaders +export { type MetadataLoader } from './loaders/loader-interface.js'; +export { FilesystemLoader } from './loaders/filesystem-loader.js'; + +// Serializers +export { type MetadataSerializer, type SerializeOptions } from './serializers/serializer-interface.js'; +export { JSONSerializer } from './serializers/json-serializer.js'; +export { YAMLSerializer } from './serializers/yaml-serializer.js'; +export { TypeScriptSerializer } from './serializers/typescript-serializer.js'; + +// Re-export types from spec +export type { + MetadataFormat, + MetadataStats, + MetadataLoadOptions, + MetadataSaveOptions, + MetadataExportOptions, + MetadataImportOptions, + MetadataLoadResult, + MetadataSaveResult, + MetadataWatchEvent, + MetadataCollectionInfo, + MetadataLoaderContract, + MetadataManagerConfig, +} from '@objectstack/spec/system'; diff --git a/packages/metadata/src/loaders/filesystem-loader.ts b/packages/metadata/src/loaders/filesystem-loader.ts new file mode 100644 index 000000000..e668957f6 --- /dev/null +++ b/packages/metadata/src/loaders/filesystem-loader.ts @@ -0,0 +1,314 @@ +/** + * Filesystem Metadata Loader + * + * Loads metadata from the filesystem using glob patterns + */ + +import * as fs from 'node:fs/promises'; +import * as path from 'node:path'; +import { glob } from 'glob'; +import { createHash } from 'node:crypto'; +import type { + MetadataLoadOptions, + MetadataLoadResult, + MetadataStats, + MetadataLoaderContract, + MetadataFormat, +} from '@objectstack/spec/system'; +import type { Logger } from '@objectstack/core'; +import type { MetadataLoader } from './loader-interface.js'; +import type { MetadataSerializer } from '../serializers/serializer-interface.js'; + +export class FilesystemLoader implements MetadataLoader { + readonly contract: MetadataLoaderContract = { + name: 'filesystem', + supportedFormats: ['json', 'yaml', 'typescript', 'javascript'], + supportsWatch: true, + supportsWrite: true, + supportsCache: true, + }; + + private cache = new Map(); + + constructor( + private rootDir: string, + private serializers: Map, + private logger?: Logger + ) {} + + async load( + type: string, + name: string, + options?: MetadataLoadOptions + ): Promise { + const startTime = Date.now(); + const { validate: _validate = true, useCache = true, ifNoneMatch } = options || {}; + + try { + // Find the file + const filePath = await this.findFile(type, name); + + if (!filePath) { + return { + data: null, + fromCache: false, + notModified: false, + loadTime: Date.now() - startTime, + }; + } + + // Get stats + const stats = await this.stat(type, name); + + if (!stats) { + return { + data: null, + fromCache: false, + notModified: false, + loadTime: Date.now() - startTime, + }; + } + + // Check cache + if (useCache && ifNoneMatch && stats.etag === ifNoneMatch) { + return { + data: null, + fromCache: true, + notModified: true, + etag: stats.etag, + stats, + loadTime: Date.now() - startTime, + }; + } + + // Check memory cache + const cacheKey = `${type}:${name}`; + if (useCache && this.cache.has(cacheKey)) { + const cached = this.cache.get(cacheKey)!; + if (cached.etag === stats.etag) { + return { + data: cached.data, + fromCache: true, + notModified: false, + etag: stats.etag, + stats, + loadTime: Date.now() - startTime, + }; + } + } + + // Load and deserialize + const content = await fs.readFile(filePath, 'utf-8'); + const serializer = this.getSerializer(stats.format); + + if (!serializer) { + throw new Error(`No serializer found for format: ${stats.format}`); + } + + const data = serializer.deserialize(content); + + // Update cache + if (useCache) { + this.cache.set(cacheKey, { + data, + etag: stats.etag, + timestamp: Date.now(), + }); + } + + return { + data, + fromCache: false, + notModified: false, + etag: stats.etag, + stats, + loadTime: Date.now() - startTime, + }; + } catch (error) { + this.logger?.error('Failed to load metadata', undefined, { + type, + name, + error: error instanceof Error ? error.message : String(error), + }); + throw error; + } + } + + async loadMany( + type: string, + options?: MetadataLoadOptions + ): Promise { + const { patterns = ['**/*'], recursive: _recursive = true, limit } = options || {}; + + const typeDir = path.join(this.rootDir, type); + const items: T[] = []; + + try { + // Build glob patterns + const globPatterns = patterns.map(pattern => + path.join(typeDir, pattern) + ); + + for (const pattern of globPatterns) { + const files = await glob(pattern, { + ignore: ['**/node_modules/**', '**/*.test.*', '**/*.spec.*'], + nodir: true, + }); + + for (const file of files) { + if (limit && items.length >= limit) { + break; + } + + try { + const content = await fs.readFile(file, 'utf-8'); + const format = this.detectFormat(file); + const serializer = this.getSerializer(format); + + if (serializer) { + const data = serializer.deserialize(content); + items.push(data); + } + } catch (error) { + this.logger?.warn('Failed to load file', { + file, + error: error instanceof Error ? error.message : String(error), + }); + } + } + + if (limit && items.length >= limit) { + break; + } + } + + return items; + } catch (error) { + this.logger?.error('Failed to load many', undefined, { + type, + patterns, + error: error instanceof Error ? error.message : String(error), + }); + throw error; + } + } + + async exists(type: string, name: string): Promise { + const filePath = await this.findFile(type, name); + return filePath !== null; + } + + async stat(type: string, name: string): Promise { + const filePath = await this.findFile(type, name); + + if (!filePath) { + return null; + } + + try { + const stats = await fs.stat(filePath); + const content = await fs.readFile(filePath, 'utf-8'); + const etag = this.generateETag(content); + const format = this.detectFormat(filePath); + + return { + size: stats.size, + modifiedAt: stats.mtime, + etag, + format, + path: filePath, + }; + } catch (error) { + this.logger?.error('Failed to stat file', undefined, { + type, + name, + filePath, + error: error instanceof Error ? error.message : String(error), + }); + return null; + } + } + + async list(type: string): Promise { + const typeDir = path.join(this.rootDir, type); + + try { + const files = await glob('**/*', { + cwd: typeDir, + ignore: ['**/node_modules/**', '**/*.test.*', '**/*.spec.*'], + nodir: true, + }); + + return files.map(file => { + const ext = path.extname(file); + const basename = path.basename(file, ext); + return basename; + }); + } catch (error) { + this.logger?.error('Failed to list', undefined, { + type, + error: error instanceof Error ? error.message : String(error), + }); + return []; + } + } + + /** + * Find file for a given type and name + */ + private async findFile(type: string, name: string): Promise { + const typeDir = path.join(this.rootDir, type); + const extensions = ['.json', '.yaml', '.yml', '.ts', '.js']; + + for (const ext of extensions) { + const filePath = path.join(typeDir, `${name}${ext}`); + + try { + await fs.access(filePath); + return filePath; + } catch { + // File doesn't exist, try next extension + } + } + + return null; + } + + /** + * Detect format from file extension + */ + private detectFormat(filePath: string): MetadataFormat { + const ext = path.extname(filePath).toLowerCase(); + + switch (ext) { + case '.json': + return 'json'; + case '.yaml': + case '.yml': + return 'yaml'; + case '.ts': + return 'typescript'; + case '.js': + return 'javascript'; + default: + return 'json'; // Default to JSON + } + } + + /** + * Get serializer for format + */ + private getSerializer(format: MetadataFormat): MetadataSerializer | undefined { + return this.serializers.get(format); + } + + /** + * Generate ETag for content + * Uses SHA-256 hash truncated to 32 characters for reasonable collision resistance + * while keeping ETag headers compact (full 64-char hash is overkill for this use case) + */ + private generateETag(content: string): string { + const hash = createHash('sha256').update(content).digest('hex').substring(0, 32); + return `"${hash}"`; + } +} diff --git a/packages/metadata/src/loaders/loader-interface.ts b/packages/metadata/src/loaders/loader-interface.ts new file mode 100644 index 000000000..8a78261ef --- /dev/null +++ b/packages/metadata/src/loaders/loader-interface.ts @@ -0,0 +1,70 @@ +/** + * Metadata Loader Interface + * + * Defines the contract for loading metadata from various sources + */ + +import type { + MetadataLoadOptions, + MetadataLoadResult, + MetadataStats, + MetadataLoaderContract, +} from '@objectstack/spec/system'; + +/** + * Abstract interface for metadata loaders + * Implementations can load from filesystem, HTTP, S3, databases, etc. + */ +export interface MetadataLoader { + /** + * Loader contract information + */ + readonly contract: MetadataLoaderContract; + + /** + * Load a single metadata item + * @param type The metadata type (e.g., 'object', 'view', 'app') + * @param name The item name/identifier + * @param options Load options + * @returns Load result with data or null if not found + */ + load( + type: string, + name: string, + options?: MetadataLoadOptions + ): Promise; + + /** + * Load multiple items matching patterns + * @param type The metadata type + * @param options Load options with patterns + * @returns Array of loaded items + */ + loadMany( + type: string, + options?: MetadataLoadOptions + ): Promise; + + /** + * Check if item exists + * @param type The metadata type + * @param name The item name + * @returns True if exists + */ + exists(type: string, name: string): Promise; + + /** + * Get item metadata (without loading full content) + * @param type The metadata type + * @param name The item name + * @returns Metadata statistics + */ + stat(type: string, name: string): Promise; + + /** + * List all items of a type + * @param type The metadata type + * @returns Array of item names + */ + list(type: string): Promise; +} diff --git a/packages/metadata/src/metadata-manager.ts b/packages/metadata/src/metadata-manager.ts new file mode 100644 index 000000000..245eb9315 --- /dev/null +++ b/packages/metadata/src/metadata-manager.ts @@ -0,0 +1,338 @@ +/** + * Metadata Manager + * + * Main orchestrator for metadata loading, saving, and persistence + */ + +import * as fs from 'node:fs/promises'; +import * as path from 'node:path'; +import { createHash } from 'node:crypto'; +import { watch as chokidarWatch, type FSWatcher } from 'chokidar'; +import type { + MetadataManagerConfig, + MetadataLoadOptions, + MetadataSaveOptions, + MetadataSaveResult, + MetadataWatchEvent, + MetadataFormat, +} from '@objectstack/spec/system'; +import { createLogger, type Logger } from '@objectstack/core'; +import { FilesystemLoader } from './loaders/filesystem-loader.js'; +import { JSONSerializer } from './serializers/json-serializer.js'; +import { YAMLSerializer } from './serializers/yaml-serializer.js'; +import { TypeScriptSerializer } from './serializers/typescript-serializer.js'; +import type { MetadataSerializer } from './serializers/serializer-interface.js'; +import type { MetadataLoader } from './loaders/loader-interface.js'; + +/** + * Watch callback function + */ +export type WatchCallback = (event: MetadataWatchEvent) => void | Promise; + +/** + * Main metadata manager class + */ +export class MetadataManager { + private loader: MetadataLoader; + private serializers: Map; + private logger: Logger; + private watcher?: FSWatcher; + private watchCallbacks = new Map>(); + + constructor(private config: MetadataManagerConfig) { + this.logger = createLogger({ level: 'info', format: 'pretty' }); + + // Initialize serializers + this.serializers = new Map(); + const formats = config.formats || ['typescript', 'json', 'yaml']; + + if (formats.includes('json')) { + this.serializers.set('json', new JSONSerializer()); + } + if (formats.includes('yaml')) { + this.serializers.set('yaml', new YAMLSerializer()); + } + if (formats.includes('typescript')) { + this.serializers.set('typescript', new TypeScriptSerializer('typescript')); + } + if (formats.includes('javascript')) { + this.serializers.set('javascript', new TypeScriptSerializer('javascript')); + } + + // Initialize loader + const rootDir = config.rootDir || process.cwd(); + this.loader = new FilesystemLoader(rootDir, this.serializers, this.logger); + + // Start watching if enabled + if (config.watch) { + this.startWatching(); + } + } + + /** + * Load a single metadata item + */ + async load( + type: string, + name: string, + options?: MetadataLoadOptions + ): Promise { + const result = await this.loader.load(type, name, options); + return result.data; + } + + /** + * Load multiple metadata items + */ + async loadMany( + type: string, + options?: MetadataLoadOptions + ): Promise { + return this.loader.loadMany(type, options); + } + + /** + * Save metadata to disk + */ + async save( + type: string, + name: string, + data: T, + options?: MetadataSaveOptions + ): Promise { + const startTime = Date.now(); + const { + format = 'typescript', + prettify = true, + indent = 2, + sortKeys = false, + backup = false, + overwrite = true, + atomic = true, + path: customPath, + } = options || {}; + + try { + // Get serializer + const serializer = this.serializers.get(format); + if (!serializer) { + throw new Error(`No serializer found for format: ${format}`); + } + + // Determine file path + const typeDir = path.join(this.config.rootDir || process.cwd(), type); + const fileName = `${name}${serializer.getExtension()}`; + const filePath = customPath || path.join(typeDir, fileName); + + // Check if file exists + if (!overwrite) { + try { + await fs.access(filePath); + throw new Error(`File already exists: ${filePath}`); + } catch (error) { + // File doesn't exist, continue + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') { + throw error; + } + } + } + + // Create directory if it doesn't exist + await fs.mkdir(path.dirname(filePath), { recursive: true }); + + // Create backup if requested + let backupPath: string | undefined; + if (backup) { + try { + await fs.access(filePath); + backupPath = `${filePath}.bak`; + await fs.copyFile(filePath, backupPath); + } catch { + // File doesn't exist, no backup needed + } + } + + // Serialize data + const content = serializer.serialize(data, { + prettify, + indent, + sortKeys, + }); + + // Write to disk (atomic or direct) + if (atomic) { + const tempPath = `${filePath}.tmp`; + await fs.writeFile(tempPath, content, 'utf-8'); + await fs.rename(tempPath, filePath); + } else { + await fs.writeFile(filePath, content, 'utf-8'); + } + + // Get stats + const stats = await fs.stat(filePath); + const etag = this.generateETag(content); + + return { + success: true, + path: filePath, + etag, + size: stats.size, + saveTime: Date.now() - startTime, + backupPath, + }; + } catch (error) { + this.logger.error('Failed to save metadata', undefined, { + type, + name, + error: error instanceof Error ? error.message : String(error), + }); + throw error; + } + } + + /** + * Check if metadata item exists + */ + async exists(type: string, name: string): Promise { + return this.loader.exists(type, name); + } + + /** + * List all items of a type + */ + async list(type: string): Promise { + return this.loader.list(type); + } + + /** + * Watch for metadata changes + */ + watch(type: string, callback: WatchCallback): void { + if (!this.watchCallbacks.has(type)) { + this.watchCallbacks.set(type, new Set()); + } + this.watchCallbacks.get(type)!.add(callback); + } + + /** + * Unwatch metadata changes + */ + unwatch(type: string, callback: WatchCallback): void { + const callbacks = this.watchCallbacks.get(type); + if (callbacks) { + callbacks.delete(callback); + if (callbacks.size === 0) { + this.watchCallbacks.delete(type); + } + } + } + + /** + * Stop all watching + */ + async stopWatching(): Promise { + if (this.watcher) { + await this.watcher.close(); + this.watcher = undefined; + this.watchCallbacks.clear(); + } + } + + /** + * Start watching for file changes + */ + private startWatching(): void { + const rootDir = this.config.rootDir || process.cwd(); + const { ignored = ['**/node_modules/**', '**/*.test.*'], persistent = true } = + this.config.watchOptions || {}; + + this.watcher = chokidarWatch(rootDir, { + ignored, + persistent, + ignoreInitial: true, + }); + + this.watcher.on('add', async (filePath) => { + await this.handleFileEvent('added', filePath); + }); + + this.watcher.on('change', async (filePath) => { + await this.handleFileEvent('changed', filePath); + }); + + this.watcher.on('unlink', async (filePath) => { + await this.handleFileEvent('deleted', filePath); + }); + + this.logger.info('File watcher started', { rootDir }); + } + + /** + * Handle file change events + */ + private async handleFileEvent( + eventType: 'added' | 'changed' | 'deleted', + filePath: string + ): Promise { + const rootDir = this.config.rootDir || process.cwd(); + const relativePath = path.relative(rootDir, filePath); + const parts = relativePath.split(path.sep); + + if (parts.length < 2) { + return; // Not a metadata file + } + + const type = parts[0]; + const fileName = parts[parts.length - 1]; + const name = path.basename(fileName, path.extname(fileName)); + + const callbacks = this.watchCallbacks.get(type); + if (!callbacks || callbacks.size === 0) { + return; + } + + let data: any = undefined; + if (eventType !== 'deleted') { + try { + data = await this.load(type, name, { useCache: false }); + } catch (error) { + this.logger.error('Failed to load changed file', undefined, { + filePath, + error: error instanceof Error ? error.message : String(error), + }); + return; + } + } + + const event: MetadataWatchEvent = { + type: eventType, + metadataType: type, + name, + path: filePath, + data, + timestamp: new Date(), + }; + + for (const callback of callbacks) { + try { + await callback(event); + } catch (error) { + this.logger.error('Watch callback error', undefined, { + type, + name, + error: error instanceof Error ? error.message : String(error), + }); + } + } + } + + /** + * Generate ETag for content + * Uses SHA-256 hash truncated to 32 characters for reasonable collision resistance + * while keeping ETag headers compact (full 64-char hash is overkill for this use case) + */ + private generateETag(content: string): string { + const hash = createHash('sha256').update(content).digest('hex').substring(0, 32); + return `"${hash}"`; + } +} diff --git a/packages/metadata/src/serializers/json-serializer.ts b/packages/metadata/src/serializers/json-serializer.ts new file mode 100644 index 000000000..be305ee06 --- /dev/null +++ b/packages/metadata/src/serializers/json-serializer.ts @@ -0,0 +1,71 @@ +/** + * JSON Metadata Serializer + * + * Handles JSON format serialization and deserialization + */ + +import type { z } from 'zod'; +import type { MetadataFormat } from '@objectstack/spec/system'; +import type { MetadataSerializer, SerializeOptions } from './serializer-interface.js'; + +export class JSONSerializer implements MetadataSerializer { + serialize(item: T, options?: SerializeOptions): string { + const { prettify = true, indent = 2, sortKeys = false } = options || {}; + + if (sortKeys) { + // Sort keys recursively + const sorted = this.sortObjectKeys(item); + return prettify + ? JSON.stringify(sorted, null, indent) + : JSON.stringify(sorted); + } + + return prettify + ? JSON.stringify(item, null, indent) + : JSON.stringify(item); + } + + deserialize(content: string, schema?: z.ZodSchema): T { + const parsed = JSON.parse(content); + + if (schema) { + return schema.parse(parsed) as T; + } + + return parsed as T; + } + + getExtension(): string { + return '.json'; + } + + canHandle(format: MetadataFormat): boolean { + return format === 'json'; + } + + getFormat(): MetadataFormat { + return 'json'; + } + + /** + * Recursively sort object keys + */ + private sortObjectKeys(obj: any): any { + if (obj === null || typeof obj !== 'object') { + return obj; + } + + if (Array.isArray(obj)) { + return obj.map(item => this.sortObjectKeys(item)); + } + + const sorted: Record = {}; + const keys = Object.keys(obj).sort(); + + for (const key of keys) { + sorted[key] = this.sortObjectKeys(obj[key]); + } + + return sorted; + } +} diff --git a/packages/metadata/src/serializers/serializer-interface.ts b/packages/metadata/src/serializers/serializer-interface.ts new file mode 100644 index 000000000..27ef295a4 --- /dev/null +++ b/packages/metadata/src/serializers/serializer-interface.ts @@ -0,0 +1,63 @@ +/** + * Metadata Serializer Interface + * + * Defines the contract for serializing/deserializing metadata + */ + +import type { z } from 'zod'; +import type { MetadataFormat } from '@objectstack/spec/system'; + +/** + * Serialization options + */ +export interface SerializeOptions { + /** Prettify output (formatted with indentation) */ + prettify?: boolean; + /** Indentation size (spaces) */ + indent?: number; + /** Sort object keys alphabetically */ + sortKeys?: boolean; + /** Include default values in output */ + includeDefaults?: boolean; +} + +/** + * Abstract interface for metadata serializers + * Implementations handle different formats (JSON, YAML, TypeScript, etc.) + */ +export interface MetadataSerializer { + /** + * Serialize object to string + * @param item The item to serialize + * @param options Serialization options + * @returns Serialized string + */ + serialize(item: T, options?: SerializeOptions): string; + + /** + * Deserialize string to object + * @param content The content to deserialize + * @param schema Optional Zod schema for validation + * @returns Deserialized object + */ + deserialize(content: string, schema?: z.ZodSchema): T; + + /** + * Get file extension for this format + * @returns File extension (e.g., '.json', '.yaml') + */ + getExtension(): string; + + /** + * Check if this serializer can handle the format + * @param format The format to check + * @returns True if can handle + */ + canHandle(format: MetadataFormat): boolean; + + /** + * Get the format this serializer handles + * @returns The metadata format + */ + getFormat(): MetadataFormat; +} diff --git a/packages/metadata/src/serializers/serializers.test.ts b/packages/metadata/src/serializers/serializers.test.ts new file mode 100644 index 000000000..9533a416a --- /dev/null +++ b/packages/metadata/src/serializers/serializers.test.ts @@ -0,0 +1,74 @@ +import { describe, it, expect } from 'vitest'; +import { JSONSerializer } from '../serializers/json-serializer'; +import { YAMLSerializer } from '../serializers/yaml-serializer'; +import { TypeScriptSerializer } from '../serializers/typescript-serializer'; + +describe('Serializers', () => { + describe('JSONSerializer', () => { + const serializer = new JSONSerializer(); + + it('should serialize to JSON', () => { + const data = { name: 'test', value: 42 }; + const result = serializer.serialize(data); + expect(result).toContain('"name"'); + expect(result).toContain('"test"'); + }); + + it('should deserialize from JSON', () => { + const json = '{"name":"test","value":42}'; + const result = serializer.deserialize(json); + expect(result).toEqual({ name: 'test', value: 42 }); + }); + + it('should prettify JSON', () => { + const data = { name: 'test' }; + const result = serializer.serialize(data, { prettify: true, indent: 2 }); + expect(result).toContain('\n'); + expect(result).toContain(' '); + }); + + it('should sort keys', () => { + const data = { zebra: 1, apple: 2, banana: 3 }; + const result = serializer.serialize(data, { sortKeys: true }); + const keys = Object.keys(JSON.parse(result)); + expect(keys).toEqual(['apple', 'banana', 'zebra']); + }); + }); + + describe('YAMLSerializer', () => { + const serializer = new YAMLSerializer(); + + it('should serialize to YAML', () => { + const data = { name: 'test', value: 42 }; + const result = serializer.serialize(data); + expect(result).toContain('name: test'); + expect(result).toContain('value: 42'); + }); + + it('should deserialize from YAML', () => { + const yaml = 'name: test\nvalue: 42'; + const result = serializer.deserialize(yaml); + expect(result).toEqual({ name: 'test', value: 42 }); + }); + }); + + describe('TypeScriptSerializer', () => { + const serializer = new TypeScriptSerializer('typescript'); + + it('should serialize to TypeScript module', () => { + const data = { name: 'test', value: 42 }; + const result = serializer.serialize(data); + expect(result).toContain('import type'); + expect(result).toContain('export const metadata'); + expect(result).toContain('export default metadata'); + }); + + it('should get correct extension', () => { + const ts = new TypeScriptSerializer('typescript'); + expect(ts.getExtension()).toBe('.ts'); + + const js = new TypeScriptSerializer('javascript'); + expect(js.getExtension()).toBe('.js'); + }); + }); +}); diff --git a/packages/metadata/src/serializers/typescript-serializer.ts b/packages/metadata/src/serializers/typescript-serializer.ts new file mode 100644 index 000000000..c49d92245 --- /dev/null +++ b/packages/metadata/src/serializers/typescript-serializer.ts @@ -0,0 +1,125 @@ +/** + * TypeScript/JavaScript Metadata Serializer + * + * Handles TypeScript/JavaScript module format serialization and deserialization + */ + +import type { z } from 'zod'; +import type { MetadataFormat } from '@objectstack/spec/system'; +import type { MetadataSerializer, SerializeOptions } from './serializer-interface.js'; + +export class TypeScriptSerializer implements MetadataSerializer { + constructor(private format: 'typescript' | 'javascript' = 'typescript') {} + + serialize(item: T, options?: SerializeOptions): string { + const { prettify = true, indent = 2 } = options || {}; + + const jsonStr = JSON.stringify(item, null, prettify ? indent : 0); + + if (this.format === 'typescript') { + return `import type { ServiceObject } from '@objectstack/spec/data';\n\n` + + `export const metadata: ServiceObject = ${jsonStr};\n\n` + + `export default metadata;\n`; + } else { + return `export const metadata = ${jsonStr};\n\n` + + `export default metadata;\n`; + } + } + + deserialize(content: string, schema?: z.ZodSchema): T { + // For TypeScript/JavaScript files, we need to extract the exported object + // Note: This is a simplified parser that works with JSON-like object literals + // For complex TypeScript with nested objects, consider using a proper TypeScript parser + + // Try to find the object literal in various export patterns + // Pattern 1: export const metadata = {...}; + let objectStart = content.indexOf('export const'); + if (objectStart === -1) { + // Pattern 2: export default {...}; + objectStart = content.indexOf('export default'); + } + + if (objectStart === -1) { + throw new Error( + 'Could not parse TypeScript/JavaScript module. ' + + 'Expected export pattern: "export const metadata = {...};" or "export default {...};"' + ); + } + + // Find the first opening brace after the export statement + const braceStart = content.indexOf('{', objectStart); + if (braceStart === -1) { + throw new Error('Could not find object literal in export statement'); + } + + // Find the matching closing brace by counting braces + // Handle string literals to avoid counting braces inside strings + let braceCount = 0; + let braceEnd = -1; + let inString = false; + let stringChar = ''; + + for (let i = braceStart; i < content.length; i++) { + const char = content[i]; + const prevChar = i > 0 ? content[i - 1] : ''; + + // Track string literals (simple handling of " and ') + if ((char === '"' || char === "'") && prevChar !== '\\') { + if (!inString) { + inString = true; + stringChar = char; + } else if (char === stringChar) { + inString = false; + stringChar = ''; + } + } + + // Count braces only when not inside strings + if (!inString) { + if (char === '{') braceCount++; + if (char === '}') { + braceCount--; + if (braceCount === 0) { + braceEnd = i; + break; + } + } + } + } + + if (braceEnd === -1) { + throw new Error('Could not find matching closing brace for object literal'); + } + + // Extract the object literal + const objectLiteral = content.substring(braceStart, braceEnd + 1); + + try { + // Parse as JSON + const parsed = JSON.parse(objectLiteral); + + if (schema) { + return schema.parse(parsed) as T; + } + + return parsed as T; + } catch (error) { + throw new Error( + `Failed to parse object literal as JSON: ${error instanceof Error ? error.message : String(error)}. ` + + 'Make sure the TypeScript/JavaScript object uses JSON-compatible syntax (no functions, comments, or trailing commas).' + ); + } + } + + getExtension(): string { + return this.format === 'typescript' ? '.ts' : '.js'; + } + + canHandle(format: MetadataFormat): boolean { + return format === 'typescript' || format === 'javascript'; + } + + getFormat(): MetadataFormat { + return this.format; + } +} diff --git a/packages/metadata/src/serializers/yaml-serializer.ts b/packages/metadata/src/serializers/yaml-serializer.ts new file mode 100644 index 000000000..128fe20c9 --- /dev/null +++ b/packages/metadata/src/serializers/yaml-serializer.ts @@ -0,0 +1,47 @@ +/** + * YAML Metadata Serializer + * + * Handles YAML format serialization and deserialization + */ + +import * as yaml from 'js-yaml'; +import type { z } from 'zod'; +import type { MetadataFormat } from '@objectstack/spec/system'; +import type { MetadataSerializer, SerializeOptions } from './serializer-interface.js'; + +export class YAMLSerializer implements MetadataSerializer { + serialize(item: T, options?: SerializeOptions): string { + const { indent = 2, sortKeys = false } = options || {}; + + return yaml.dump(item, { + indent, + sortKeys, + lineWidth: -1, // Disable line wrapping + noRefs: true, // Disable YAML references + }); + } + + deserialize(content: string, schema?: z.ZodSchema): T { + // Use JSON_SCHEMA to prevent arbitrary code execution + // This restricts YAML to JSON-compatible types only + const parsed = yaml.load(content, { schema: yaml.JSON_SCHEMA }); + + if (schema) { + return schema.parse(parsed) as T; + } + + return parsed as T; + } + + getExtension(): string { + return '.yaml'; + } + + canHandle(format: MetadataFormat): boolean { + return format === 'yaml'; + } + + getFormat(): MetadataFormat { + return 'yaml'; + } +} diff --git a/packages/metadata/tsconfig.json b/packages/metadata/tsconfig.json new file mode 100644 index 000000000..b38211580 --- /dev/null +++ b/packages/metadata/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.json", + "include": ["src/**/*"], + "exclude": ["node_modules", "dist"], + "compilerOptions": { + "outDir": "dist", + "rootDir": "src", + "module": "ES2020", + "moduleResolution": "bundler" + } +} diff --git a/packages/objectql/src/protocol.ts b/packages/objectql/src/protocol.ts index 20e1b2175..facee33da 100644 --- a/packages/objectql/src/protocol.ts +++ b/packages/objectql/src/protocol.ts @@ -121,7 +121,8 @@ export class ObjectStackProtocolImplementation implements ObjectStackProtocol { const records = await this.engine.find(request.object, options); return { object: request.object, - records, + value: records, // OData compaibility + records, // Legacy total: records.length, hasMore: false }; diff --git a/packages/spec/json-schema/system/MetadataCollectionInfo.json b/packages/spec/json-schema/system/MetadataCollectionInfo.json new file mode 100644 index 000000000..68228e130 --- /dev/null +++ b/packages/spec/json-schema/system/MetadataCollectionInfo.json @@ -0,0 +1,53 @@ +{ + "$ref": "#/definitions/MetadataCollectionInfo", + "definitions": { + "MetadataCollectionInfo": { + "type": "object", + "properties": { + "type": { + "type": "string", + "description": "Collection type" + }, + "count": { + "type": "integer", + "minimum": 0, + "description": "Number of items" + }, + "formats": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "json", + "yaml", + "typescript", + "javascript" + ] + }, + "description": "Formats in collection" + }, + "totalSize": { + "type": "integer", + "minimum": 0, + "description": "Total size in bytes" + }, + "lastModified": { + "type": "string", + "format": "date-time", + "description": "Last modification date" + }, + "location": { + "type": "string", + "description": "Collection location" + } + }, + "required": [ + "type", + "count", + "formats" + ], + "additionalProperties": false + } + }, + "$schema": "http://json-schema.org/draft-07/schema#" +} \ No newline at end of file diff --git a/packages/spec/json-schema/system/MetadataExportOptions.json b/packages/spec/json-schema/system/MetadataExportOptions.json new file mode 100644 index 000000000..b4db73416 --- /dev/null +++ b/packages/spec/json-schema/system/MetadataExportOptions.json @@ -0,0 +1,49 @@ +{ + "$ref": "#/definitions/MetadataExportOptions", + "definitions": { + "MetadataExportOptions": { + "type": "object", + "properties": { + "output": { + "type": "string", + "description": "Output file path" + }, + "format": { + "type": "string", + "enum": [ + "json", + "yaml", + "typescript", + "javascript" + ], + "default": "json", + "description": "Export format" + }, + "filter": { + "type": "string", + "description": "Filter items to export" + }, + "includeStats": { + "type": "boolean", + "default": false, + "description": "Include metadata statistics" + }, + "compress": { + "type": "boolean", + "default": false, + "description": "Compress output (gzip)" + }, + "prettify": { + "type": "boolean", + "default": true, + "description": "Pretty print output" + } + }, + "required": [ + "output" + ], + "additionalProperties": false + } + }, + "$schema": "http://json-schema.org/draft-07/schema#" +} \ No newline at end of file diff --git a/packages/spec/json-schema/system/MetadataFormat.json b/packages/spec/json-schema/system/MetadataFormat.json new file mode 100644 index 000000000..5f515c0ec --- /dev/null +++ b/packages/spec/json-schema/system/MetadataFormat.json @@ -0,0 +1,15 @@ +{ + "$ref": "#/definitions/MetadataFormat", + "definitions": { + "MetadataFormat": { + "type": "string", + "enum": [ + "json", + "yaml", + "typescript", + "javascript" + ] + } + }, + "$schema": "http://json-schema.org/draft-07/schema#" +} \ No newline at end of file diff --git a/packages/spec/json-schema/system/MetadataImportOptions.json b/packages/spec/json-schema/system/MetadataImportOptions.json new file mode 100644 index 000000000..7e9460abe --- /dev/null +++ b/packages/spec/json-schema/system/MetadataImportOptions.json @@ -0,0 +1,42 @@ +{ + "$ref": "#/definitions/MetadataImportOptions", + "definitions": { + "MetadataImportOptions": { + "type": "object", + "properties": { + "conflictResolution": { + "type": "string", + "enum": [ + "skip", + "overwrite", + "merge", + "fail" + ], + "default": "merge", + "description": "How to handle existing items" + }, + "validate": { + "type": "boolean", + "default": true, + "description": "Validate before import" + }, + "dryRun": { + "type": "boolean", + "default": false, + "description": "Simulate import without saving" + }, + "continueOnError": { + "type": "boolean", + "default": false, + "description": "Continue if validation fails" + }, + "transform": { + "type": "string", + "description": "Transform items before import" + } + }, + "additionalProperties": false + } + }, + "$schema": "http://json-schema.org/draft-07/schema#" +} \ No newline at end of file diff --git a/packages/spec/json-schema/system/MetadataLoadOptions.json b/packages/spec/json-schema/system/MetadataLoadOptions.json new file mode 100644 index 000000000..3a6fa7edf --- /dev/null +++ b/packages/spec/json-schema/system/MetadataLoadOptions.json @@ -0,0 +1,52 @@ +{ + "$ref": "#/definitions/MetadataLoadOptions", + "definitions": { + "MetadataLoadOptions": { + "type": "object", + "properties": { + "patterns": { + "type": "array", + "items": { + "type": "string" + }, + "description": "File glob patterns" + }, + "ifNoneMatch": { + "type": "string", + "description": "ETag for conditional request" + }, + "ifModifiedSince": { + "type": "string", + "format": "date-time", + "description": "Only load if modified after this date" + }, + "validate": { + "type": "boolean", + "default": true, + "description": "Validate against schema" + }, + "useCache": { + "type": "boolean", + "default": true, + "description": "Enable caching" + }, + "filter": { + "type": "string", + "description": "Filter predicate as string" + }, + "limit": { + "type": "integer", + "minimum": 1, + "description": "Maximum items to load" + }, + "recursive": { + "type": "boolean", + "default": true, + "description": "Search subdirectories" + } + }, + "additionalProperties": false + } + }, + "$schema": "http://json-schema.org/draft-07/schema#" +} \ No newline at end of file diff --git a/packages/spec/json-schema/system/MetadataLoadResult.json b/packages/spec/json-schema/system/MetadataLoadResult.json new file mode 100644 index 000000000..6d246d031 --- /dev/null +++ b/packages/spec/json-schema/system/MetadataLoadResult.json @@ -0,0 +1,86 @@ +{ + "$ref": "#/definitions/MetadataLoadResult", + "definitions": { + "MetadataLoadResult": { + "type": "object", + "properties": { + "data": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "description": "Loaded metadata" + }, + "fromCache": { + "type": "boolean", + "default": false, + "description": "Loaded from cache" + }, + "notModified": { + "type": "boolean", + "default": false, + "description": "Not modified since last request" + }, + "etag": { + "type": "string", + "description": "Entity tag" + }, + "stats": { + "type": "object", + "properties": { + "size": { + "type": "integer", + "minimum": 0, + "description": "File size in bytes" + }, + "modifiedAt": { + "type": "string", + "format": "date-time", + "description": "Last modified date" + }, + "etag": { + "type": "string", + "description": "Entity tag for cache validation" + }, + "format": { + "type": "string", + "enum": [ + "json", + "yaml", + "typescript", + "javascript" + ], + "description": "Serialization format" + }, + "path": { + "type": "string", + "description": "File system path" + }, + "metadata": { + "type": "object", + "additionalProperties": {}, + "description": "Provider-specific metadata" + } + }, + "required": [ + "size", + "modifiedAt", + "etag", + "format" + ], + "additionalProperties": false, + "description": "Metadata statistics" + }, + "loadTime": { + "type": "number", + "minimum": 0, + "description": "Load duration in ms" + } + }, + "additionalProperties": false + } + }, + "$schema": "http://json-schema.org/draft-07/schema#" +} \ No newline at end of file diff --git a/packages/spec/json-schema/system/MetadataLoaderContract.json b/packages/spec/json-schema/system/MetadataLoaderContract.json new file mode 100644 index 000000000..199008433 --- /dev/null +++ b/packages/spec/json-schema/system/MetadataLoaderContract.json @@ -0,0 +1,48 @@ +{ + "$ref": "#/definitions/MetadataLoaderContract", + "definitions": { + "MetadataLoaderContract": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Loader identifier" + }, + "supportedFormats": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "json", + "yaml", + "typescript", + "javascript" + ] + }, + "description": "Supported formats" + }, + "supportsWatch": { + "type": "boolean", + "default": false, + "description": "Supports file watching" + }, + "supportsWrite": { + "type": "boolean", + "default": true, + "description": "Supports write operations" + }, + "supportsCache": { + "type": "boolean", + "default": true, + "description": "Supports caching" + } + }, + "required": [ + "name", + "supportedFormats" + ], + "additionalProperties": false + } + }, + "$schema": "http://json-schema.org/draft-07/schema#" +} \ No newline at end of file diff --git a/packages/spec/json-schema/system/MetadataManagerConfig.json b/packages/spec/json-schema/system/MetadataManagerConfig.json new file mode 100644 index 000000000..6dbbf076e --- /dev/null +++ b/packages/spec/json-schema/system/MetadataManagerConfig.json @@ -0,0 +1,108 @@ +{ + "$ref": "#/definitions/MetadataManagerConfig", + "definitions": { + "MetadataManagerConfig": { + "type": "object", + "properties": { + "rootDir": { + "type": "string", + "description": "Root directory path" + }, + "formats": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "json", + "yaml", + "typescript", + "javascript" + ] + }, + "default": [ + "typescript", + "json", + "yaml" + ], + "description": "Enabled formats" + }, + "cache": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "default": true, + "description": "Enable caching" + }, + "ttl": { + "type": "integer", + "minimum": 0, + "default": 3600, + "description": "Cache TTL in seconds" + }, + "maxSize": { + "type": "integer", + "minimum": 0, + "description": "Max cache size in bytes" + } + }, + "additionalProperties": false, + "description": "Cache settings" + }, + "watch": { + "type": "boolean", + "default": false, + "description": "Enable file watching" + }, + "watchOptions": { + "type": "object", + "properties": { + "ignored": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Patterns to ignore" + }, + "persistent": { + "type": "boolean", + "default": true, + "description": "Keep process running" + }, + "ignoreInitial": { + "type": "boolean", + "default": true, + "description": "Ignore initial add events" + } + }, + "additionalProperties": false, + "description": "File watcher options" + }, + "validation": { + "type": "object", + "properties": { + "strict": { + "type": "boolean", + "default": true, + "description": "Strict validation" + }, + "throwOnError": { + "type": "boolean", + "default": true, + "description": "Throw on validation error" + } + }, + "additionalProperties": false, + "description": "Validation settings" + }, + "loaderOptions": { + "type": "object", + "additionalProperties": {}, + "description": "Loader-specific configuration" + } + }, + "additionalProperties": false + } + }, + "$schema": "http://json-schema.org/draft-07/schema#" +} \ No newline at end of file diff --git a/packages/spec/json-schema/system/MetadataSaveOptions.json b/packages/spec/json-schema/system/MetadataSaveOptions.json new file mode 100644 index 000000000..c749961eb --- /dev/null +++ b/packages/spec/json-schema/system/MetadataSaveOptions.json @@ -0,0 +1,64 @@ +{ + "$ref": "#/definitions/MetadataSaveOptions", + "definitions": { + "MetadataSaveOptions": { + "type": "object", + "properties": { + "format": { + "type": "string", + "enum": [ + "json", + "yaml", + "typescript", + "javascript" + ], + "default": "typescript", + "description": "Output format" + }, + "prettify": { + "type": "boolean", + "default": true, + "description": "Format with indentation" + }, + "indent": { + "type": "integer", + "minimum": 0, + "maximum": 8, + "default": 2, + "description": "Indentation spaces" + }, + "sortKeys": { + "type": "boolean", + "default": false, + "description": "Sort object keys" + }, + "includeDefaults": { + "type": "boolean", + "default": false, + "description": "Include default values" + }, + "backup": { + "type": "boolean", + "default": false, + "description": "Create backup file" + }, + "overwrite": { + "type": "boolean", + "default": true, + "description": "Overwrite existing file" + }, + "atomic": { + "type": "boolean", + "default": true, + "description": "Use atomic write operation" + }, + "path": { + "type": "string", + "description": "Custom output path" + } + }, + "additionalProperties": false + } + }, + "$schema": "http://json-schema.org/draft-07/schema#" +} \ No newline at end of file diff --git a/packages/spec/json-schema/system/MetadataSaveResult.json b/packages/spec/json-schema/system/MetadataSaveResult.json new file mode 100644 index 000000000..ef378699f --- /dev/null +++ b/packages/spec/json-schema/system/MetadataSaveResult.json @@ -0,0 +1,42 @@ +{ + "$ref": "#/definitions/MetadataSaveResult", + "definitions": { + "MetadataSaveResult": { + "type": "object", + "properties": { + "success": { + "type": "boolean", + "description": "Save successful" + }, + "path": { + "type": "string", + "description": "Output path" + }, + "etag": { + "type": "string", + "description": "Generated entity tag" + }, + "size": { + "type": "integer", + "minimum": 0, + "description": "File size" + }, + "saveTime": { + "type": "number", + "minimum": 0, + "description": "Save duration in ms" + }, + "backupPath": { + "type": "string", + "description": "Backup file path" + } + }, + "required": [ + "success", + "path" + ], + "additionalProperties": false + } + }, + "$schema": "http://json-schema.org/draft-07/schema#" +} \ No newline at end of file diff --git a/packages/spec/json-schema/system/MetadataStats.json b/packages/spec/json-schema/system/MetadataStats.json new file mode 100644 index 000000000..eea204916 --- /dev/null +++ b/packages/spec/json-schema/system/MetadataStats.json @@ -0,0 +1,51 @@ +{ + "$ref": "#/definitions/MetadataStats", + "definitions": { + "MetadataStats": { + "type": "object", + "properties": { + "size": { + "type": "integer", + "minimum": 0, + "description": "File size in bytes" + }, + "modifiedAt": { + "type": "string", + "format": "date-time", + "description": "Last modified date" + }, + "etag": { + "type": "string", + "description": "Entity tag for cache validation" + }, + "format": { + "type": "string", + "enum": [ + "json", + "yaml", + "typescript", + "javascript" + ], + "description": "Serialization format" + }, + "path": { + "type": "string", + "description": "File system path" + }, + "metadata": { + "type": "object", + "additionalProperties": {}, + "description": "Provider-specific metadata" + } + }, + "required": [ + "size", + "modifiedAt", + "etag", + "format" + ], + "additionalProperties": false + } + }, + "$schema": "http://json-schema.org/draft-07/schema#" +} \ No newline at end of file diff --git a/packages/spec/json-schema/system/MetadataWatchEvent.json b/packages/spec/json-schema/system/MetadataWatchEvent.json new file mode 100644 index 000000000..25b4e5b0d --- /dev/null +++ b/packages/spec/json-schema/system/MetadataWatchEvent.json @@ -0,0 +1,48 @@ +{ + "$ref": "#/definitions/MetadataWatchEvent", + "definitions": { + "MetadataWatchEvent": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "added", + "changed", + "deleted" + ], + "description": "Event type" + }, + "metadataType": { + "type": "string", + "description": "Type of metadata" + }, + "name": { + "type": "string", + "description": "Item identifier" + }, + "path": { + "type": "string", + "description": "File path" + }, + "data": { + "description": "Item data" + }, + "timestamp": { + "type": "string", + "format": "date-time", + "description": "Event timestamp" + } + }, + "required": [ + "type", + "metadataType", + "name", + "path", + "timestamp" + ], + "additionalProperties": false + } + }, + "$schema": "http://json-schema.org/draft-07/schema#" +} \ No newline at end of file diff --git a/packages/spec/src/system/index.ts b/packages/spec/src/system/index.ts index 9252e7682..2f16c9c78 100644 --- a/packages/spec/src/system/index.ts +++ b/packages/spec/src/system/index.ts @@ -43,6 +43,9 @@ export * from './cache.zod'; // Message Queue Protocol export * from './message-queue.zod'; +// Metadata Loader Protocol +export * from './metadata-loader.zod'; + // Search Engine Protocol export * from './search-engine.zod'; diff --git a/packages/spec/src/system/metadata-loader.test.ts b/packages/spec/src/system/metadata-loader.test.ts new file mode 100644 index 000000000..bdacb0cb7 --- /dev/null +++ b/packages/spec/src/system/metadata-loader.test.ts @@ -0,0 +1,408 @@ +import { describe, it, expect } from 'vitest'; +import { + MetadataFormatSchema, + MetadataStatsSchema, + MetadataLoadOptionsSchema, + MetadataSaveOptionsSchema, + MetadataExportOptionsSchema, + MetadataImportOptionsSchema, + MetadataLoadResultSchema, + MetadataSaveResultSchema, + MetadataWatchEventSchema, + MetadataCollectionInfoSchema, + MetadataLoaderContractSchema, + MetadataManagerConfigSchema, +} from './metadata-loader.zod'; + +describe('MetadataLoaderProtocol', () => { + describe('MetadataFormatSchema', () => { + it('should accept valid formats', () => { + expect(MetadataFormatSchema.parse('json')).toBe('json'); + expect(MetadataFormatSchema.parse('yaml')).toBe('yaml'); + expect(MetadataFormatSchema.parse('typescript')).toBe('typescript'); + expect(MetadataFormatSchema.parse('javascript')).toBe('javascript'); + }); + + it('should reject invalid formats', () => { + expect(() => MetadataFormatSchema.parse('xml')).toThrow(); + expect(() => MetadataFormatSchema.parse('toml')).toThrow(); + }); + }); + + describe('MetadataStatsSchema', () => { + it('should validate metadata statistics', () => { + const stats = { + size: 1024, + modifiedAt: new Date('2026-01-31T00:00:00Z'), + etag: '"abc123"', + format: 'json' as const, + }; + + const result = MetadataStatsSchema.parse(stats); + expect(result.size).toBe(1024); + expect(result.etag).toBe('"abc123"'); + expect(result.format).toBe('json'); + }); + + it('should allow optional fields', () => { + const stats = { + size: 2048, + modifiedAt: new Date(), + etag: '"xyz789"', + format: 'yaml' as const, + path: '/metadata/objects/customer.object.yaml', + metadata: { encoding: 'utf-8' }, + }; + + const result = MetadataStatsSchema.parse(stats); + expect(result.path).toBe('/metadata/objects/customer.object.yaml'); + expect(result.metadata).toEqual({ encoding: 'utf-8' }); + }); + + it('should reject negative size', () => { + const stats = { + size: -100, + modifiedAt: new Date(), + etag: '"abc"', + format: 'json' as const, + }; + + expect(() => MetadataStatsSchema.parse(stats)).toThrow(); + }); + }); + + describe('MetadataLoadOptionsSchema', () => { + it('should apply default values', () => { + const options = {}; + const result = MetadataLoadOptionsSchema.parse(options); + + expect(result.validate).toBe(true); + expect(result.useCache).toBe(true); + expect(result.recursive).toBe(true); + }); + + it('should accept all options', () => { + const options = { + patterns: ['**/*.object.ts', '**/*.object.json'], + ifNoneMatch: '"etag123"', + ifModifiedSince: new Date('2026-01-01T00:00:00Z'), + validate: false, + useCache: false, + filter: '(item) => item.name.startsWith("sys_")', + limit: 100, + recursive: false, + }; + + const result = MetadataLoadOptionsSchema.parse(options); + expect(result.patterns).toHaveLength(2); + expect(result.limit).toBe(100); + expect(result.validate).toBe(false); + }); + }); + + describe('MetadataSaveOptionsSchema', () => { + it('should apply default values', () => { + const options = {}; + const result = MetadataSaveOptionsSchema.parse(options); + + expect(result.format).toBe('typescript'); + expect(result.prettify).toBe(true); + expect(result.indent).toBe(2); + expect(result.overwrite).toBe(true); + expect(result.atomic).toBe(true); + }); + + it('should validate indent range', () => { + expect(() => + MetadataSaveOptionsSchema.parse({ indent: -1 }) + ).toThrow(); + + expect(() => + MetadataSaveOptionsSchema.parse({ indent: 10 }) + ).toThrow(); + + expect( + MetadataSaveOptionsSchema.parse({ indent: 4 }).indent + ).toBe(4); + }); + + it('should accept custom path', () => { + const options = { + path: '/custom/path/object.ts', + format: 'json' as const, + }; + + const result = MetadataSaveOptionsSchema.parse(options); + expect(result.path).toBe('/custom/path/object.ts'); + expect(result.format).toBe('json'); + }); + }); + + describe('MetadataExportOptionsSchema', () => { + it('should require output path', () => { + expect(() => MetadataExportOptionsSchema.parse({})).toThrow(); + + const options = { output: './export/objects.json' }; + const result = MetadataExportOptionsSchema.parse(options); + expect(result.output).toBe('./export/objects.json'); + }); + + it('should apply defaults', () => { + const options = { output: './export.json' }; + const result = MetadataExportOptionsSchema.parse(options); + + expect(result.format).toBe('json'); + expect(result.includeStats).toBe(false); + expect(result.compress).toBe(false); + expect(result.prettify).toBe(true); + }); + }); + + describe('MetadataImportOptionsSchema', () => { + it('should apply default conflict resolution', () => { + const options = {}; + const result = MetadataImportOptionsSchema.parse(options); + + expect(result.conflictResolution).toBe('merge'); + expect(result.validate).toBe(true); + expect(result.dryRun).toBe(false); + expect(result.continueOnError).toBe(false); + }); + + it('should accept all conflict strategies', () => { + const strategies = ['skip', 'overwrite', 'merge', 'fail'] as const; + + strategies.forEach(strategy => { + const result = MetadataImportOptionsSchema.parse({ + conflictResolution: strategy + }); + expect(result.conflictResolution).toBe(strategy); + }); + }); + + it('should accept transform function', () => { + const options = { + transform: '(item) => ({ ...item, imported: true })', + }; + + const result = MetadataImportOptionsSchema.parse(options); + expect(result.transform).toBeDefined(); + }); + }); + + describe('MetadataLoadResultSchema', () => { + it('should validate load result', () => { + const result = { + data: { name: 'customer', label: 'Customer' }, + fromCache: false, + notModified: false, + }; + + const validated = MetadataLoadResultSchema.parse(result); + expect(validated.data).toBeDefined(); + expect(validated.fromCache).toBe(false); + }); + + it('should accept null data (not found)', () => { + const result = { + data: null, + fromCache: false, + notModified: false, + }; + + const validated = MetadataLoadResultSchema.parse(result); + expect(validated.data).toBeNull(); + }); + + it('should include optional fields', () => { + const result = { + data: { name: 'test' }, + fromCache: true, + notModified: true, + etag: '"abc123"', + stats: { + size: 512, + modifiedAt: new Date(), + etag: '"abc123"', + format: 'typescript' as const, + }, + loadTime: 45.5, + }; + + const validated = MetadataLoadResultSchema.parse(result); + expect(validated.etag).toBe('"abc123"'); + expect(validated.loadTime).toBe(45.5); + expect(validated.stats).toBeDefined(); + }); + }); + + describe('MetadataSaveResultSchema', () => { + it('should validate save result', () => { + const result = { + success: true, + path: '/metadata/objects/customer.object.ts', + }; + + const validated = MetadataSaveResultSchema.parse(result); + expect(validated.success).toBe(true); + expect(validated.path).toBeDefined(); + }); + + it('should include optional fields', () => { + const result = { + success: true, + path: '/metadata/objects/customer.object.ts', + etag: '"new-etag"', + size: 2048, + saveTime: 12.3, + backupPath: '/metadata/objects/customer.object.ts.bak', + }; + + const validated = MetadataSaveResultSchema.parse(result); + expect(validated.size).toBe(2048); + expect(validated.backupPath).toBeDefined(); + }); + }); + + describe('MetadataWatchEventSchema', () => { + it('should validate watch events', () => { + const events = [ + { + type: 'added' as const, + metadataType: 'object', + name: 'customer', + path: '/objects/customer.object.ts', + data: { name: 'customer' }, + timestamp: new Date(), + }, + { + type: 'changed' as const, + metadataType: 'view', + name: 'customer_list', + path: '/views/customer_list.view.ts', + timestamp: new Date(), + }, + { + type: 'deleted' as const, + metadataType: 'app', + name: 'old_app', + path: '/apps/old_app.ts', + timestamp: new Date(), + }, + ]; + + events.forEach(event => { + const validated = MetadataWatchEventSchema.parse(event); + expect(validated.type).toBe(event.type); + expect(validated.metadataType).toBeDefined(); + }); + }); + }); + + describe('MetadataCollectionInfoSchema', () => { + it('should validate collection info', () => { + const info = { + type: 'object', + count: 42, + formats: ['typescript', 'json'] as const, + }; + + const validated = MetadataCollectionInfoSchema.parse(info); + expect(validated.count).toBe(42); + expect(validated.formats).toHaveLength(2); + }); + + it('should accept optional fields', () => { + const info = { + type: 'view', + count: 15, + formats: ['yaml'] as const, + totalSize: 51200, + lastModified: new Date('2026-01-31T00:00:00Z'), + location: '/metadata/views', + }; + + const validated = MetadataCollectionInfoSchema.parse(info); + expect(validated.totalSize).toBe(51200); + expect(validated.location).toBe('/metadata/views'); + }); + }); + + describe('MetadataLoaderContractSchema', () => { + it('should validate loader contract', () => { + const contract = { + name: 'filesystem', + supportedFormats: ['json', 'yaml', 'typescript'] as const, + }; + + const validated = MetadataLoaderContractSchema.parse(contract); + expect(validated.name).toBe('filesystem'); + expect(validated.supportsWatch).toBe(false); // default + expect(validated.supportsWrite).toBe(true); // default + expect(validated.supportsCache).toBe(true); // default + }); + + it('should allow custom capabilities', () => { + const contract = { + name: 'http', + supportedFormats: ['json'] as const, + supportsWatch: false, + supportsWrite: false, + supportsCache: true, + }; + + const validated = MetadataLoaderContractSchema.parse(contract); + expect(validated.supportsWrite).toBe(false); + expect(validated.supportsCache).toBe(true); + }); + }); + + describe('MetadataManagerConfigSchema', () => { + it('should apply defaults', () => { + const config = {}; + const validated = MetadataManagerConfigSchema.parse(config); + + expect(validated.formats).toEqual(['typescript', 'json', 'yaml']); + expect(validated.watch).toBe(false); + }); + + it('should validate complete configuration', () => { + const config = { + rootDir: '/metadata', + formats: ['typescript', 'json'] as const, + cache: { + enabled: true, + ttl: 7200, + maxSize: 10485760, // 10MB + }, + watch: true, + watchOptions: { + ignored: ['**/node_modules/**', '**/*.test.ts'], + persistent: true, + ignoreInitial: true, + }, + validation: { + strict: true, + throwOnError: true, + }, + loaderOptions: { + encoding: 'utf-8', + }, + }; + + const validated = MetadataManagerConfigSchema.parse(config); + expect(validated.rootDir).toBe('/metadata'); + expect(validated.cache?.ttl).toBe(7200); + expect(validated.watchOptions?.ignored).toHaveLength(2); + expect(validated.loaderOptions?.encoding).toBe('utf-8'); + }); + + it('should reject negative TTL', () => { + const config = { + cache: { enabled: true, ttl: -100 }, + }; + + expect(() => MetadataManagerConfigSchema.parse(config)).toThrow(); + }); + }); +}); diff --git a/packages/spec/src/system/metadata-loader.zod.ts b/packages/spec/src/system/metadata-loader.zod.ts new file mode 100644 index 000000000..47a951814 --- /dev/null +++ b/packages/spec/src/system/metadata-loader.zod.ts @@ -0,0 +1,456 @@ +import { z } from 'zod'; + +/** + * # Metadata Loader Protocol + * + * Defines the standard interface for loading and saving metadata in ObjectStack. + * This protocol enables consistent metadata operations across different storage backends + * (filesystem, HTTP, S3, databases) and serialization formats (JSON, YAML, TypeScript). + */ + +/** + * Metadata Format Enum + * Supported serialization formats for metadata + */ +export const MetadataFormatSchema = z.enum(['json', 'yaml', 'typescript', 'javascript']); + +/** + * Metadata Statistics + * Information about a metadata item without loading its full content + */ +export const MetadataStatsSchema = z.object({ + /** + * Size of the metadata file in bytes + */ + size: z.number().int().min(0).describe('File size in bytes'), + + /** + * Last modification timestamp + */ + modifiedAt: z.date().describe('Last modified date'), + + /** + * ETag for cache validation + * Used for conditional requests (If-None-Match header) + */ + etag: z.string().describe('Entity tag for cache validation'), + + /** + * Serialization format + */ + format: MetadataFormatSchema.describe('Serialization format'), + + /** + * Full file path (if applicable) + */ + path: z.string().optional().describe('File system path'), + + /** + * Additional metadata provider-specific properties + */ + metadata: z.record(z.any()).optional().describe('Provider-specific metadata'), +}); + +/** + * Metadata Load Options + */ +export const MetadataLoadOptionsSchema = z.object({ + /** + * Glob patterns to match files + * Example: ["**\/*.object.ts", "**\/*.object.json"] + */ + patterns: z.array(z.string()).optional().describe('File glob patterns'), + + /** + * If-None-Match header for conditional loading + * Only load if ETag doesn't match + */ + ifNoneMatch: z.string().optional().describe('ETag for conditional request'), + + /** + * If-Modified-Since header for conditional loading + */ + ifModifiedSince: z.date().optional().describe('Only load if modified after this date'), + + /** + * Whether to validate against Zod schema + */ + validate: z.boolean().default(true).describe('Validate against schema'), + + /** + * Whether to use cache if available + */ + useCache: z.boolean().default(true).describe('Enable caching'), + + /** + * Filter function (serialized as string) + * Example: "(item) => item.name.startsWith('sys_')" + */ + filter: z.string().optional().describe('Filter predicate as string'), + + /** + * Maximum number of items to load + */ + limit: z.number().int().min(1).optional().describe('Maximum items to load'), + + /** + * Recursively search subdirectories + */ + recursive: z.boolean().default(true).describe('Search subdirectories'), +}); + +/** + * Metadata Save Options + */ +export const MetadataSaveOptionsSchema = z.object({ + /** + * Serialization format + */ + format: MetadataFormatSchema.default('typescript').describe('Output format'), + + /** + * Prettify output (formatted with indentation) + */ + prettify: z.boolean().default(true).describe('Format with indentation'), + + /** + * Indentation size (spaces) + */ + indent: z.number().int().min(0).max(8).default(2).describe('Indentation spaces'), + + /** + * Sort object keys alphabetically + */ + sortKeys: z.boolean().default(false).describe('Sort object keys'), + + /** + * Include default values in output + */ + includeDefaults: z.boolean().default(false).describe('Include default values'), + + /** + * Create backup before overwriting + */ + backup: z.boolean().default(false).describe('Create backup file'), + + /** + * Overwrite if exists + */ + overwrite: z.boolean().default(true).describe('Overwrite existing file'), + + /** + * Atomic write (write to temp file, then rename) + */ + atomic: z.boolean().default(true).describe('Use atomic write operation'), + + /** + * Custom file path (overrides default location) + */ + path: z.string().optional().describe('Custom output path'), +}); + +/** + * Metadata Export Options + */ +export const MetadataExportOptionsSchema = z.object({ + /** + * Output file path + */ + output: z.string().describe('Output file path'), + + /** + * Export format + */ + format: MetadataFormatSchema.default('json').describe('Export format'), + + /** + * Filter predicate as string + */ + filter: z.string().optional().describe('Filter items to export'), + + /** + * Include statistics in export + */ + includeStats: z.boolean().default(false).describe('Include metadata statistics'), + + /** + * Compress output + */ + compress: z.boolean().default(false).describe('Compress output (gzip)'), + + /** + * Pretty print output + */ + prettify: z.boolean().default(true).describe('Pretty print output'), +}); + +/** + * Metadata Import Options + */ +export const MetadataImportOptionsSchema = z.object({ + /** + * Conflict resolution strategy + */ + conflictResolution: z.enum(['skip', 'overwrite', 'merge', 'fail']) + .default('merge') + .describe('How to handle existing items'), + + /** + * Validate items against schema + */ + validate: z.boolean().default(true).describe('Validate before import'), + + /** + * Dry run (don't actually save) + */ + dryRun: z.boolean().default(false).describe('Simulate import without saving'), + + /** + * Continue on errors + */ + continueOnError: z.boolean().default(false).describe('Continue if validation fails'), + + /** + * Transform function (as string) + * Example: "(item) => ({ ...item, imported: true })" + */ + transform: z.string().optional().describe('Transform items before import'), +}); + +/** + * Metadata Loader Result + * Result of a metadata load operation + */ +export const MetadataLoadResultSchema = z.object({ + /** + * Loaded data + */ + data: z.any().nullable().describe('Loaded metadata'), + + /** + * Whether data came from cache (304 Not Modified) + */ + fromCache: z.boolean().default(false).describe('Loaded from cache'), + + /** + * Not modified (conditional request matched) + */ + notModified: z.boolean().default(false).describe('Not modified since last request'), + + /** + * ETag of loaded data + */ + etag: z.string().optional().describe('Entity tag'), + + /** + * Statistics about loaded data + */ + stats: MetadataStatsSchema.optional().describe('Metadata statistics'), + + /** + * Load time in milliseconds + */ + loadTime: z.number().min(0).optional().describe('Load duration in ms'), +}); + +/** + * Metadata Save Result + */ +export const MetadataSaveResultSchema = z.object({ + /** + * Whether save was successful + */ + success: z.boolean().describe('Save successful'), + + /** + * Path where file was saved + */ + path: z.string().describe('Output path'), + + /** + * Generated ETag + */ + etag: z.string().optional().describe('Generated entity tag'), + + /** + * File size in bytes + */ + size: z.number().int().min(0).optional().describe('File size'), + + /** + * Save time in milliseconds + */ + saveTime: z.number().min(0).optional().describe('Save duration in ms'), + + /** + * Backup path (if created) + */ + backupPath: z.string().optional().describe('Backup file path'), +}); + +/** + * Metadata Watch Event + */ +export const MetadataWatchEventSchema = z.object({ + /** + * Event type + */ + type: z.enum(['added', 'changed', 'deleted']).describe('Event type'), + + /** + * Metadata type (e.g., 'object', 'view', 'app') + */ + metadataType: z.string().describe('Type of metadata'), + + /** + * Item name/identifier + */ + name: z.string().describe('Item identifier'), + + /** + * Full file path + */ + path: z.string().describe('File path'), + + /** + * Loaded item data (for added/changed events) + */ + data: z.any().optional().describe('Item data'), + + /** + * Timestamp + */ + timestamp: z.date().describe('Event timestamp'), +}); + +/** + * Metadata Collection Info + * Summary of a metadata collection + */ +export const MetadataCollectionInfoSchema = z.object({ + /** + * Collection type (e.g., 'object', 'view', 'app') + */ + type: z.string().describe('Collection type'), + + /** + * Total items in collection + */ + count: z.number().int().min(0).describe('Number of items'), + + /** + * Formats found in collection + */ + formats: z.array(MetadataFormatSchema).describe('Formats in collection'), + + /** + * Total size in bytes + */ + totalSize: z.number().int().min(0).optional().describe('Total size in bytes'), + + /** + * Last modified timestamp + */ + lastModified: z.date().optional().describe('Last modification date'), + + /** + * Collection location (path or URL) + */ + location: z.string().optional().describe('Collection location'), +}); + +/** + * Metadata Loader Interface Contract + * Defines the standard methods all metadata loaders must implement + */ +export const MetadataLoaderContractSchema = z.object({ + /** + * Loader name/identifier + */ + name: z.string().describe('Loader identifier'), + + /** + * Supported formats + */ + supportedFormats: z.array(MetadataFormatSchema).describe('Supported formats'), + + /** + * Whether loader supports watching for changes + */ + supportsWatch: z.boolean().default(false).describe('Supports file watching'), + + /** + * Whether loader supports saving + */ + supportsWrite: z.boolean().default(true).describe('Supports write operations'), + + /** + * Whether loader supports caching + */ + supportsCache: z.boolean().default(true).describe('Supports caching'), +}); + +/** + * Metadata Manager Configuration + */ +export const MetadataManagerConfigSchema = z.object({ + /** + * Root directory for metadata (for filesystem loaders) + */ + rootDir: z.string().optional().describe('Root directory path'), + + /** + * Enabled serialization formats + */ + formats: z.array(MetadataFormatSchema).default(['typescript', 'json', 'yaml']).describe('Enabled formats'), + + /** + * Cache configuration + */ + cache: z.object({ + enabled: z.boolean().default(true).describe('Enable caching'), + ttl: z.number().int().min(0).default(3600).describe('Cache TTL in seconds'), + maxSize: z.number().int().min(0).optional().describe('Max cache size in bytes'), + }).optional().describe('Cache settings'), + + /** + * Watch for file changes + */ + watch: z.boolean().default(false).describe('Enable file watching'), + + /** + * Watch options + */ + watchOptions: z.object({ + ignored: z.array(z.string()).optional().describe('Patterns to ignore'), + persistent: z.boolean().default(true).describe('Keep process running'), + ignoreInitial: z.boolean().default(true).describe('Ignore initial add events'), + }).optional().describe('File watcher options'), + + /** + * Validation settings + */ + validation: z.object({ + strict: z.boolean().default(true).describe('Strict validation'), + throwOnError: z.boolean().default(true).describe('Throw on validation error'), + }).optional().describe('Validation settings'), + + /** + * Loader-specific options + */ + loaderOptions: z.record(z.any()).optional().describe('Loader-specific configuration'), +}); + +// Export types +export type MetadataFormat = z.infer; +export type MetadataStats = z.infer; +export type MetadataLoadOptions = z.input; +export type MetadataSaveOptions = z.infer; +export type MetadataExportOptions = z.infer; +export type MetadataImportOptions = z.infer; +export type MetadataLoadResult = z.infer; +export type MetadataSaveResult = z.infer; +export type MetadataWatchEvent = z.infer; +export type MetadataCollectionInfo = z.infer; +export type MetadataLoaderContract = z.infer; +export type MetadataManagerConfig = z.infer; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 143c68376..e5d018e5d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -395,6 +395,43 @@ importers: specifier: ^1.0.0 version: 1.6.1(@types/node@20.19.30)(lightningcss@1.30.2) + packages/metadata: + dependencies: + '@objectstack/core': + specifier: workspace:* + version: link:../core + '@objectstack/spec': + specifier: workspace:* + version: link:../spec + '@objectstack/types': + specifier: workspace:* + version: link:../types + chokidar: + specifier: ^3.5.3 + version: 3.6.0 + glob: + specifier: ^10.3.10 + version: 10.5.0 + js-yaml: + specifier: ^4.1.0 + version: 4.1.1 + zod: + specifier: ^3.22.4 + version: 3.25.76 + devDependencies: + '@types/js-yaml': + specifier: ^4.0.9 + version: 4.0.9 + '@types/node': + specifier: ^20.0.0 + version: 20.19.30 + typescript: + specifier: ^5.0.0 + version: 5.9.3 + vitest: + specifier: ^1.0.0 + version: 1.6.1(@types/node@20.19.30)(lightningcss@1.30.2) + packages/objectql: dependencies: '@objectstack/core': @@ -1991,6 +2028,9 @@ packages: '@types/hast@3.0.4': resolution: {integrity: sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ==} + '@types/js-yaml@4.0.9': + resolution: {integrity: sha512-k4MGaQl5TGo/iipqb2UDG2UwjXziSWkh0uysQelTlJpX1qGlpUZYm8PnO4DxG1qBomtJUdYJ6qR6xdIah10JLg==} + '@types/mdast@4.0.4': resolution: {integrity: sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA==} @@ -2144,6 +2184,10 @@ packages: any-promise@1.3.0: resolution: {integrity: sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==} + anymatch@3.1.3: + resolution: {integrity: sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==} + engines: {node: '>= 8'} + arg@4.1.3: resolution: {integrity: sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==} @@ -2200,6 +2244,10 @@ packages: resolution: {integrity: sha512-pbnl5XzGBdrFU/wT4jqmJVPn2B6UHPBOhzMQkY/SPUPB6QtUXtmBHBIwCbXJol93mOpGMnQyP/+BB19q04xj7g==} engines: {node: '>=4'} + binary-extensions@2.3.0: + resolution: {integrity: sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==} + engines: {node: '>=8'} + brace-expansion@2.0.2: resolution: {integrity: sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==} @@ -2265,6 +2313,10 @@ packages: resolution: {integrity: sha512-PAJdDJusoxnwm1VwW07VWwUN1sl7smmC3OKggvndJFadxxDRyFJBX/ggnu/KE4kQAB7a3Dp8f/YXC1FlUprWmA==} engines: {node: '>= 16'} + chokidar@3.6.0: + resolution: {integrity: sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==} + engines: {node: '>= 8.10.0'} + chokidar@4.0.3: resolution: {integrity: sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==} engines: {node: '>= 14.16.0'} @@ -2757,6 +2809,10 @@ packages: is-alphanumerical@2.0.1: resolution: {integrity: sha512-hmbYhX/9MUMF5uh7tOXyK/n0ZvWpad5caBA17GsC6vyuCqaWliRG5K1qS9inmUhEMaOBIW7/whAnSwveW/LtZw==} + is-binary-path@2.1.0: + resolution: {integrity: sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==} + engines: {node: '>=8'} + is-decimal@2.0.1: resolution: {integrity: sha512-AAB9hiomQs5DXWcRB1rqsxGUstbRroFOPPVAomNk/3XHR5JyEZChOyTWe2oayKnsSsr/kcGqF+z6yuH6HHpN0A==} @@ -3236,6 +3292,10 @@ packages: node-releases@2.0.27: resolution: {integrity: sha512-nmh3lCkYZ3grZvqcCH+fjmQ7X+H0OeZgP40OierEaAptX4XofMh5kwNbWh7lBduUzCcV/8kZ+NDLCwm2iorIlA==} + normalize-path@3.0.0: + resolution: {integrity: sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==} + engines: {node: '>=0.10.0'} + npm-run-path@5.3.0: resolution: {integrity: sha512-ppwTtiJZq0O/ai0z7yfudtBpWIoxM8yE6nHi1X47eFR2EWORqfbu6CnPlNsjeN683eT0qG6H/Pyf9fCcvjnnnQ==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} @@ -3516,6 +3576,10 @@ packages: resolution: {integrity: sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + readdirp@3.6.0: + resolution: {integrity: sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==} + engines: {node: '>=8.10.0'} + readdirp@4.1.2: resolution: {integrity: sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==} engines: {node: '>= 14.18.0'} @@ -5491,6 +5555,8 @@ snapshots: dependencies: '@types/unist': 3.0.3 + '@types/js-yaml@4.0.9': {} + '@types/mdast@4.0.4': dependencies: '@types/unist': 3.0.3 @@ -5664,6 +5730,11 @@ snapshots: any-promise@1.3.0: {} + anymatch@3.1.3: + dependencies: + normalize-path: 3.0.0 + picomatch: 2.3.1 + arg@4.1.3: {} argparse@1.0.10: @@ -5707,6 +5778,8 @@ snapshots: dependencies: is-windows: 1.0.2 + binary-extensions@2.3.0: {} + brace-expansion@2.0.2: dependencies: balanced-match: 1.0.2 @@ -5775,6 +5848,18 @@ snapshots: check-error@2.1.3: {} + chokidar@3.6.0: + dependencies: + anymatch: 3.1.3 + braces: 3.0.3 + glob-parent: 5.1.2 + is-binary-path: 2.1.0 + is-glob: 4.0.3 + normalize-path: 3.0.0 + readdirp: 3.6.0 + optionalDependencies: + fsevents: 2.3.3 + chokidar@4.0.3: dependencies: readdirp: 4.1.2 @@ -6318,6 +6403,10 @@ snapshots: is-alphabetical: 2.0.1 is-decimal: 2.0.1 + is-binary-path@2.1.0: + dependencies: + binary-extensions: 2.3.0 + is-decimal@2.0.1: {} is-extglob@2.1.1: {} @@ -7031,6 +7120,8 @@ snapshots: node-releases@2.0.27: {} + normalize-path@3.0.0: {} + npm-run-path@5.3.0: dependencies: path-key: 4.0.0 @@ -7301,6 +7392,10 @@ snapshots: process: 0.11.10 string_decoder: 1.3.0 + readdirp@3.6.0: + dependencies: + picomatch: 2.3.1 + readdirp@4.1.2: {} readdirp@5.0.0: {} diff --git "a/\345\205\203\346\225\260\346\215\256\347\256\241\347\220\206\346\236\266\346\236\204\350\257\204\344\274\260\346\212\245\345\221\212.md" "b/\345\205\203\346\225\260\346\215\256\347\256\241\347\220\206\346\236\266\346\236\204\350\257\204\344\274\260\346\212\245\345\221\212.md" new file mode 100644 index 000000000..8e34af382 --- /dev/null +++ "b/\345\205\203\346\225\260\346\215\256\347\256\241\347\220\206\346\236\266\346\236\204\350\257\204\344\274\260\346\212\245\345\221\212.md" @@ -0,0 +1,504 @@ +# ObjectStack 元数据加载/保存功能包评估报告 + +**日期**: 2026年1月31日 +**评估对象**: 元数据加载/保存功能应该放在哪个软件包中 +**评估方法**: 全面扫描系统现有软件包、协议和微内核架构 + +--- + +## 一、执行摘要 (Executive Summary) + +经过对ObjectStack微内核架构的全面分析，**建议创建新的独立包 `@objectstack/metadata`（Layer 3）**，专门负责元数据的加载、保存、序列化和持久化操作。 + +这个决策基于以下关键因素： +- ✅ 符合微内核架构的关注点分离原则 +- ✅ 避免违反现有包的单一职责原则 +- ✅ 为未来功能扩展提供清晰的基础 +- ✅ 保持向后兼容性，支持渐进式迁移 + +--- + +## 二、现状分析 (Current State Analysis) + +### 2.1 现有软件包结构 + +ObjectStack采用分层架构，共6层： + +| 层级 | 包名 | 职责 | +|-----|------|-----| +| Layer 0 | @objectstack/spec | 协议定义（Zod schemas） | +| Layer 1 | @objectstack/types | 共享类型定义 | +| Layer 2 | @objectstack/core | 微内核（插件生命周期、服务注册） | +| Layer 3 | @objectstack/objectql | 查询引擎（内存注册表） | +| Layer 3 | @objectstack/runtime | 运行时工具（插件包装器） | +| Layer 4 | @objectstack/client | 客户端SDK | +| Layer 4 | @objectstack/client-react | React集成 | +| Layer 5 | @objectstack/driver-* | 驱动插件 | +| Layer 6 | @objectstack/cli | 命令行工具 | + +### 2.2 现有元数据相关组件 + +当前元数据功能分散在多个包中： + +1. **SchemaRegistry** (`@objectstack/objectql`) + - 位置：`packages/objectql/src/registry.ts` + - 功能：内存中的元数据注册表 + - 限制：仅支持运行时存储，无持久化能力 + +2. **AppPlugin/DriverPlugin** (`@objectstack/runtime`) + - 位置：`packages/runtime/src/app-plugin.ts` + - 功能：加载应用清单并注册为服务 + - 限制：仅处理清单加载，无通用化 + +3. **Metadata Hooks** (`@objectstack/client-react`) + - 位置：`packages/client-react/src/metadata-hooks.tsx` + - 功能：客户端获取元数据（useObject, useView） + - 限制：仅客户端，依赖HTTP API + +4. **Hub/Composer协议** (`@objectstack/spec/hub`) + - 位置：`packages/spec/src/hub/composer.zod.ts` + - 功能：云端清单编排和组合 + - 限制：仅协议定义，无实现 + +### 2.3 存在的问题 + +1. **职责分散**：元数据I/O逻辑散落在多个包中 +2. **缺乏统一接口**：没有标准的加载/保存API +3. **重复代码**：序列化逻辑可能在多处重复 +4. **扩展困难**：添加新格式（YAML, TypeScript）需要改多处 +5. **缺乏持久化**：SchemaRegistry只在内存中，无法保存到磁盘 + +--- + +## 三、候选方案评估 (Options Evaluation) + +### 方案A：放入 @objectstack/spec ❌ + +**理由拒绝：** +- Spec包的核心约束是"零运行时依赖（除Zod外）" +- 添加I/O功能需要引入 `fs`, `glob`, `js-yaml` 等依赖 +- 违反了"协议定义与实现分离"的原则 +- Spec应该定义"是什么"，而不是"怎么做" + +**影响：** +- 破坏包的纯净性 +- 增加不必要的依赖 +- 混淆协议层和实现层 + +### 方案B：放入 @objectstack/core ❌ + +**理由拒绝：** +- Core是最小化微内核，只负责插件生命周期和服务注册 +- 添加元数据I/O是特定领域逻辑，会膨胀内核 +- 违反"微内核应该保持最小"的设计原则 +- 内核应该对领域逻辑保持无知 + +**影响：** +- 违背微内核架构设计 +- 增加内核复杂度 +- 降低内核稳定性 + +### 方案C：放入 @objectstack/objectql ❌ + +**理由拒绝：** +- ObjectQL专注于查询执行和驱动管理 +- 当前的SchemaRegistry只是内存存储 +- 添加持久化会违反"单一职责原则" +- ObjectQL层不应该知道文件系统和序列化格式 + +**影响：** +- 混淆查询引擎和元数据管理的职责 +- ObjectQL变得臃肿 +- 难以独立测试和维护 + +### 方案D：放入 @objectstack/runtime ❌ + +**理由拒绝：** +- Runtime已经是一个"万金油"包，职责不够清晰 +- 添加更多功能会使边界更加模糊 +- Runtime应该专注于插件包装和HTTP工具 +- 混合I/O逻辑会导致包失去焦点 + +**影响：** +- 进一步模糊包的职责 +- 增加维护复杂度 +- 不符合清晰架构原则 + +### 方案E：放入 @objectstack/cli ❌ + +**理由拒绝：** +- CLI是Layer 6的独立工具 +- 服务器运行时不应该依赖CLI工具 +- 创建了不合理的向上依赖 +- CLI应该使用底层库，而不是被底层使用 + +**影响：** +- 破坏依赖层次 +- 无法在服务器端使用 +- 违反依赖倒置原则 + +--- + +## 四、推荐方案 (Recommended Solution) + +### ✅ 方案F：创建新包 @objectstack/metadata (Layer 3) + +#### 4.1 核心优势 + +1. **清晰的职责边界** + - 专注于元数据的序列化、反序列化和持久化 + - 不涉及查询执行（ObjectQL的职责） + - 不涉及插件生命周期（Core的职责） + +2. **符合微内核原则** + - 作为可选的第三方包，不影响内核 + - 通过服务注册进行集成 + - 可以被其他包依赖和使用 + +3. **易于扩展** + - 添加新的序列化格式（XML, TOML）只需实现接口 + - 添加新的加载器（HTTP, S3, Database）只需添加插件 + - 支持自定义缓存策略 + +4. **向后兼容** + - 不破坏现有代码 + - SchemaRegistry可以继续使用 + - 提供桥接方案实现双向同步 + +#### 4.2 包的核心职责 + +```typescript +// 1. 元数据序列化/反序列化 +interface MetadataSerializer { + serialize(item: T, options?: SerializeOptions): string; + deserialize(content: string, schema?: ZodSchema): T; +} + +// 2. 文件系统操作 +interface MetadataLoader { + load(type: string, name: string): Promise; + save(type: string, name: string, data: T): Promise; + watch(type: string, callback: WatchCallback): void; +} + +// 3. 缓存管理 +interface MetadataCache { + get(key: string): Promise; + set(key: string, value: T, ttl?: number): Promise; + invalidate(key: string): Promise; +} + +// 4. 导入/导出 +interface MetadataExporter { + exportCollection(type: string, options: ExportOptions): Promise; + importCollection(path: string, options: ImportOptions): Promise; +} + +// 5. 验证 +interface MetadataValidator { + validate(data: T, schema: ZodSchema): ValidationResult; + migrate(data: any, fromVersion: string, toVersion: string): any; +} +``` + +#### 4.3 支持的功能 + +| 功能类别 | 具体功能 | +|---------|---------| +| **格式支持** | JSON, YAML, TypeScript, JavaScript | +| **加载策略** | 文件系统, HTTP, S3, 数据库（扩展） | +| **缓存策略** | ETag, 内存缓存, Redis（扩展） | +| **文件监听** | 开发模式热重载 | +| **验证** | Zod schema验证, 版本迁移 | +| **导入导出** | 批量操作, 冲突解决 | +| **注册表集成** | 与SchemaRegistry双向同步 | + +#### 4.4 依赖关系 + +``` +@objectstack/metadata +├── 直接依赖： +│ ├── @objectstack/spec # 使用Zod schemas验证 +│ ├── @objectstack/core # 日志、服务注册 +│ ├── @objectstack/types # 共享类型定义 +│ ├── glob # 文件匹配 +│ ├── js-yaml # YAML解析 +│ └── chokidar (可选) # 文件监听 +│ +└── 被以下包使用： + ├── @objectstack/cli # 代码生成、scaffolding + ├── @objectstack/runtime # 启动时加载清单 + ├── @objectstack/objectql # 注册表持久化 + └── 未来：@objectstack/hub # 云端元数据同步 +``` + +#### 4.5 包结构 + +``` +packages/metadata/ +├── src/ +│ ├── loaders/ +│ │ ├── filesystem-loader.ts # 从磁盘加载 +│ │ ├── http-loader.ts # 从HTTP/S3加载 +│ │ ├── loader-interface.ts # 加载器抽象接口 +│ │ └── composite-loader.ts # 组合多个加载器 +│ ├── serializers/ +│ │ ├── json-serializer.ts # JSON格式 +│ │ ├── yaml-serializer.ts # YAML格式 +│ │ ├── typescript-serializer.ts # TypeScript模块格式 +│ │ └── serializer-interface.ts # 序列化器抽象接口 +│ ├── validators/ +│ │ ├── schema-validator.ts # Zod验证 +│ │ ├── migration-validator.ts # 版本迁移 +│ │ └── validator-interface.ts # 验证器接口 +│ ├── cache/ +│ │ ├── etag-cache.ts # ETag缓存 +│ │ ├── memory-cache.ts # 内存缓存 +│ │ └── cache-interface.ts # 缓存接口 +│ ├── registry-bridge/ +│ │ ├── schema-registry-sync.ts # SchemaRegistry同步 +│ │ └── event-emitter.ts # 变更通知 +│ ├── exporters/ +│ │ ├── collection-exporter.ts # 批量导出 +│ │ └── selective-exporter.ts # 过滤导出 +│ ├── importers/ +│ │ ├── collection-importer.ts # 批量导入 +│ │ └── conflict-resolver.ts # 冲突解决 +│ ├── metadata-manager.ts # 主编排器 +│ └── index.ts +├── package.json +├── tsconfig.json +└── README.md +``` + +--- + +## 五、实施计划 (Implementation Plan) + +### 阶段1：基础设施 (第1周) +- [ ] 创建包结构和配置 +- [ ] 实现核心接口定义 +- [ ] 实现文件系统加载器 +- [ ] 实现JSON/YAML序列化器 +- [ ] 添加Zod验证集成 +- [ ] 编写单元测试 + +### 阶段2：高级功能 (第2周) +- [ ] 添加TypeScript/JavaScript序列化器 +- [ ] 实现ETag缓存机制 +- [ ] 添加文件监听（开发模式） +- [ ] 实现导出/导入工具 +- [ ] 集成测试 + +### 阶段3：集成 (第3周) +- [ ] 创建SchemaRegistry持久化桥接 +- [ ] 更新Runtime使用metadata manager +- [ ] 集成到CLI命令 +- [ ] 添加HTTP加载器（云同步） +- [ ] 端到端测试 + +### 阶段4：文档与优化 (第4周) +- [ ] 完整API文档 +- [ ] 使用指南和示例 +- [ ] 更新ARCHITECTURE.md +- [ ] 更新PACKAGE-DEPENDENCIES.md +- [ ] 性能基准测试和优化 + +--- + +## 六、使用示例 (Usage Examples) + +### 6.1 基本使用 + +```typescript +import { MetadataManager } from '@objectstack/metadata'; +import { ServiceObject } from '@objectstack/spec/data'; + +// 创建管理器 +const manager = new MetadataManager({ + rootDir: './metadata', + formats: ['typescript', 'json', 'yaml'], + cache: { enabled: true, ttl: 3600 }, + watch: process.env.NODE_ENV === 'development', +}); + +// 加载元数据 +const customer = await manager.load('object', 'customer'); + +// 保存元数据 +await manager.save('object', 'project', projectObject, { + format: 'typescript', + prettify: true, +}); + +// 批量加载 +const objects = await manager.loadCollection('object', { + patterns: ['**/*.object.ts', '**/*.object.json'], +}); +``` + +### 6.2 与SchemaRegistry集成 + +```typescript +import { SchemaRegistry } from '@objectstack/objectql'; +import { SchemaRegistryPersistence } from '@objectstack/metadata'; + +const persistence = new SchemaRegistryPersistence( + SchemaRegistry, + manager +); + +// 从磁盘加载所有对象到注册表 +await persistence.loadAll(); + +// 保存注册表到磁盘 +await persistence.saveAll(); + +// 启用自动同步（文件变更自动更新注册表） +persistence.enableAutoSync(); +``` + +### 6.3 CLI集成 + +```typescript +// CLI命令: objectstack generate object +import { MetadataManager } from '@objectstack/metadata'; + +export async function generateObject(name: string, options: any) { + const manager = new MetadataManager({ rootDir: './src/objects' }); + + const object: ServiceObject = { + name, + label: options.label || name, + fields: { + name: { type: 'text', label: 'Name' }, + }, + }; + + await manager.save('object', name, object, { + format: 'typescript', + }); + + console.log(`✓ Created ${name}.object.ts`); +} +``` + +--- + +## 七、影响分析 (Impact Analysis) + +### 7.1 积极影响 + +| 方面 | 影响 | +|------|------| +| **架构清晰度** | ↑↑ 职责明确，边界清晰 | +| **可维护性** | ↑↑ 独立包，易于维护 | +| **可扩展性** | ↑↑ 插件化设计，易于扩展 | +| **可测试性** | ↑↑ 独立测试，易于mock | +| **开发体验** | ↑ 统一API，减少学习成本 | +| **性能** | ↑ 缓存优化，减少I/O | + +### 7.2 潜在风险 + +| 风险 | 严重度 | 缓解措施 | +|------|--------|---------| +| **包数量增加** | 低 | 职责清晰可抵消复杂度 | +| **性能开销** | 低 | 实施缓存和懒加载 | +| **迁移成本** | 中 | 提供向后兼容和迁移指南 | +| **采用阻力** | 低 | 可选使用，渐进迁移 | + +### 7.3 向后兼容性 + +- ✅ **不破坏现有代码**：SchemaRegistry继续工作 +- ✅ **可选采用**：各包可以逐步迁移 +- ✅ **共存支持**：新旧方案可以并存 +- ✅ **渐进式迁移**：提供迁移路径和工具 + +--- + +## 八、协议定义 (Protocol Definition) + +已在 `@objectstack/spec` 中定义元数据加载器协议： + +**文件位置：** `packages/spec/src/system/metadata-loader.zod.ts` + +**主要协议：** +- `MetadataFormat` - 支持的格式枚举 +- `MetadataStats` - 元数据统计信息 +- `MetadataLoadOptions` - 加载选项 +- `MetadataSaveOptions` - 保存选项 +- `MetadataExportOptions` - 导出选项 +- `MetadataImportOptions` - 导入选项 +- `MetadataLoaderContract` - 加载器接口契约 +- `MetadataManagerConfig` - 管理器配置 + +所有协议使用Zod定义，支持： +- 运行时验证 +- TypeScript类型推导 +- JSON Schema生成 + +--- + +## 九、结论与建议 (Conclusion & Recommendations) + +### 9.1 核心结论 + +经过全面评估，**强烈建议创建独立的 `@objectstack/metadata` 包（Layer 3）**，原因如下： + +1. ✅ **符合微内核架构原则** - 清晰的职责分离 +2. ✅ **不破坏现有包的边界** - 各包保持单一职责 +3. ✅ **为未来扩展奠定基础** - 云同步、版本控制、多租户 +4. ✅ **提升开发体验** - 统一的API和清晰的文档 +5. ✅ **保持向后兼容** - 渐进式迁移，不破坏现有代码 + +### 9.2 立即行动建议 + +1. **审批方案** - 与团队讨论并确认架构决策 +2. **创建包结构** - 按照实施计划Phase 1开始 +3. **更新文档** - 确保架构文档同步更新 +4. **渐进实施** - 先实现核心功能，再扩展高级功能 + +### 9.3 长期价值 + +这个新包将成为ObjectStack元数据管理的基石，支持： +- 📦 **插件生态** - 统一的元数据加载标准 +- ☁️ **云端同步** - Hub/Composer集成 +- 🔄 **版本控制** - 元数据版本管理和迁移 +- 🏢 **多租户** - 租户级别的元数据隔离 +- 🔌 **扩展性** - 自定义加载器和序列化器 + +--- + +## 十、附录 (Appendix) + +### A. 相关文档链接 + +- [完整架构文档](content/docs/developers/metadata-management.mdx) +- [元数据加载器协议](packages/spec/src/system/metadata-loader.zod.ts) +- [协议测试](packages/spec/src/system/metadata-loader.test.ts) +- [ARCHITECTURE.md](ARCHITECTURE.md) +- [PACKAGE-DEPENDENCIES.md](PACKAGE-DEPENDENCIES.md) + +### B. 技术栈 + +| 技术 | 用途 | +|------|------| +| Zod | Schema定义和验证 | +| glob | 文件模式匹配 | +| js-yaml | YAML解析 | +| chokidar | 文件监听 | +| TypeScript | 类型系统 | + +### C. 参考架构 + +借鉴了以下优秀架构： +- **Kubernetes** - CRD (Custom Resource Definitions) +- **VS Code** - Extension API +- **OSGi** - 模块化和插件系统 +- **Eclipse** - Plugin Architecture + +--- + +**评估完成时间**: 2026-01-31 +**下次评审**: Phase 1 实施后 +**状态**: ✅ 建议已提交，等待审批