From fbfc5c4498490863f3263f9f980d610b83883dd7 Mon Sep 17 00:00:00 2001 From: indexzero Date: Sun, 1 Feb 2026 02:29:00 -0500 Subject: [PATCH 1/2] feat(view) add local directory origin adapter for views Add origin adapter abstraction to support querying packuments from different sources: - CacheAdapter: Wraps existing cache-based packument storage - LocalDirAdapter: Reads JSON files from local directory - createOriginAdapter: Factory auto-detects origin type Views can now be defined over local directories using: - Relative paths: ./local-data/ - Absolute paths: /data/npm-archive/ - file:// URLs: file:///path/to/dir/ This enables comparing npm cache against local snapshots, archived data, or exported packument sets. Co-Authored-By: Claude Opus 4.5 --- doc/cli-reference.md | 89 +++++++++++++ src/view/index.js | 6 + src/view/origin-adapter.js | 141 ++++++++++++++++++++ src/view/origin-adapter.test.js | 228 ++++++++++++++++++++++++++++++++ src/view/query.js | 19 +-- 5 files changed, 474 insertions(+), 9 deletions(-) create mode 100644 src/view/origin-adapter.js create mode 100644 src/view/origin-adapter.test.js diff --git a/doc/cli-reference.md b/doc/cli-reference.md index 35f945d..89ef742 100644 --- a/doc/cli-reference.md +++ b/doc/cli-reference.md @@ -616,6 +616,95 @@ npx _all_docs cache create-index > previous-index.txt --- +## view + +Commands for defining and querying views over cached data. + +Views can be created over two types of origins: +- **Registry cache**: Data fetched from npm or other registries +- **Local directory**: JSON packument files in a directory + +### view define + +Define a named view for querying packuments. + +```bash +npx _all_docs view define [options] +``` + +**Options:** +- `--origin ` - Data origin: encoded name (npm), URL, or local path +- `--registry ` - Registry URL (alternative to origin) +- `--select ` - Field selection expression +- `--type ` - Entity type: packument, partition (default: packument) +- `--force`, `-f` - Overwrite existing view definition + +**Origin Types:** + +| Type | Example | Description | +|------|---------|-------------| +| Encoded name | `npm` | Pre-defined registry origin | +| Registry URL | `https://npm.example.com` | Custom registry | +| Local path | `./local-data/` | Directory of JSON files | +| file:// URL | `file:///data/archive/` | Explicit file URL | + +**Examples:** + +```bash +# Define view over npm registry cache +npx _all_docs view define npm-pkgs --origin npm + +# Define view over local directory of packuments +npx _all_docs view define local-snapshot --origin ./local-packuments/ + +# Using file:// URL for local directory +npx _all_docs view define archive --origin file:///data/npm-archive/ +``` + +### view query + +Query a defined view and output results. + +```bash +npx _all_docs view query [options] +``` + +**Options:** +- `--limit ` - Maximum records to return +- `--filter ` - Filter expression +- `--count` - Only output the count of matching records + +### view join + +Join two views on a common key. + +```bash +npx _all_docs view join [options] +``` + +This enables comparing packages across different sources: + +```bash +# Compare npm cache against local snapshot +npx _all_docs view define npm --origin npm +npx _all_docs view define snapshot --origin ./snapshot/ +npx _all_docs view join npm snapshot --diff --select 'name' +``` + +### view list + +List all defined views. + +### view show + +Show details of a defined view. + +### view delete + +Delete a defined view. + +--- + ## Troubleshooting ### Common Issues diff --git a/src/view/index.js b/src/view/index.js index c306711..cce10d9 100644 --- a/src/view/index.js +++ b/src/view/index.js @@ -11,3 +11,9 @@ export { createProjection, createFilter } from './projection.js'; +export { + CacheAdapter, + LocalDirAdapter, + createOriginAdapter, + isLocalOrigin +} from './origin-adapter.js'; diff --git a/src/view/origin-adapter.js b/src/view/origin-adapter.js new file mode 100644 index 0000000..4fd8183 --- /dev/null +++ b/src/view/origin-adapter.js @@ -0,0 +1,141 @@ +/** + * Origin adapters for view queries + * + * Provides a unified interface for iterating packuments from different sources: + * - CacheAdapter: Reads from @_all_docs cache storage + * - LocalDirAdapter: Reads JSON files from local directory + */ +import { readdir, readFile } from 'node:fs/promises'; +import { join, basename } from 'node:path'; +import { existsSync } from 'node:fs'; + +/** + * Check if an origin string represents a local path + * @param {string} origin - Origin string (URL, encoded origin, or path) + * @returns {boolean} True if origin is a local path + */ +export function isLocalOrigin(origin) { + if (!origin) return false; + + // file:// URL + if (origin.startsWith('file://')) return true; + + // Absolute path (Unix) + if (origin.startsWith('/')) return true; + + // Relative path starting with ./ + if (origin.startsWith('./') || origin.startsWith('../')) return true; + + // Windows absolute path (C:\, D:\, etc.) + if (/^[A-Za-z]:[\\\/]/.test(origin)) return true; + + // Path that exists on disk + if (existsSync(origin)) return true; + + return false; +} + +/** + * Normalize origin to a filesystem path + * @param {string} origin - Origin string + * @returns {string} Filesystem path + */ +function normalizePath(origin) { + if (origin.startsWith('file://')) { + return origin.replace('file://', ''); + } + return origin; +} + +/** + * Cache adapter - reads from @_all_docs cache storage + */ +export class CacheAdapter { + /** + * @param {Cache} cache - Cache instance + * @param {string} keyPrefix - Cache key prefix for this view + */ + constructor(cache, keyPrefix) { + this.cache = cache; + this.keyPrefix = keyPrefix; + } + + /** + * Iterate keys matching the prefix + * @yields {string} Cache keys + */ + async *keys() { + yield* this.cache.keys(this.keyPrefix); + } + + /** + * Fetch a packument by cache key + * @param {string} key - Cache key + * @returns {Promise} Packument or null + */ + async fetch(key) { + try { + const entry = await this.cache.fetch(key); + if (!entry) return null; + return entry.body || entry; + } catch { + return null; + } + } +} + +/** + * Local directory adapter - reads JSON files from a directory + */ +export class LocalDirAdapter { + /** + * @param {string} dirPath - Path to directory containing packument JSON files + */ + constructor(dirPath) { + this.dirPath = normalizePath(dirPath); + } + + /** + * Iterate JSON files in the directory + * @yields {string} Filenames (used as keys) + */ + async *keys() { + const files = await readdir(this.dirPath); + for (const file of files) { + if (file.endsWith('.json')) { + yield file; + } + } + } + + /** + * Fetch a packument by filename + * @param {string} key - Filename + * @returns {Promise} Packument or null + */ + async fetch(key) { + try { + const filePath = join(this.dirPath, key); + const content = await readFile(filePath, 'utf8'); + return JSON.parse(content); + } catch { + return null; + } + } +} + +/** + * Create an appropriate origin adapter based on view configuration + * @param {View} view - The view definition + * @param {Cache} cache - Cache instance (used for cache-based origins) + * @returns {CacheAdapter|LocalDirAdapter} Origin adapter + */ +export function createOriginAdapter(view, cache) { + const origin = view.registry || view.origin; + + if (isLocalOrigin(origin)) { + return new LocalDirAdapter(origin); + } + + return new CacheAdapter(cache, view.getCacheKeyPrefix()); +} diff --git a/src/view/origin-adapter.test.js b/src/view/origin-adapter.test.js new file mode 100644 index 0000000..3b0402d --- /dev/null +++ b/src/view/origin-adapter.test.js @@ -0,0 +1,228 @@ +import { describe, it, beforeEach, afterEach } from 'node:test'; +import { strict as assert } from 'node:assert'; +import { mkdirSync, writeFileSync, rmSync } from 'node:fs'; +import { join } from 'node:path'; +import { + isLocalOrigin, + LocalDirAdapter, + CacheAdapter, + createOriginAdapter +} from './origin-adapter.js'; + +describe('isLocalOrigin', () => { + it('returns true for file:// URLs', () => { + assert.equal(isLocalOrigin('file:///path/to/dir'), true); + assert.equal(isLocalOrigin('file://./relative'), true); + }); + + it('returns true for absolute Unix paths', () => { + assert.equal(isLocalOrigin('/usr/local/data'), true); + assert.equal(isLocalOrigin('/tmp/test'), true); + }); + + it('returns true for relative paths with ./', () => { + assert.equal(isLocalOrigin('./data'), true); + assert.equal(isLocalOrigin('../parent/data'), true); + }); + + it('returns false for registry URLs', () => { + assert.equal(isLocalOrigin('https://registry.npmjs.org'), false); + assert.equal(isLocalOrigin('npm'), false); + }); + + it('returns false for encoded origins', () => { + assert.equal(isLocalOrigin('npm.exale.com'), false); + assert.equal(isLocalOrigin('regiry.npmjs.org'), false); + }); + + it('returns false for null/undefined', () => { + assert.equal(isLocalOrigin(null), false); + assert.equal(isLocalOrigin(undefined), false); + }); +}); + +describe('LocalDirAdapter', () => { + const testDir = join(import.meta.dirname, 'test-origin-fixtures'); + + beforeEach(() => { + mkdirSync(testDir, { recursive: true }); + }); + + afterEach(() => { + rmSync(testDir, { recursive: true, force: true }); + }); + + it('lists only JSON files as keys', async () => { + // Create test files + writeFileSync(join(testDir, 'lodash.json'), '{"name":"lodash"}'); + writeFileSync(join(testDir, 'express.json'), '{"name":"express"}'); + writeFileSync(join(testDir, 'README.md'), '# test'); + writeFileSync(join(testDir, 'config.txt'), 'config'); + + const adapter = new LocalDirAdapter(testDir); + const keys = []; + + for await (const key of adapter.keys()) { + keys.push(key); + } + + assert.equal(keys.length, 2); + assert.ok(keys.includes('lodash.json')); + assert.ok(keys.includes('express.json')); + assert.ok(!keys.includes('README.md')); + assert.ok(!keys.includes('config.txt')); + }); + + it('fetches and parses JSON files', async () => { + writeFileSync( + join(testDir, 'react.json'), + JSON.stringify({ name: 'react', version: '18.2.0' }) + ); + + const adapter = new LocalDirAdapter(testDir); + const result = await adapter.fetch('react.json'); + + assert.deepEqual(result, { name: 'react', version: '18.2.0' }); + }); + + it('returns null for missing files', async () => { + const adapter = new LocalDirAdapter(testDir); + const result = await adapter.fetch('nonexistent.json'); + + assert.equal(result, null); + }); + + it('returns null for invalid JSON', async () => { + writeFileSync(join(testDir, 'invalid.json'), 'not valid json{'); + + const adapter = new LocalDirAdapter(testDir); + const result = await adapter.fetch('invalid.json'); + + assert.equal(result, null); + }); + + it('handles file:// URL paths', async () => { + writeFileSync(join(testDir, 'test.json'), '{"name":"test"}'); + + const adapter = new LocalDirAdapter(`file://${testDir}`); + const result = await adapter.fetch('test.json'); + + assert.deepEqual(result, { name: 'test' }); + }); +}); + +describe('CacheAdapter', () => { + it('delegates keys() to cache.keys()', async () => { + const mockKeys = ['v1:packument:npm:a', 'v1:packument:npm:b']; + const mockCache = { + async *keys(prefix) { + for (const k of mockKeys.filter(k => k.startsWith(prefix))) { + yield k; + } + }, + async fetch() { return null; } + }; + + const adapter = new CacheAdapter(mockCache, 'v1:packument:npm:'); + const keys = []; + + for await (const key of adapter.keys()) { + keys.push(key); + } + + assert.deepEqual(keys, mockKeys); + }); + + it('extracts body from cache entry', async () => { + const mockCache = { + async *keys() {}, + async fetch(key) { + return { body: { name: 'lodash' }, meta: 'ignored' }; + } + }; + + const adapter = new CacheAdapter(mockCache, 'v1:'); + const result = await adapter.fetch('v1:packument:npm:abc'); + + assert.deepEqual(result, { name: 'lodash' }); + }); + + it('returns null on fetch error', async () => { + const mockCache = { + async *keys() {}, + async fetch() { + throw new Error('Not found'); + } + }; + + const adapter = new CacheAdapter(mockCache, 'v1:'); + const result = await adapter.fetch('nonexistent'); + + assert.equal(result, null); + }); +}); + +describe('createOriginAdapter', () => { + it('creates LocalDirAdapter for file:// origin', () => { + const view = { + registry: 'file:///path/to/dir', + origin: 'ignored', + getCacheKeyPrefix: () => 'v1:packument:local:' + }; + + const adapter = createOriginAdapter(view, null); + + assert.ok(adapter instanceof LocalDirAdapter); + }); + + it('creates LocalDirAdapter for absolute path', () => { + const view = { + registry: '/absolute/path', + origin: 'ignored', + getCacheKeyPrefix: () => 'v1:packument:local:' + }; + + const adapter = createOriginAdapter(view, null); + + assert.ok(adapter instanceof LocalDirAdapter); + }); + + it('creates LocalDirAdapter for relative path origin', () => { + const view = { + registry: null, + origin: './relative/path', + getCacheKeyPrefix: () => 'v1:packument:local:' + }; + + const adapter = createOriginAdapter(view, null); + + assert.ok(adapter instanceof LocalDirAdapter); + }); + + it('creates CacheAdapter for registry URL', () => { + const mockCache = {}; + const view = { + registry: 'https://registry.npmjs.org', + origin: 'npm', + getCacheKeyPrefix: () => 'v1:packument:npm:' + }; + + const adapter = createOriginAdapter(view, mockCache); + + assert.ok(adapter instanceof CacheAdapter); + assert.equal(adapter.keyPrefix, 'v1:packument:npm:'); + }); + + it('creates CacheAdapter for encoded origin', () => { + const mockCache = {}; + const view = { + registry: null, + origin: 'npm', + getCacheKeyPrefix: () => 'v1:packument:npm:' + }; + + const adapter = createOriginAdapter(view, mockCache); + + assert.ok(adapter instanceof CacheAdapter); + }); +}); diff --git a/src/view/query.js b/src/view/query.js index 441b253..164067c 100644 --- a/src/view/query.js +++ b/src/view/query.js @@ -2,19 +2,21 @@ * Query execution for views */ import { createProjection, createFilter } from './projection.js'; +import { createOriginAdapter } from './origin-adapter.js'; /** * Query a view, yielding projected records * @param {View} view - The view to query - * @param {Cache} cache - The cache instance + * @param {Cache} cache - The cache instance (optional for local origins) * @param {Object} options - Query options * @param {number} [options.limit] - Maximum records to return * @param {string} [options.where] - Additional filter expression * @param {boolean} [options.progress] - Show progress on stderr + * @param {object} [options.adapter] - Custom origin adapter (auto-detected if not provided) * @yields {Object} Projected records */ export async function* queryView(view, cache, options = {}) { - const { limit, where, progress = false } = options; + const { limit, where, progress = false, adapter: providedAdapter } = options; // Compile projection from view's select const project = createProjection({ select: view.select }); @@ -22,11 +24,13 @@ export async function* queryView(view, cache, options = {}) { // Compile additional filter if provided const filter = createFilter({ where }); - const prefix = view.getCacheKeyPrefix(); + // Get or create the origin adapter + const adapter = providedAdapter || createOriginAdapter(view, cache); + let count = 0; let yielded = 0; - for await (const key of cache.keys(prefix)) { + for await (const key of adapter.keys()) { // Check limit if (limit && yielded >= limit) break; @@ -38,11 +42,8 @@ export async function* queryView(view, cache, options = {}) { } try { - const entry = await cache.fetch(key); - if (!entry) continue; - - // Cache entries wrap the response - packument is in body - const value = entry.body || entry; + const value = await adapter.fetch(key); + if (!value) continue; // Apply view's projection const projected = project(value); From dbdb12623168e1599129ce1ec067ed243052e1b4 Mon Sep 17 00:00:00 2001 From: indexzero Date: Sun, 1 Feb 2026 23:00:09 -0500 Subject: [PATCH 2/2] refactor(cache) move local directory adapter to storage driver layer LocalDirAdapter in src/view was reimplementing the storage driver interface with different method names (keys/fetch vs list/get). Moving it to src/cache as LocalDirStorageDriver: - Implements proper storage driver interface (list, get, has) - Read-only stubs for put, delete, clear - createStorageDriver({ LOCAL_DIR: path }) creates the driver - isLocalPath() exported for origin detection - View query.js simplified to use cache directly - CLI detects local paths and creates appropriate driver This enables mounting local packument directories as virtual caches for any cache consumer, not just views. Co-Authored-By: Claude Opus 4.5 --- cli/cli/src/cmd/view/query.js | 10 +- src/cache/index.js | 2 +- src/cache/local-dir-driver.js | 159 +++++++++++++++++ src/cache/storage-driver.js | 12 ++ src/cache/test/local-dir-driver.test.js | 216 ++++++++++++++++++++++ src/view/index.js | 9 +- src/view/origin-adapter.js | 141 --------------- src/view/origin-adapter.test.js | 228 ------------------------ src/view/query.js | 24 ++- 9 files changed, 413 insertions(+), 388 deletions(-) create mode 100644 src/cache/local-dir-driver.js create mode 100644 src/cache/test/local-dir-driver.test.js delete mode 100644 src/view/origin-adapter.js delete mode 100644 src/view/origin-adapter.test.js diff --git a/cli/cli/src/cmd/view/query.js b/cli/cli/src/cmd/view/query.js index 583ff91..e5a803f 100644 --- a/cli/cli/src/cmd/view/query.js +++ b/cli/cli/src/cmd/view/query.js @@ -1,5 +1,5 @@ import { ViewStore, queryView, countView, collectView } from '@_all_docs/view'; -import { Cache, createStorageDriver } from '@_all_docs/cache'; +import { Cache, createStorageDriver, isLocalPath } from '@_all_docs/cache'; export const usage = `Usage: _all_docs view query [options] @@ -43,8 +43,12 @@ export const command = async (cli) => { process.exit(1); } - const driver = await createStorageDriver({ CACHE_DIR: cli.dir('packuments') }); - const cache = new Cache({ path: cli.dir('packuments'), driver }); + // Create appropriate storage driver based on view's origin + const origin = view.registry || view.origin; + const driver = isLocalPath(origin) + ? await createStorageDriver({ LOCAL_DIR: origin }) + : await createStorageDriver({ CACHE_DIR: cli.dir('packuments') }); + const cache = new Cache({ path: origin, driver }); const options = { limit: cli.values.limit ? parseInt(cli.values.limit, 10) : undefined, diff --git a/src/cache/index.js b/src/cache/index.js index 5c7e66b..ff30158 100644 --- a/src/cache/index.js +++ b/src/cache/index.js @@ -3,5 +3,5 @@ export { BaseHTTPClient, createAgent, createDispatcher } from './http.js'; export { CacheEntry } from './entry.js'; export { createCacheKey, decodeCacheKey, createPartitionKey, createPackumentKey, encodeOrigin } from './cache-key.js'; export { PartitionCheckpoint } from './checkpoint.js'; -export { createStorageDriver } from './storage-driver.js'; +export { createStorageDriver, LocalDirStorageDriver, isLocalPath } from './storage-driver.js'; export { AuthError, TempError, PermError, categorizeHttpError } from './errors.js'; \ No newline at end of file diff --git a/src/cache/local-dir-driver.js b/src/cache/local-dir-driver.js new file mode 100644 index 0000000..fb84ab2 --- /dev/null +++ b/src/cache/local-dir-driver.js @@ -0,0 +1,159 @@ +/** + * Local directory storage driver - reads packument JSON files from a directory + * + * This is a read-only storage driver that allows mounting existing directories + * of packument JSON files as a virtual cache. Useful for analyzing local datasets + * without importing them into the cache. + */ +import { readdir, readFile, access } from 'node:fs/promises'; +import { join } from 'node:path'; +import { existsSync } from 'node:fs'; + +/** + * Check if an origin string represents a local path + * @param {string} origin - Origin string (URL, encoded origin, or path) + * @returns {boolean} True if origin is a local path + */ +export function isLocalPath(origin) { + if (!origin) return false; + + // file:// URL + if (origin.startsWith('file://')) return true; + + // Absolute path (Unix) + if (origin.startsWith('/')) return true; + + // Relative path starting with ./ + if (origin.startsWith('./') || origin.startsWith('../')) return true; + + // Windows absolute path (C:\, D:\, etc.) + if (/^[A-Za-z]:[\\\/]/.test(origin)) return true; + + // Path that exists on disk (fallback check) + if (existsSync(origin)) return true; + + return false; +} + +/** + * Normalize origin to a filesystem path + * @param {string} origin - Origin string + * @returns {string} Filesystem path + */ +function normalizePath(origin) { + if (origin.startsWith('file://')) { + return origin.replace('file://', ''); + } + return origin; +} + +/** + * Read-only storage driver for local directories of JSON files + */ +export class LocalDirStorageDriver { + /** + * @param {string} dirPath - Path to directory containing packument JSON files + */ + constructor(dirPath) { + this.dirPath = normalizePath(dirPath); + this.supportsBatch = false; + this.supportsBloom = false; + } + + /** + * Get a packument by key (filename) + * @param {string} key - Filename (with or without .json extension) + * @returns {Promise} Parsed JSON content + * @throws {Error} If file not found or invalid JSON + */ + async get(key) { + const filename = key.endsWith('.json') ? key : `${key}.json`; + const filePath = join(this.dirPath, filename); + + try { + const content = await readFile(filePath, 'utf8'); + return JSON.parse(content); + } catch (error) { + if (error.code === 'ENOENT') { + throw new Error(`Key not found: ${key}`); + } + throw error; + } + } + + /** + * Check if a key exists + * @param {string} key - Filename + * @returns {Promise} + */ + async has(key) { + const filename = key.endsWith('.json') ? key : `${key}.json`; + const filePath = join(this.dirPath, filename); + + try { + await access(filePath); + return true; + } catch { + return false; + } + } + + /** + * List all JSON files in the directory + * Note: prefix is ignored for local directories since the directory path + * itself serves as the namespace isolation. + * @param {string} [_prefix] - Ignored for local directories + * @yields {string} Filenames + */ + async *list(_prefix) { + const files = await readdir(this.dirPath); + for (const file of files) { + if (file.endsWith('.json')) { + yield file; + } + } + } + + /** + * Put is not supported - this is a read-only driver + * @throws {Error} Always throws + */ + async put(_key, _value) { + throw new Error('LocalDirStorageDriver is read-only'); + } + + /** + * Delete is not supported - this is a read-only driver + * @throws {Error} Always throws + */ + async delete(_key) { + throw new Error('LocalDirStorageDriver is read-only'); + } + + /** + * Clear is not supported - this is a read-only driver + * @throws {Error} Always throws + */ + async clear() { + throw new Error('LocalDirStorageDriver is read-only'); + } + + /** + * Batch put is not supported - this is a read-only driver + * @throws {Error} Always throws + */ + async putBatch(_entries) { + throw new Error('LocalDirStorageDriver is read-only'); + } + + /** + * Get metadata info for a file (basic implementation) + * @param {string} key - Filename + * @returns {Promise} Basic info or null if not found + */ + async info(key) { + const exists = await this.has(key); + if (!exists) return null; + return { key, path: join(this.dirPath, key) }; + } +} diff --git a/src/cache/storage-driver.js b/src/cache/storage-driver.js index 296e1cb..cf48131 100644 --- a/src/cache/storage-driver.js +++ b/src/cache/storage-driver.js @@ -1,9 +1,21 @@ +import { LocalDirStorageDriver, isLocalPath } from './local-dir-driver.js'; + +export { LocalDirStorageDriver, isLocalPath }; + /** * Creates a storage driver based on the runtime environment * @param {Object} env - Environment configuration + * @param {string} [env.LOCAL_DIR] - Local directory path (read-only driver) + * @param {string} [env.CACHE_DIR] - Cache directory path + * @param {string} [env.RUNTIME] - Runtime environment (node, cloudflare, fastly, cloudrun) * @returns {Object} Storage driver instance */ export async function createStorageDriver(env) { + // Local directory takes precedence - it's a read-only mount + if (env?.LOCAL_DIR) { + return new LocalDirStorageDriver(env.LOCAL_DIR); + } + const runtime = env?.RUNTIME || 'node'; switch (runtime) { diff --git a/src/cache/test/local-dir-driver.test.js b/src/cache/test/local-dir-driver.test.js new file mode 100644 index 0000000..18501fd --- /dev/null +++ b/src/cache/test/local-dir-driver.test.js @@ -0,0 +1,216 @@ +import { describe, it, beforeEach, afterEach } from 'node:test'; +import assert from 'node:assert'; +import { mkdtemp, writeFile, rm, mkdir } from 'node:fs/promises'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { LocalDirStorageDriver, isLocalPath } from '../local-dir-driver.js'; + +describe('LocalDirStorageDriver', () => { + let tempDir; + let driver; + + beforeEach(async () => { + tempDir = await mkdtemp(join(tmpdir(), 'local-dir-driver-test-')); + driver = new LocalDirStorageDriver(tempDir); + }); + + afterEach(async () => { + await rm(tempDir, { recursive: true, force: true }); + }); + + describe('list()', () => { + it('should list JSON files in directory', async () => { + await writeFile(join(tempDir, 'lodash.json'), JSON.stringify({ name: 'lodash' })); + await writeFile(join(tempDir, 'react.json'), JSON.stringify({ name: 'react' })); + await writeFile(join(tempDir, 'readme.md'), 'not json'); + + const files = []; + for await (const file of driver.list()) { + files.push(file); + } + + assert.strictEqual(files.length, 2); + assert.ok(files.includes('lodash.json')); + assert.ok(files.includes('react.json')); + assert.ok(!files.includes('readme.md')); + }); + + it('should ignore prefix parameter', async () => { + await writeFile(join(tempDir, 'lodash.json'), JSON.stringify({ name: 'lodash' })); + + const files = []; + for await (const file of driver.list('v1:packument:npm:')) { + files.push(file); + } + + assert.strictEqual(files.length, 1); + assert.strictEqual(files[0], 'lodash.json'); + }); + + it('should handle empty directory', async () => { + const files = []; + for await (const file of driver.list()) { + files.push(file); + } + + assert.strictEqual(files.length, 0); + }); + }); + + describe('get()', () => { + it('should read and parse JSON file', async () => { + const packument = { name: 'lodash', version: '4.17.21' }; + await writeFile(join(tempDir, 'lodash.json'), JSON.stringify(packument)); + + const result = await driver.get('lodash.json'); + assert.deepStrictEqual(result, packument); + }); + + it('should add .json extension if missing', async () => { + const packument = { name: 'lodash', version: '4.17.21' }; + await writeFile(join(tempDir, 'lodash.json'), JSON.stringify(packument)); + + const result = await driver.get('lodash'); + assert.deepStrictEqual(result, packument); + }); + + it('should throw on missing file', async () => { + await assert.rejects( + driver.get('nonexistent.json'), + /Key not found: nonexistent\.json/ + ); + }); + + it('should throw on invalid JSON', async () => { + await writeFile(join(tempDir, 'invalid.json'), 'not valid json'); + + await assert.rejects( + driver.get('invalid.json'), + /Unexpected token/ + ); + }); + }); + + describe('has()', () => { + it('should return true for existing file', async () => { + await writeFile(join(tempDir, 'lodash.json'), JSON.stringify({ name: 'lodash' })); + + const exists = await driver.has('lodash.json'); + assert.strictEqual(exists, true); + }); + + it('should return true with implicit .json extension', async () => { + await writeFile(join(tempDir, 'lodash.json'), JSON.stringify({ name: 'lodash' })); + + const exists = await driver.has('lodash'); + assert.strictEqual(exists, true); + }); + + it('should return false for missing file', async () => { + const exists = await driver.has('nonexistent.json'); + assert.strictEqual(exists, false); + }); + }); + + describe('read-only methods', () => { + it('should throw on put()', async () => { + await assert.rejects( + driver.put('key', { value: 'data' }), + /LocalDirStorageDriver is read-only/ + ); + }); + + it('should throw on delete()', async () => { + await assert.rejects( + driver.delete('key'), + /LocalDirStorageDriver is read-only/ + ); + }); + + it('should throw on clear()', async () => { + await assert.rejects( + driver.clear(), + /LocalDirStorageDriver is read-only/ + ); + }); + + it('should throw on putBatch()', async () => { + await assert.rejects( + driver.putBatch([{ key: 'k', value: 'v' }]), + /LocalDirStorageDriver is read-only/ + ); + }); + }); + + describe('info()', () => { + it('should return info for existing file', async () => { + await writeFile(join(tempDir, 'lodash.json'), JSON.stringify({ name: 'lodash' })); + + const info = await driver.info('lodash.json'); + assert.ok(info); + assert.strictEqual(info.key, 'lodash.json'); + assert.ok(info.path.includes('lodash.json')); + }); + + it('should return null for missing file', async () => { + const info = await driver.info('nonexistent.json'); + assert.strictEqual(info, null); + }); + }); + + describe('file:// URL support', () => { + it('should handle file:// URL paths', async () => { + const fileUrlPath = `file://${tempDir}`; + const urlDriver = new LocalDirStorageDriver(fileUrlPath); + + await writeFile(join(tempDir, 'test.json'), JSON.stringify({ name: 'test' })); + + const files = []; + for await (const file of urlDriver.list()) { + files.push(file); + } + + assert.strictEqual(files.length, 1); + assert.strictEqual(files[0], 'test.json'); + }); + }); +}); + +describe('isLocalPath', () => { + it('should detect file:// URLs', () => { + assert.strictEqual(isLocalPath('file:///path/to/dir'), true); + }); + + it('should detect absolute Unix paths', () => { + assert.strictEqual(isLocalPath('/path/to/dir'), true); + }); + + it('should detect relative paths with ./', () => { + assert.strictEqual(isLocalPath('./path/to/dir'), true); + }); + + it('should detect relative paths with ../', () => { + assert.strictEqual(isLocalPath('../path/to/dir'), true); + }); + + it('should detect Windows absolute paths', () => { + assert.strictEqual(isLocalPath('C:\\path\\to\\dir'), true); + assert.strictEqual(isLocalPath('D:/path/to/dir'), true); + }); + + it('should reject registry URLs', () => { + assert.strictEqual(isLocalPath('https://registry.npmjs.org'), false); + assert.strictEqual(isLocalPath('http://localhost:4873'), false); + }); + + it('should reject encoded origins', () => { + assert.strictEqual(isLocalPath('npm'), false); + assert.strictEqual(isLocalPath('registry.npmjs.org'), false); + }); + + it('should handle null/undefined', () => { + assert.strictEqual(isLocalPath(null), false); + assert.strictEqual(isLocalPath(undefined), false); + assert.strictEqual(isLocalPath(''), false); + }); +}); diff --git a/src/view/index.js b/src/view/index.js index cce10d9..d45e197 100644 --- a/src/view/index.js +++ b/src/view/index.js @@ -1,5 +1,8 @@ /** * View module - predicate + projection over cached data + * + * For local directory support, use createStorageDriver({ LOCAL_DIR: path }) + * from @_all_docs/cache when creating the cache instance. */ export { View } from './view.js'; export { ViewStore } from './store.js'; @@ -11,9 +14,3 @@ export { createProjection, createFilter } from './projection.js'; -export { - CacheAdapter, - LocalDirAdapter, - createOriginAdapter, - isLocalOrigin -} from './origin-adapter.js'; diff --git a/src/view/origin-adapter.js b/src/view/origin-adapter.js deleted file mode 100644 index 4fd8183..0000000 --- a/src/view/origin-adapter.js +++ /dev/null @@ -1,141 +0,0 @@ -/** - * Origin adapters for view queries - * - * Provides a unified interface for iterating packuments from different sources: - * - CacheAdapter: Reads from @_all_docs cache storage - * - LocalDirAdapter: Reads JSON files from local directory - */ -import { readdir, readFile } from 'node:fs/promises'; -import { join, basename } from 'node:path'; -import { existsSync } from 'node:fs'; - -/** - * Check if an origin string represents a local path - * @param {string} origin - Origin string (URL, encoded origin, or path) - * @returns {boolean} True if origin is a local path - */ -export function isLocalOrigin(origin) { - if (!origin) return false; - - // file:// URL - if (origin.startsWith('file://')) return true; - - // Absolute path (Unix) - if (origin.startsWith('/')) return true; - - // Relative path starting with ./ - if (origin.startsWith('./') || origin.startsWith('../')) return true; - - // Windows absolute path (C:\, D:\, etc.) - if (/^[A-Za-z]:[\\\/]/.test(origin)) return true; - - // Path that exists on disk - if (existsSync(origin)) return true; - - return false; -} - -/** - * Normalize origin to a filesystem path - * @param {string} origin - Origin string - * @returns {string} Filesystem path - */ -function normalizePath(origin) { - if (origin.startsWith('file://')) { - return origin.replace('file://', ''); - } - return origin; -} - -/** - * Cache adapter - reads from @_all_docs cache storage - */ -export class CacheAdapter { - /** - * @param {Cache} cache - Cache instance - * @param {string} keyPrefix - Cache key prefix for this view - */ - constructor(cache, keyPrefix) { - this.cache = cache; - this.keyPrefix = keyPrefix; - } - - /** - * Iterate keys matching the prefix - * @yields {string} Cache keys - */ - async *keys() { - yield* this.cache.keys(this.keyPrefix); - } - - /** - * Fetch a packument by cache key - * @param {string} key - Cache key - * @returns {Promise} Packument or null - */ - async fetch(key) { - try { - const entry = await this.cache.fetch(key); - if (!entry) return null; - return entry.body || entry; - } catch { - return null; - } - } -} - -/** - * Local directory adapter - reads JSON files from a directory - */ -export class LocalDirAdapter { - /** - * @param {string} dirPath - Path to directory containing packument JSON files - */ - constructor(dirPath) { - this.dirPath = normalizePath(dirPath); - } - - /** - * Iterate JSON files in the directory - * @yields {string} Filenames (used as keys) - */ - async *keys() { - const files = await readdir(this.dirPath); - for (const file of files) { - if (file.endsWith('.json')) { - yield file; - } - } - } - - /** - * Fetch a packument by filename - * @param {string} key - Filename - * @returns {Promise} Packument or null - */ - async fetch(key) { - try { - const filePath = join(this.dirPath, key); - const content = await readFile(filePath, 'utf8'); - return JSON.parse(content); - } catch { - return null; - } - } -} - -/** - * Create an appropriate origin adapter based on view configuration - * @param {View} view - The view definition - * @param {Cache} cache - Cache instance (used for cache-based origins) - * @returns {CacheAdapter|LocalDirAdapter} Origin adapter - */ -export function createOriginAdapter(view, cache) { - const origin = view.registry || view.origin; - - if (isLocalOrigin(origin)) { - return new LocalDirAdapter(origin); - } - - return new CacheAdapter(cache, view.getCacheKeyPrefix()); -} diff --git a/src/view/origin-adapter.test.js b/src/view/origin-adapter.test.js deleted file mode 100644 index 3b0402d..0000000 --- a/src/view/origin-adapter.test.js +++ /dev/null @@ -1,228 +0,0 @@ -import { describe, it, beforeEach, afterEach } from 'node:test'; -import { strict as assert } from 'node:assert'; -import { mkdirSync, writeFileSync, rmSync } from 'node:fs'; -import { join } from 'node:path'; -import { - isLocalOrigin, - LocalDirAdapter, - CacheAdapter, - createOriginAdapter -} from './origin-adapter.js'; - -describe('isLocalOrigin', () => { - it('returns true for file:// URLs', () => { - assert.equal(isLocalOrigin('file:///path/to/dir'), true); - assert.equal(isLocalOrigin('file://./relative'), true); - }); - - it('returns true for absolute Unix paths', () => { - assert.equal(isLocalOrigin('/usr/local/data'), true); - assert.equal(isLocalOrigin('/tmp/test'), true); - }); - - it('returns true for relative paths with ./', () => { - assert.equal(isLocalOrigin('./data'), true); - assert.equal(isLocalOrigin('../parent/data'), true); - }); - - it('returns false for registry URLs', () => { - assert.equal(isLocalOrigin('https://registry.npmjs.org'), false); - assert.equal(isLocalOrigin('npm'), false); - }); - - it('returns false for encoded origins', () => { - assert.equal(isLocalOrigin('npm.exale.com'), false); - assert.equal(isLocalOrigin('regiry.npmjs.org'), false); - }); - - it('returns false for null/undefined', () => { - assert.equal(isLocalOrigin(null), false); - assert.equal(isLocalOrigin(undefined), false); - }); -}); - -describe('LocalDirAdapter', () => { - const testDir = join(import.meta.dirname, 'test-origin-fixtures'); - - beforeEach(() => { - mkdirSync(testDir, { recursive: true }); - }); - - afterEach(() => { - rmSync(testDir, { recursive: true, force: true }); - }); - - it('lists only JSON files as keys', async () => { - // Create test files - writeFileSync(join(testDir, 'lodash.json'), '{"name":"lodash"}'); - writeFileSync(join(testDir, 'express.json'), '{"name":"express"}'); - writeFileSync(join(testDir, 'README.md'), '# test'); - writeFileSync(join(testDir, 'config.txt'), 'config'); - - const adapter = new LocalDirAdapter(testDir); - const keys = []; - - for await (const key of adapter.keys()) { - keys.push(key); - } - - assert.equal(keys.length, 2); - assert.ok(keys.includes('lodash.json')); - assert.ok(keys.includes('express.json')); - assert.ok(!keys.includes('README.md')); - assert.ok(!keys.includes('config.txt')); - }); - - it('fetches and parses JSON files', async () => { - writeFileSync( - join(testDir, 'react.json'), - JSON.stringify({ name: 'react', version: '18.2.0' }) - ); - - const adapter = new LocalDirAdapter(testDir); - const result = await adapter.fetch('react.json'); - - assert.deepEqual(result, { name: 'react', version: '18.2.0' }); - }); - - it('returns null for missing files', async () => { - const adapter = new LocalDirAdapter(testDir); - const result = await adapter.fetch('nonexistent.json'); - - assert.equal(result, null); - }); - - it('returns null for invalid JSON', async () => { - writeFileSync(join(testDir, 'invalid.json'), 'not valid json{'); - - const adapter = new LocalDirAdapter(testDir); - const result = await adapter.fetch('invalid.json'); - - assert.equal(result, null); - }); - - it('handles file:// URL paths', async () => { - writeFileSync(join(testDir, 'test.json'), '{"name":"test"}'); - - const adapter = new LocalDirAdapter(`file://${testDir}`); - const result = await adapter.fetch('test.json'); - - assert.deepEqual(result, { name: 'test' }); - }); -}); - -describe('CacheAdapter', () => { - it('delegates keys() to cache.keys()', async () => { - const mockKeys = ['v1:packument:npm:a', 'v1:packument:npm:b']; - const mockCache = { - async *keys(prefix) { - for (const k of mockKeys.filter(k => k.startsWith(prefix))) { - yield k; - } - }, - async fetch() { return null; } - }; - - const adapter = new CacheAdapter(mockCache, 'v1:packument:npm:'); - const keys = []; - - for await (const key of adapter.keys()) { - keys.push(key); - } - - assert.deepEqual(keys, mockKeys); - }); - - it('extracts body from cache entry', async () => { - const mockCache = { - async *keys() {}, - async fetch(key) { - return { body: { name: 'lodash' }, meta: 'ignored' }; - } - }; - - const adapter = new CacheAdapter(mockCache, 'v1:'); - const result = await adapter.fetch('v1:packument:npm:abc'); - - assert.deepEqual(result, { name: 'lodash' }); - }); - - it('returns null on fetch error', async () => { - const mockCache = { - async *keys() {}, - async fetch() { - throw new Error('Not found'); - } - }; - - const adapter = new CacheAdapter(mockCache, 'v1:'); - const result = await adapter.fetch('nonexistent'); - - assert.equal(result, null); - }); -}); - -describe('createOriginAdapter', () => { - it('creates LocalDirAdapter for file:// origin', () => { - const view = { - registry: 'file:///path/to/dir', - origin: 'ignored', - getCacheKeyPrefix: () => 'v1:packument:local:' - }; - - const adapter = createOriginAdapter(view, null); - - assert.ok(adapter instanceof LocalDirAdapter); - }); - - it('creates LocalDirAdapter for absolute path', () => { - const view = { - registry: '/absolute/path', - origin: 'ignored', - getCacheKeyPrefix: () => 'v1:packument:local:' - }; - - const adapter = createOriginAdapter(view, null); - - assert.ok(adapter instanceof LocalDirAdapter); - }); - - it('creates LocalDirAdapter for relative path origin', () => { - const view = { - registry: null, - origin: './relative/path', - getCacheKeyPrefix: () => 'v1:packument:local:' - }; - - const adapter = createOriginAdapter(view, null); - - assert.ok(adapter instanceof LocalDirAdapter); - }); - - it('creates CacheAdapter for registry URL', () => { - const mockCache = {}; - const view = { - registry: 'https://registry.npmjs.org', - origin: 'npm', - getCacheKeyPrefix: () => 'v1:packument:npm:' - }; - - const adapter = createOriginAdapter(view, mockCache); - - assert.ok(adapter instanceof CacheAdapter); - assert.equal(adapter.keyPrefix, 'v1:packument:npm:'); - }); - - it('creates CacheAdapter for encoded origin', () => { - const mockCache = {}; - const view = { - registry: null, - origin: 'npm', - getCacheKeyPrefix: () => 'v1:packument:npm:' - }; - - const adapter = createOriginAdapter(view, mockCache); - - assert.ok(adapter instanceof CacheAdapter); - }); -}); diff --git a/src/view/query.js b/src/view/query.js index 164067c..1b67bbc 100644 --- a/src/view/query.js +++ b/src/view/query.js @@ -1,22 +1,25 @@ /** * Query execution for views + * + * The cache instance should be configured with the appropriate storage driver: + * - For registry origins: Use createStorageDriver({ CACHE_DIR: ... }) + * - For local directories: Use createStorageDriver({ LOCAL_DIR: ... }) */ import { createProjection, createFilter } from './projection.js'; -import { createOriginAdapter } from './origin-adapter.js'; /** * Query a view, yielding projected records * @param {View} view - The view to query - * @param {Cache} cache - The cache instance (optional for local origins) + * @param {Cache} cache - The cache instance configured with appropriate driver * @param {Object} options - Query options * @param {number} [options.limit] - Maximum records to return * @param {string} [options.where] - Additional filter expression * @param {boolean} [options.progress] - Show progress on stderr - * @param {object} [options.adapter] - Custom origin adapter (auto-detected if not provided) + * @param {string} [options.keyPrefix] - Override key prefix (defaults to view.getCacheKeyPrefix()) * @yields {Object} Projected records */ export async function* queryView(view, cache, options = {}) { - const { limit, where, progress = false, adapter: providedAdapter } = options; + const { limit, where, progress = false, keyPrefix } = options; // Compile projection from view's select const project = createProjection({ select: view.select }); @@ -24,13 +27,13 @@ export async function* queryView(view, cache, options = {}) { // Compile additional filter if provided const filter = createFilter({ where }); - // Get or create the origin adapter - const adapter = providedAdapter || createOriginAdapter(view, cache); + // Get key prefix - for local dirs this will be ignored by the driver + const prefix = keyPrefix ?? view.getCacheKeyPrefix(); let count = 0; let yielded = 0; - for await (const key of adapter.keys()) { + for await (const key of cache.keys(prefix)) { // Check limit if (limit && yielded >= limit) break; @@ -42,8 +45,11 @@ export async function* queryView(view, cache, options = {}) { } try { - const value = await adapter.fetch(key); - if (!value) continue; + const entry = await cache.fetch(key); + if (!entry) continue; + + // Extract the packument body from cache entry + const value = entry.body || entry; // Apply view's projection const projected = project(value);