diff --git a/.claude/scheduled_tasks.lock b/.claude/scheduled_tasks.lock new file mode 100644 index 000000000..910d70b00 --- /dev/null +++ b/.claude/scheduled_tasks.lock @@ -0,0 +1 @@ +{"sessionId":"5206baae-267d-4d02-bedd-55b8d6b8cf05","pid":3219805,"procStart":"41160224","acquiredAt":1778799452286} \ No newline at end of file diff --git a/bin/cliOperations.ts b/bin/cliOperations.ts index 676d7e2b7..65b4b9324 100644 --- a/bin/cliOperations.ts +++ b/bin/cliOperations.ts @@ -3,18 +3,22 @@ import { loadCredentials, saveCredentials, normalizeTarget } from './cliCredentials.ts'; import { isJWTExpired } from '../security/tokenAuthentication.ts'; import * as envMgr from '../utility/environment/environmentManager.ts'; -envMgr.initSync(); import * as terms from '../utility/hdbTerms.ts'; import { httpRequest } from '../utility/common_utils.ts'; import * as path from 'path'; -import * as fs from 'fs-extra'; +import fs from 'fs-extra'; import * as YAML from 'yaml'; import { streamPackagedDirectory, getPackagedDirectorySize } from '../components/packageComponent.ts'; import { buildMultipartBody } from './multipartBuilder.ts'; import { parseSSE } from './sseConsumer.ts'; import { DeployRenderer } from './deployRenderer.ts'; -import { getHdbPid } from '../utility/processManagement/processManagement.js'; -import { initConfig, getConfigPath } from '../config/configUtils.js'; +import { getHdbPid } from '../utility/processManagement/processManagement.ts'; +import { initConfig, getConfigPath } from '../config/configUtils.ts'; +try { + envMgr.initSync(); +} catch { + /* tolerate ESM cycle TDZ; bin entry will re-call later */ +} const OP_ALIASES = { deploy: 'deploy_component', package: 'package_component' }; diff --git a/bin/copyDb.ts b/bin/copyDb.ts index e58936c4e..cd5646fa3 100644 --- a/bin/copyDb.ts +++ b/bin/copyDb.ts @@ -1,7 +1,8 @@ import { getDatabases, getDefaultCompression, resetDatabases } from '../resources/databases.ts'; import { open, asBinary } from 'lmdb'; import { join } from 'path'; -import { move, remove } from 'fs-extra'; +import _fs_extra from 'fs-extra'; +const { move, remove } = _fs_extra; import { existsSync, mkdirSync } from 'node:fs'; import { get } from '../utility/environment/environmentManager.ts'; import OpenEnvironmentObject from '../utility/lmdb/OpenEnvironmentObject.ts'; @@ -10,7 +11,7 @@ import { INTERNAL_DBIS_NAME, AUDIT_STORE_NAME } from '../utility/lmdb/terms.ts'; import { CONFIG_PARAMS, DATABASES_DIR_NAME } from '../utility/hdbTerms.ts'; import { AUDIT_STORE_OPTIONS } from '../resources/auditStore.ts'; import { describeSchema } from '../dataLayer/schemaDescribe.ts'; -import { updateConfigValue } from '../config/configUtils.js'; +import { updateConfigValue } from '../config/configUtils.ts'; import * as hdbLogger from '../utility/logging/harper_logger.ts'; import { RocksDatabase, type RocksDatabaseOptions } from '@harperfast/rocksdb-js'; import { RocksIndexStore } from '../resources/RocksIndexStore.ts'; diff --git a/bin/harper.ts b/bin/harper.ts index baa6edd20..e0ff0c1ff 100644 --- a/bin/harper.ts +++ b/bin/harper.ts @@ -4,7 +4,6 @@ import * as fs from 'node:fs'; import * as path from 'node:path'; import logger from '../utility/logging/harper_logger.ts'; -import * as cliOperations from './cliOperations.ts'; import { packageJson } from '../utility/packageUtils.js'; import checkNode from '../launchServiceScripts/utility/checkNodeVersion.js'; import * as hdbTerms from '../utility/hdbTerms.ts'; @@ -40,6 +39,30 @@ version - Print the version deploy - Deploy the application locally or remotely with target= `; +/** + * Initialize the environment manager. Call before dynamically importing the + * subcommand module so any module-load reads of `env.get(…)` see a populated + * configuration. Side-effectful initialization is deferred to `onStartup(…)` + * hooks; subcommand modules drain them via their own lifecycle.runStartup() calls. + */ +async function initEnv() { + const env = await import('../utility/environment/environmentManager.ts'); + env.initSync(); +} + +/** + * Resolve a default-exported function from a dynamically-imported module that + * may have been loaded from a .ts source (ESM: `mod.default` is the function) + * or a tsc-compiled .js (CJS-wrapped: `mod.default` is the CJS exports object + * and `mod.default.default` is the function). + */ +function getDefaultExport(mod: any): any { + if (typeof mod === 'function') return mod; + if (typeof mod.default === 'function') return mod.default; + if (typeof mod.default?.default === 'function') return mod.default.default; + return mod.default ?? mod; +} + async function harper() { let nodeResults = checkNode(); @@ -63,45 +86,50 @@ async function harper() { switch (service) { case SERVICE_ACTIONS_ENUM.HELP: return HELP; - case SERVICE_ACTIONS_ENUM.START: - return require('./run').launch(); - case SERVICE_ACTIONS_ENUM.INSTALL: - return (require('./install').default || require('./install'))(); - case SERVICE_ACTIONS_ENUM.STOP: - return (require('./stop').default || require('./stop'))().then(() => { + case SERVICE_ACTIONS_ENUM.START: { + await initEnv(); + const mod = await import('./run.ts'); + // runStartup() is drained inside run.ts main() after initialize() + // completes, so server.X singletons and auth's table() resolution + // see a populated env / installed hdbBasePath. + return mod.launch(); + } + case SERVICE_ACTIONS_ENUM.INSTALL: { + return getDefaultExport(await import('./install.ts'))(); + } + case SERVICE_ACTIONS_ENUM.STOP: { + await initEnv(); + return getDefaultExport(await import('./stop.ts'))().then(() => { process.exit(0); }); + } case SERVICE_ACTIONS_ENUM.RESTART: - return require('./restart').restart({}); + return (await import('./restart.ts')).restart({}); case SERVICE_ACTIONS_ENUM.VERSION: return packageJson.version; case SERVICE_ACTIONS_ENUM.UPGRADE: logger.setLogLevel(hdbTerms.LOG_LEVELS.INFO); - // The require is here to better control the flow of imports when this module is called. - return require('./upgrade.js') - .upgrade(null) - .then(() => 'Your instance of Harper is up to date!'); - case SERVICE_ACTIONS_ENUM.STATUS: - return (require('./status').default || require('./status'))(); + return (await import('./upgrade.ts')).upgrade(null).then(() => 'Your instance of Harper is up to date!'); + case SERVICE_ACTIONS_ENUM.STATUS: { + return getDefaultExport(await import('./status.ts'))(); + } case SERVICE_ACTIONS_ENUM.LOGIN: { const target = process.argv[3]; const username = process.argv[4]; - const { login } = require('./login'); - return login(target, username); + return (await import('./login.ts')).login(target, username); } case SERVICE_ACTIONS_ENUM.LOGOUT: { const target = process.argv[3]; - const { logout } = require('./logout'); - return logout(target); + return (await import('./logout.ts')).logout(target); } case SERVICE_ACTIONS_ENUM.RENEWCERTS: - return require('../security/keys') + return (await import('../security/keys.ts')) .renewSelfSigned() .then(() => 'Successfully renewed self-signed certificates'); case SERVICE_ACTIONS_ENUM.COPYDB: { let sourceDb = process.argv[3]; let targetDbPath = process.argv[4]; - return require('./copyDb').copyDb(sourceDb, targetDbPath); + return (await import('./copyDb.ts')).copyDb(sourceDb, targetDbPath); } case SERVICE_ACTIONS_ENUM.DEV: process.env.DEV_MODE = 'true'; @@ -140,17 +168,27 @@ async function harper() { } } // fall through - case undefined: // run harperdb in the foreground in standard mode - return require('./run').main(); - default: + case undefined: { + // run harperdb in the foreground in standard mode. + // runStartup() is drained inside run.ts main() after initialize(). + await initEnv(); + const mod = await import('./run.ts'); + return mod.main(); + } + default: { + const cliOperations = await import('./cliOperations.ts'); const cliApiOp = cliOperations.buildRequest(); logger.trace('calling cli operations with:', cliApiOp); await cliOperations.cliOperations(cliApiOp); return; + } } } export { harper }; -if (require.main === module) { +// In CJS (dist), `module` is defined and we check the entry. In ESM (typestrip), +// `module` is undefined; this file is only run directly as a CLI, so treat as main. +const isEntry = typeof module === 'undefined' || require.main === module; +if (isEntry) { harper() .then((message) => { if (message) { diff --git a/bin/restart.ts b/bin/restart.ts index f266f430f..154689c39 100644 --- a/bin/restart.ts +++ b/bin/restart.ts @@ -4,17 +4,20 @@ import minimist from 'minimist'; import { isMainThread, parentPort } from 'worker_threads'; import * as hdbTerms from '../utility/hdbTerms.ts'; import hdbLogger from '../utility/logging/harper_logger.ts'; -import * as processMan from '../utility/processManagement/processManagement.js'; +import * as processMan from '../utility/processManagement/processManagement.ts'; import { compactOnStart } from './copyDb.ts'; -import { restartWorkers, onMessageByType, shutdownWorkersNow } from '../server/threads/manageThreads.js'; +import { restartWorkers, onMessageByType, shutdownWorkersNow } from '../server/threads/manageThreads.ts'; import { handleHDBError, hdbErrors } from '../utility/errors/hdbError.ts'; const { HTTP_STATUS_CODES } = hdbErrors; import * as envMgr from '../utility/environment/environmentManager.ts'; import * as path from 'node:path'; import { unlinkSync } from 'node:fs'; import { getThisNodeName } from '../server/nodeName.ts'; -envMgr.initSync(); - +try { + envMgr.initSync(); +} catch { + /* tolerate ESM cycle TDZ; bin entry will re-call later */ +} const RESTART_RESPONSE = `Restarting Harper. This may take up to ${hdbTerms.RESTART_TIMEOUT_MS / 1000} seconds.`; const INVALID_SERVICE_ERR = 'Invalid service'; @@ -66,7 +69,7 @@ async function restart(req: any) { // and shut down. hdbLogger.debug('Shutdown workers'); await shutdownWorkersNow(); - const { closeServers } = require('../server/threads/threadServer.js'); + const { closeServers } = await import('../server/threads/threadServer.ts'); await closeServers(); await processMan.cleanupChildrenProcesses(false); // remove pid file so it doesn't trip up the launch diff --git a/bin/run.ts b/bin/run.ts index c932d1cf8..cca07bc7b 100644 --- a/bin/run.ts +++ b/bin/run.ts @@ -1,24 +1,27 @@ 'use strict'; import * as env from '../utility/environment/environmentManager.ts'; -env.initSync(); - +try { + env.initSync(); +} catch { + /* tolerate ESM cycle TDZ; bin entry will re-call later */ +} // This unused restart import is here so that main thread loads ITC event listener defined in restart file. Do not remove. import './restart.ts'; import * as terms from '../utility/hdbTerms.ts'; const { CONFIG_PARAMS } = terms; import hdbLogger from '../utility/logging/harper_logger.ts'; -import * as fs from 'fs-extra'; +import fs from 'fs-extra'; import * as path from 'path'; -import checkJwtTokens from '../utility/install/checkJWTTokensExist.js'; +import checkJwtTokens from '../utility/install/checkJWTTokensExist.ts'; import { install } from '../utility/install/installer.ts'; import chalk from 'chalk'; import { packageJson } from '../utility/packageUtils.js'; import * as hdbUtils from '../utility/common_utils.ts'; import * as installation from '../utility/installation.ts'; -import * as configUtils from '../config/configUtils.js'; +import * as configUtils from '../config/configUtils.ts'; import assignCMDENVVariables from '../utility/assignCmdEnvVariables.ts'; -import * as upgrade from './upgrade.js'; +import * as upgrade from './upgrade.ts'; import { compactOnStart, migrateOnStart } from './copyDb.ts'; import minimist from 'minimist'; import * as keys from '../security/keys.ts'; @@ -27,7 +30,7 @@ import * as hdbInfoController from '../dataLayer/hdbInfoController.ts'; import { isReadOnlyMode } from '../resources/databases.ts'; import { getThisNodeName } from '../server/nodeName.ts'; import * as hdbTerms from '../utility/hdbTerms.ts'; -import { getHdbPid, isProcessRunning } from '../utility/processManagement/processManagement.js'; +import { getHdbPid, isProcessRunning } from '../utility/processManagement/processManagement.ts'; import { PACKAGE_ROOT } from '../utility/packageUtils.js'; let pmUtils; @@ -108,7 +111,7 @@ async function initialize(calledByInstall = false, calledByMain = false) { // If HARPER_SET_CONFIG is present, filter out any config keys that are set in it // to prevent individual env vars from overriding explicit runtime configuration - const { filterArgsAgainstRuntimeConfig } = require('../config/harperConfigEnvVars'); + const { filterArgsAgainstRuntimeConfig } = await import('../config/harperConfigEnvVars.ts'); parsedArgs = filterArgsAgainstRuntimeConfig(parsedArgs); if (!hdbUtils.isEmpty(parsedArgs) && !hdbUtils.isEmptyOrZeroLength(Object.keys(parsedArgs))) { @@ -199,6 +202,12 @@ async function main(calledByInstall = false) { } await initialize(calledByInstall, true); + // Drain onStartup hooks after install/initialize so server.X singletons + // (server.http, server.getUser, etc.) and auth.ts's onStartup table() + // call see the populated env / installed hdbBasePath. + const lifecycle = await import('../utility/lifecycle.ts'); + await lifecycle.runStartup(); + if (env.get(terms.CONFIG_PARAMS.STORAGE_COMPACTONSTART)) await compactOnStart(); if (env.get(terms.CONFIG_PARAMS.STORAGE_MIGRATEONSTART)) await migrateOnStart(); @@ -233,7 +242,7 @@ function started() { async function launch(exit = true) { skipExitListeners = !exit; try { - if (pmUtils === undefined) pmUtils = require('../utility/processManagement/processManagement.js'); + if (pmUtils === undefined) pmUtils = await import('../utility/processManagement/processManagement.ts'); hdbLogger.debug('initializing processManagement...'); await initialize(); hdbLogger.debug('Starting new main process'); diff --git a/bin/status.ts b/bin/status.ts index 6d78c5567..d2f80af50 100644 --- a/bin/status.ts +++ b/bin/status.ts @@ -1,6 +1,6 @@ 'use strict'; -import * as fs from 'fs-extra'; +import fs from 'fs-extra'; import * as path from 'path'; import * as YAML from 'yaml'; @@ -9,8 +9,11 @@ import hdbLog from '../utility/logging/harper_logger.ts'; import * as systemInformation from '../utility/environment/systemInformation.ts'; import * as envMgr from '../utility/environment/environmentManager.ts'; import * as installation from '../utility/installation.ts'; -envMgr.initSync(); - +try { + envMgr.initSync(); +} catch { + /* tolerate ESM cycle TDZ; bin entry will re-call later */ +} const STATUSES = { RUNNING: 'running', STOPPED: 'stopped', diff --git a/bin/upgrade.js b/bin/upgrade.ts similarity index 82% rename from bin/upgrade.js rename to bin/upgrade.ts index 152b7b8eb..41b748cfa 100644 --- a/bin/upgrade.js +++ b/bin/upgrade.ts @@ -1,33 +1,28 @@ -'use strict'; - /** * The upgrade module is used to facilitate the upgrade process for existing instances of HDB that pull down a new version * of HDB from NPM that requires a specific upgrade script be run - e.g. there are changes required for the settings.js * config file, a data model change requires a re-indexing script is run, etc. */ -const env = require('../utility/environment/environmentManager.ts'); -env.initSync(); - -const chalk = require('chalk'); -const hdbLogger = require('../utility/logging/harper_logger.ts'); -const hdbTerms = require('../utility/hdbTerms.ts'); -const directivesManager = require('../upgrade/directivesManager.ts'); -const installation = require('../utility/installation.ts'); -const hdbInfoController = require('../dataLayer/hdbInfoController.ts'); -const upgradePrompt = require('../upgrade/upgradePrompt.ts'); -const globalSchema = require('../utility/globalSchema.ts'); -const { packageJson } = require('../utility/packageUtils.js'); -const promisify = require('util').promisify; +import * as env from '../utility/environment/environmentManager.ts'; + +import chalk from 'chalk'; +import _hdbLogger from '../utility/logging/harper_logger.ts'; +const hdbLogger = _hdbLogger; +import * as hdbTerms from '../utility/hdbTerms.ts'; +import * as directivesManager from '../upgrade/directivesManager.ts'; +import * as installation from '../utility/installation.ts'; +import * as hdbInfoController from '../dataLayer/hdbInfoController.ts'; +import * as upgradePrompt from '../upgrade/upgradePrompt.ts'; +import * as globalSchema from '../utility/globalSchema.ts'; +import { packageJson } from '../utility/packageUtils.js'; +import { promisify } from 'util'; const pSchemaToGlobal = promisify(globalSchema.setSchemaDataToGlobal); let pm2Utils; const { UPGRADE_VERSION } = hdbTerms.UPGRADE_JSON_FIELD_NAMES_ENUM; -module.exports = { - upgrade, -}; - +export { upgrade }; /** * Runs the upgrade directives, if needed, for an updated version of Harper. * @@ -39,7 +34,7 @@ async function upgrade(upgradeObj) { // Requiring the processManagement mod will create the .pm2 dir. This code is here to allow install to set // pm2 env vars before that is done. - if (pm2Utils === undefined) pm2Utils = require('../utility/processManagement/processManagement.js'); + if (pm2Utils === undefined) pm2Utils = await import('../utility/processManagement/processManagement.ts'); //We have to make sure HDB is installed before doing anything else const installed = installation.isHdbInstalled(env, hdbLogger); diff --git a/components/Application.ts b/components/Application.ts index c82aa4ced..0b80557b9 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -1,5 +1,5 @@ import { type Logger } from '../utility/logging/logger.ts'; -import { getConfigObj, getConfigValue, getConfigPath } from '../config/configUtils.js'; +import { getConfigObj, getConfigValue, getConfigPath } from '../config/configUtils.ts'; import { CONFIG_PARAMS } from '../utility/hdbTerms.ts'; import logger from '../utility/logging/harper_logger.ts'; import { broadcastDeployStart, broadcastDeployEnd } from './deployLifecycle.ts'; diff --git a/components/Component.ts b/components/Component.ts index 7f04e5761..8ea23daf2 100644 --- a/components/Component.ts +++ b/components/Component.ts @@ -1,8 +1,8 @@ -import { resolveBaseURLPath } from './resolveBaseURLPath'; -import { deriveCommonPatternBase } from './deriveCommonPatternBase'; -import { deriveGlobOptions, FastGlobOptions, FilesOption } from './deriveGlobOptions'; -import { scan } from 'micromatch'; - +import { resolveBaseURLPath } from './resolveBaseURLPath.ts'; +import { deriveCommonPatternBase } from './deriveCommonPatternBase.ts'; +import { deriveGlobOptions, type FastGlobOptions, type FilesOption } from './deriveGlobOptions.ts'; +import _micromatch from 'micromatch'; +const { scan } = _micromatch; interface ComponentConfig { files: FilesOption; urlPath?: string; diff --git a/components/ComponentV1.ts b/components/ComponentV1.ts index 4804a271e..f02bd8551 100644 --- a/components/ComponentV1.ts +++ b/components/ComponentV1.ts @@ -3,12 +3,12 @@ import fg from 'fast-glob'; import { Resources } from '../resources/Resources.ts'; import harperLogger from '../utility/logging/harper_logger.ts'; import { resolveBaseURLPath } from './resolveBaseURLPath.ts'; -import { deriveGlobOptions, FastGlobOptions, FilesOption } from './deriveGlobOptions.ts'; +import { deriveGlobOptions, type FastGlobOptions, type FilesOption } from './deriveGlobOptions.ts'; import { basename, join } from 'node:path'; import { readFile } from 'node:fs/promises'; import { deriveURLPath } from './deriveURLPath.ts'; -import { scan } from 'micromatch'; - +import _micromatch from 'micromatch'; +const { scan } = _micromatch; interface ComponentV1Config { files: string | string[] | FilesOption; /** @deprecated */ path?: string; diff --git a/components/EntryHandler.ts b/components/EntryHandler.ts index d4b2b8072..feda05ac9 100644 --- a/components/EntryHandler.ts +++ b/components/EntryHandler.ts @@ -2,14 +2,14 @@ import { type Logger } from '../utility/logging/logger.ts'; import { loggerWithTag } from '../utility/logging/harper_logger.ts'; import type { Stats } from 'node:fs'; import { EventEmitter, once } from 'node:events'; -import { Component, FileAndURLPathConfig } from './Component.ts'; -import chokidar, { FSWatcher, FSWatcherEventMap } from 'chokidar'; +import { Component, type FileAndURLPathConfig } from './Component.ts'; +import chokidar, { FSWatcher, type FSWatcherEventMap } from 'chokidar'; import { join } from 'node:path'; import { readFile } from 'node:fs/promises'; -import { FilesOption } from './deriveGlobOptions.ts'; +import { type FilesOption } from './deriveGlobOptions.ts'; import { deriveURLPath } from './deriveURLPath.ts'; -import { isMatch } from 'micromatch'; - +import _micromatch from 'micromatch'; +const { isMatch } = _micromatch; export interface BaseEntry { stats?: Stats; urlPath: string; diff --git a/components/OptionsWatcher.ts b/components/OptionsWatcher.ts index 0f02342dd..babbac053 100644 --- a/components/OptionsWatcher.ts +++ b/components/OptionsWatcher.ts @@ -6,8 +6,8 @@ import chokidar, { type FSWatcher } from 'chokidar'; import { readFile } from 'node:fs/promises'; import { isDeepStrictEqual } from 'util'; import { DEFAULT_CONFIG } from './DEFAULT_CONFIG.ts'; -import { cloneDeep } from 'lodash'; - +import _lodash from 'lodash'; +const { cloneDeep } = _lodash; export interface Config { [key: string]: ConfigValue; } diff --git a/components/PluginModule.ts b/components/PluginModule.ts index 6b0781018..47d107193 100644 --- a/components/PluginModule.ts +++ b/components/PluginModule.ts @@ -1,4 +1,4 @@ -import { Scope } from './Scope'; +import { Scope } from './Scope.ts'; export interface PluginModule { handleApplication: (scope: Scope) => void | Promise; diff --git a/components/Scope.ts b/components/Scope.ts index 228c9769b..835be192b 100644 --- a/components/Scope.ts +++ b/components/Scope.ts @@ -4,11 +4,11 @@ import { EventEmitter, once } from 'node:events'; import { databaseEventsEmitter, table } from '../resources/databases.ts'; import { server, type Server } from '../server/Server.ts'; import { EntryHandler, type EntryHandlerEventMap, type onEntryEventHandler } from './EntryHandler.ts'; -import { OptionsWatcher, OptionsWatcherEventMap } from './OptionsWatcher.ts'; +import { OptionsWatcher, type OptionsWatcherEventMap } from './OptionsWatcher.ts'; import { resources, type Resources } from '../resources/Resources.ts'; import { Models, models as modelsSingleton } from '../resources/models/Models.ts'; import type { FileAndURLPathConfig } from './Component.ts'; -import { FilesOption } from './deriveGlobOptions.ts'; +import { type FilesOption } from './deriveGlobOptions.ts'; import { requestRestart } from './requestRestart.ts'; import { ApplicationScope } from './ApplicationScope.ts'; import { deployLifecycle } from './deployLifecycle.ts'; diff --git a/components/componentLoader.ts b/components/componentLoader.ts index 7d944cadb..18c3f6696 100644 --- a/components/componentLoader.ts +++ b/components/componentLoader.ts @@ -1,4 +1,5 @@ -import { onMessageByType } from '../server/threads/manageThreads.js'; +import { onMessageByType } from '../server/threads/manageThreads.ts'; +import { onStartup } from '../utility/lifecycle.ts'; import { readdirSync, readFileSync, @@ -25,7 +26,7 @@ import * as staticFiles from '../server/static.ts'; import * as loadEnv from '../resources/loadEnv.ts'; import harperLogger from '../utility/logging/harper_logger.ts'; import * as dataLoader from '../resources/dataLoader.ts'; -import { restartWorkers, getWorkerIndex } from '../server/threads/manageThreads.js'; +import { restartWorkers, getWorkerIndex } from '../server/threads/manageThreads.ts'; import { resetRestartNeeded, subscribeToRestartRequests } from './requestRestart.ts'; import { scopedImport } from '../security/jsLoader.ts'; import { server } from '../server/Server.ts'; @@ -34,7 +35,7 @@ import { table } from '../resources/databases.ts'; import { getHdbBasePath } from '../utility/environment/environmentManager.ts'; import * as auth from '../security/auth.ts'; import * as mqtt from '../server/mqtt.ts'; -import { getConfigObj, getConfigPath } from '../config/configUtils.js'; +import { getConfigObj, getConfigPath } from '../config/configUtils.ts'; import { bootstrapModels } from '../resources/models/bootstrap.ts'; import { ErrorResource } from '../resources/ErrorResource.ts'; import { Scope } from './Scope.ts'; @@ -45,7 +46,7 @@ import * as mcpComponent from './mcp/index.ts'; import { Status } from '../server/status/index.ts'; import { lifecycle as componentLifecycle } from './status/index.ts'; import { DEFAULT_CONFIG } from './DEFAULT_CONFIG.ts'; -import { PluginModule } from './PluginModule.ts'; +import { type PluginModule } from './PluginModule.ts'; import { getEnvBuiltInComponents } from './Application.ts'; import { pathToFileURL } from 'node:url'; @@ -98,7 +99,7 @@ export const TRUSTED_RESOURCE_PLUGINS: any = { roles, jsResource: jsHandler, get fastifyRoutes() { - return require('../server/fastifyRoutes'); + return _fastifyRoutes; }, login, static: staticFiles, @@ -115,19 +116,23 @@ export const TRUSTED_RESOURCE_PLUGINS: any = { login: ... */ }; -if (isMainThread) { - TRUSTED_RESOURCE_PLUGINS.operationsApi = require('../server/operationsServer'); -} else { - // The HTTP operations API itself only binds in the main thread, but worker threads still - // dispatch operations — most notably, the replication WebSocket handler in workers receives - // inter-node operations like `add_node_back` and calls `server.operation(...)`. That requires - // `server.operation` / `server.registerOperation` to be wired up here too, and the operation - // function map to be initialized, BEFORE component plugins (replication, etc.) load and call - // `server.registerOperation?.({...})` at their module top level. Requiring serverUtilities - // directly (rather than the full operationsServer) avoids binding the fastify HTTP layer in - // workers while still installing the dispatch machinery. - require('../server/serverHelpers/serverUtilities'); -} +let _fastifyRoutes: any; +onStartup(async () => { + _fastifyRoutes = await import('../server/fastifyRoutes.ts'); + if (isMainThread) { + TRUSTED_RESOURCE_PLUGINS.operationsApi = await import('../server/operationsServer.ts'); + } else { + // The HTTP operations API itself only binds in the main thread, but worker threads still + // dispatch operations — most notably, the replication WebSocket handler in workers receives + // inter-node operations like `add_node_back` and calls `server.operation(...)`. That requires + // `server.operation` / `server.registerOperation` to be wired up here too, and the operation + // function map to be initialized, BEFORE component plugins (replication, etc.) load and call + // `server.registerOperation?.({...})` at their module top level. Loading serverUtilities + // directly (rather than the full operationsServer) avoids binding the fastify HTTP layer in + // workers while still installing the dispatch machinery. + await import('../server/serverHelpers/serverUtilities.ts'); + } +}); for (const { name, packageIdentifier } of getEnvBuiltInComponents()) { TRUSTED_RESOURCE_PLUGINS[name] = packageIdentifier; @@ -315,7 +320,19 @@ export async function loadComponent( } else { config = DEFAULT_CONFIG; } + // getConfigObj() can return undefined when the harper config has not yet been + // initialised (e.g. test paths that touch the loader without going through + // bin/harper.ts). Treat that the same as a missing config rather than crashing + // on `config.extensionModule` below. For non-root components, an empty/null + // parse result means an intentionally-empty config file — do NOT fall back to + // DEFAULT_CONFIG, otherwise OptionsWatcher waits forever for plugins that the + // file doesn't actually declare and the worker hangs on scope.ready. + if (isRoot) config ??= DEFAULT_CONFIG; applicationScope.config ??= config; + if (!config) { + // Empty/comment-only config file on a non-root component: nothing to load. + return undefined; + } // #629 (Phase 2 of #510): populate the model-backend registry from the root // config's `models:` block before any user `handleApplication(scope)` runs, diff --git a/components/operations.js b/components/operations.ts similarity index 91% rename from components/operations.js rename to components/operations.ts index b2dbba014..a42a11113 100644 --- a/components/operations.js +++ b/components/operations.ts @@ -1,25 +1,26 @@ -'use strict'; - -const path = require('node:path'); -const { isMainThread } = require('node:worker_threads'); -const fs = require('fs-extra'); -const fg = require('fast-glob'); -const normalize = require('normalize-path'); -const validator = require('./operationsValidation.js'); -const log = require('../utility/logging/harper_logger.ts'); -const hdbTerms = require('../utility/hdbTerms.ts'); -const env = require('../utility/environment/environmentManager.ts'); -const configUtils = require('../config/configUtils.js'); -const hdbUtils = require('../utility/common_utils.ts'); -const { handleHDBError, hdbErrors } = require('../utility/errors/hdbError.ts'); +import path from 'node:path'; +import { isMainThread } from 'node:worker_threads'; +import fs from 'fs-extra'; +import fg from 'fast-glob'; +import normalize from 'normalize-path'; +import * as validator from './operationsValidation.ts'; +import log from '../utility/logging/harper_logger.ts'; +import * as hdbTerms from '../utility/hdbTerms.ts'; +import * as env from '../utility/environment/environmentManager.ts'; +import * as configUtils from '../config/configUtils.ts'; +import * as hdbUtils from '../utility/common_utils.ts'; +import { handleHDBError, hdbErrors } from '../utility/errors/hdbError.ts'; +import * as serverUtilities from '../server/serverHelpers/serverUtilities.ts'; +import { internal as statusInternal } from './status/index.ts'; const { HDB_ERROR_MSGS, HTTP_STATUS_CODES } = hdbErrors; -const manageThreads = require('../server/threads/manageThreads.js'); -const { packageDirectory } = require('../components/packageComponent.ts'); -const { Resources } = require('../resources/Resources.ts'); -const { Application, prepareApplication } = require('./Application.ts'); -const { server } = require('../server/Server.ts'); -const { DeploymentRecorder, awaitDeploymentRow } = require('./deploymentRecorder.ts'); -const { ProgressEmitter } = require('../server/serverHelpers/progressEmitter.ts'); +import * as manageThreads from '../server/threads/manageThreads.ts'; +import { packageDirectory } from '../components/packageComponent.ts'; +import { Resources } from '../resources/Resources.ts'; +import { Application, prepareApplication as _prepareApplication } from './Application.ts'; +const prepareApplication = _prepareApplication; +import { server } from '../server/Server.ts'; +import { DeploymentRecorder, awaitDeploymentRow } from './deploymentRecorder.ts'; +import { ProgressEmitter } from '../server/serverHelpers/progressEmitter.ts'; /** * Read the settings.js file and return the @@ -41,7 +42,7 @@ function customFunctionsStatus() { new Error(), HDB_ERROR_MSGS.FUNCTION_STATUS, HTTP_STATUS_CODES.INTERNAL_SERVER_ERROR, - log.ERR, + undefined, err ); } @@ -77,7 +78,7 @@ function getCustomFunctions() { new Error(), HDB_ERROR_MSGS.GET_FUNCTIONS, HTTP_STATUS_CODES.INTERNAL_SERVER_ERROR, - log.ERR, + undefined, err ); } @@ -116,7 +117,7 @@ function getCustomFunction(req) { new Error(), HDB_ERROR_MSGS.GET_FUNCTION, HTTP_STATUS_CODES.INTERNAL_SERVER_ERROR, - log.ERR, + undefined, err ); } @@ -156,7 +157,7 @@ async function setCustomFunction(req) { new Error(), HDB_ERROR_MSGS.SET_FUNCTION, HTTP_STATUS_CODES.INTERNAL_SERVER_ERROR, - log.ERR, + undefined, err ); } @@ -196,7 +197,7 @@ async function dropCustomFunction(req) { new Error(), HDB_ERROR_MSGS.DROP_FUNCTION, HTTP_STATUS_CODES.INTERNAL_SERVER_ERROR, - log.ERR, + undefined, err ); } @@ -244,7 +245,7 @@ async function addComponent(req) { new Error(), HDB_ERROR_MSGS.ADD_FUNCTION, HTTP_STATUS_CODES.INTERNAL_SERVER_ERROR, - log.ERR, + undefined, err ); } @@ -270,7 +271,7 @@ async function dropCustomFunctionProject(req) { const cfDir = configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT); const { project } = req; - let apps = env.get(hdbTerms.CONFIG_PARAMS.APPS); + let apps = env.get((hdbTerms.CONFIG_PARAMS as any).APPS); if (!hdbUtils.isEmptyOrZeroLength(apps)) { let appFound = false; for (const [i, app] of apps.entries()) { @@ -282,7 +283,7 @@ async function dropCustomFunctionProject(req) { } if (appFound) { - configUtils.updateConfigValue(hdbTerms.CONFIG_PARAMS.APPS, apps); + configUtils.updateConfigValue((hdbTerms.CONFIG_PARAMS as any).APPS, apps); return `Successfully deleted project: ${project}`; } @@ -299,7 +300,7 @@ async function dropCustomFunctionProject(req) { new Error(), HDB_ERROR_MSGS.DROP_FUNCTION_PROJECT, HTTP_STATUS_CODES.INTERNAL_SERVER_ERROR, - log.ERR, + undefined, err ); } @@ -365,8 +366,11 @@ async function deployComponent(req) { // Write to root config if the request contains a package identifier if (req.package) { // Check if trying to overwrite a core component (requires force) - // Lazy-load to avoid circular dependency with componentLoader - const { TRUSTED_RESOURCE_PLUGINS } = require('./componentLoader.ts'); + // Lazy-load to avoid circular dependency with componentLoader. + // (Also keeps componentLoader/dataLoader out of the test-time module graph + // when callers only require this file for unrelated exports — pulling + // dataLoader in transitively defeats dataLoader.test.js's forComponent stub.) + const { TRUSTED_RESOURCE_PLUGINS } = await import('./componentLoader.ts'); if (TRUSTED_RESOURCE_PLUGINS[req.project] && !req.force) { throw handleHDBError( new Error(), @@ -375,7 +379,7 @@ async function deployComponent(req) { ); } - const applicationConfig = { package: req.package }; + const applicationConfig: any = { package: req.package }; // Avoid writing an empty `install:` block if (req.install_command || req.install_timeout || req.install_allow_scripts !== undefined) { applicationConfig.install = { @@ -465,11 +469,11 @@ async function deployComponent(req) { const pseudoResources = new Resources(); pseudoResources.isWorker = true; - const componentLoader = require('./componentLoader.ts').default || require('./componentLoader.ts'); + const componentLoader = await import('./componentLoader.ts'); let lastError; componentLoader.setErrorReporter((error) => (lastError = error)); emit('phase', { phase: 'load', status: 'start' }); - await componentLoader.loadComponent( + await (componentLoader.loadComponent as any)( application.dirPath, pseudoResources, undefined, @@ -508,7 +512,7 @@ async function deployComponent(req) { } : undefined; emit('phase', { phase: 'replicate', status: 'start' }); - let response = await server.replication.replicateOperation(req, { onPeerResult }); + let response: any = await server.replication.replicateOperation(req, { onPeerResult }); emit('phase', { phase: 'replicate', status: 'done' }); if (recorder && response?.replicated) { // Fallback path for replicators that don't honor onPeerResult: re-record the @@ -522,13 +526,12 @@ async function deployComponent(req) { emit('phase', { phase: 'restart', status: 'done' }); response.message = `Successfully deployed: ${application.name}, restarting Harper`; } else if (rollingRestart) { - const serverUtilities = require('../server/serverHelpers/serverUtilities.ts'); emit('phase', { phase: 'restart', status: 'start' }); - const jobResponse = await serverUtilities.executeJob({ + const jobResponse: any = await serverUtilities.executeJob({ operation: 'restart_service', service: 'http', replicated: true, - }); + } as any); emit('phase', { phase: 'restart', status: 'done' }); response.restartJobId = jobResponse.job_id; @@ -657,7 +660,6 @@ async function getComponents() { if (sourcePackage) entry.package = sourcePackage; } - const { internal: statusInternal } = require('./status/index.ts'); let consolidatedStatuses; try { @@ -797,16 +799,16 @@ async function dropComponent(req) { return response; } -exports.customFunctionsStatus = customFunctionsStatus; -exports.getCustomFunctions = getCustomFunctions; -exports.getCustomFunction = getCustomFunction; -exports.setCustomFunction = setCustomFunction; -exports.dropCustomFunction = dropCustomFunction; -exports.addComponent = addComponent; -exports.dropCustomFunctionProject = dropCustomFunctionProject; -exports.packageComponent = packageComponent; -exports.deployComponent = deployComponent; -exports.getComponents = getComponents; -exports.getComponentFile = getComponentFile; -exports.setComponentFile = setComponentFile; -exports.dropComponent = dropComponent; +export { customFunctionsStatus }; +export { getCustomFunctions }; +export { getCustomFunction }; +export { setCustomFunction }; +export { dropCustomFunction }; +export { addComponent }; +export { dropCustomFunctionProject }; +export { packageComponent }; +export { deployComponent }; +export { getComponents }; +export { getComponentFile }; +export { setComponentFile }; +export { dropComponent }; diff --git a/components/operationsValidation.js b/components/operationsValidation.ts similarity index 93% rename from components/operationsValidation.js rename to components/operationsValidation.ts index d5d43190f..86a8777fd 100644 --- a/components/operationsValidation.js +++ b/components/operationsValidation.ts @@ -1,19 +1,16 @@ -'use strict'; - -const Joi = require('joi'); -const fs = require('fs-extra'); -const path = require('path'); -const validator = require('../validation/validationWrapper.ts'); -const hdbTerms = require('../utility/hdbTerms.ts'); -const hdbLogger = require('../utility/logging/harper_logger.ts'); -const configUtils = require('../config/configUtils.js'); -const { hdbErrors } = require('../utility/errors/hdbError.ts'); +import Joi from 'joi'; +import fs from 'fs-extra'; +import path from 'path'; +import * as validator from '../validation/validationWrapper.ts'; +import * as hdbTerms from '../utility/hdbTerms.ts'; +import hdbLogger from '../utility/logging/harper_logger.ts'; +import * as configUtils from '../config/configUtils.ts'; +import { hdbErrors } from '../utility/errors/hdbError.ts'; const { HDB_ERROR_MSGS } = hdbErrors; - // File name can only be alphanumeric, dash and underscores const PROJECT_FILE_NAME_REGEX = /^[a-zA-Z0-9-_]+$/; -module.exports = { +export { getDropCustomFunctionValidator, setCustomFunctionValidator, addComponentValidator, @@ -24,7 +21,6 @@ module.exports = { getComponentFileValidator, dropComponentFileValidator, }; - /** * Check to see if a project dir exists in the custom functions dir. * @param checkExists - determine if validator returns error if exists or vice versa diff --git a/components/status/crossThread.ts b/components/status/crossThread.ts index c4fb05740..a3ee2128e 100644 --- a/components/status/crossThread.ts +++ b/components/status/crossThread.ts @@ -5,8 +5,8 @@ * and aggregating it into a unified view. */ -import { sendItcEvent } from '../../server/threads/itc.js'; -import { getWorkerIndex, onMessageByType, getWorkerCount } from '../../server/threads/manageThreads.js'; +import { sendItcEvent } from '../../server/threads/itc.ts'; +import { getWorkerIndex, onMessageByType, getWorkerCount } from '../../server/threads/manageThreads.ts'; import { ITC_EVENT_TYPES } from '../../utility/hdbTerms.ts'; import { loggerWithTag } from '../../utility/logging/logger.ts'; import { ComponentStatusRegistry } from './ComponentStatusRegistry.ts'; diff --git a/config/RootConfigWatcher.ts b/config/RootConfigWatcher.ts index d51abdae9..da0bddd4f 100644 --- a/config/RootConfigWatcher.ts +++ b/config/RootConfigWatcher.ts @@ -1,6 +1,6 @@ import chokidar, { FSWatcher } from 'chokidar'; import { readFile } from 'node:fs/promises'; -import { getConfigFilePath } from './configUtils.js'; +import { getConfigFilePath } from './configUtils.ts'; import { EventEmitter, once } from 'node:events'; import { parse } from 'yaml'; diff --git a/config/configUtils.js b/config/configUtils.ts similarity index 92% rename from config/configUtils.js rename to config/configUtils.ts index 3b9a98c6f..c775a777b 100644 --- a/config/configUtils.js +++ b/config/configUtils.ts @@ -1,34 +1,40 @@ -'use strict'; - -const hdbTerms = require('../utility/hdbTerms.ts'); -const hdbUtils = require('../utility/common_utils.ts'); -const logger = require('../utility/logging/harper_logger.ts'); -const { configValidator } = require('../validation/configValidator.ts'); -const fs = require('fs-extra'); -const YAML = require('yaml'); -const path = require('path'); -const { threadId } = require('node:worker_threads'); -const { randomBytes } = require('node:crypto'); -const isNumber = require('is-number'); -const PropertiesReader = require('properties-reader'); -const _ = require('lodash'); -const { handleHDBError } = require('../utility/errors/hdbError.ts'); -const { HTTP_STATUS_CODES, HDB_ERROR_MSGS } = require('../utility/errors/commonErrors.ts'); -const { server } = require('../server/Server.ts'); -const { getBackupDirPath } = require('./configHelpers.ts'); -const { PACKAGE_ROOT } = require('../utility/packageUtils'); - +import * as hdbTerms from '../utility/hdbTerms.ts'; +import * as hdbUtils from '../utility/common_utils.ts'; +import _logger from '../utility/logging/harper_logger.ts'; +const logger = _logger; +import { configValidator as _configValidator } from '../validation/configValidator.ts'; +const configValidator = _configValidator; +import _fs from 'fs-extra'; +const fs = _fs; +import _YAML from 'yaml'; +const YAML = _YAML; +import path from 'path'; +import { threadId } from 'node:worker_threads'; +import { randomBytes } from 'node:crypto'; +import isNumber from 'is-number'; +import _PropertiesReader from 'properties-reader'; +const PropertiesReader = _PropertiesReader; +import _ from 'lodash'; +import { handleHDBError } from '../utility/errors/hdbError.ts'; +import { HTTP_STATUS_CODES, HDB_ERROR_MSGS } from '../utility/errors/commonErrors.ts'; +import { server } from '../server/Server.ts'; +import { getBackupDirPath } from './configHelpers.ts'; +import { PACKAGE_ROOT } from '../utility/packageUtils.js'; + +import * as env from '../utility/environment/environmentManager.ts'; +import { applyRuntimeEnvConfig as _applyRuntimeEnvConfig } from './harperConfigEnvVars.ts'; +const applyRuntimeEnvConfig = _applyRuntimeEnvConfig; const { DATABASES_PARAM_CONFIG, CONFIG_PARAMS, CONFIG_PARAM_MAP } = hdbTerms; -const UNINIT_GET_CONFIG_ERR = 'Unable to get config value because config is uninitialized'; -const CONFIG_INIT_MSG = 'Config successfully initialized'; -const BACKUP_ERR = 'Error backing up config file'; -const EMPTY_GET_VALUE = 'Empty parameter sent to getConfigValue'; -const DEFAULT_CONFIG_FILE_PATH = path.join(PACKAGE_ROOT, 'static', hdbTerms.HDB_DEFAULT_CONFIG_FILE); +var UNINIT_GET_CONFIG_ERR = 'Unable to get config value because config is uninitialized'; +var CONFIG_INIT_MSG = 'Config successfully initialized'; +var BACKUP_ERR = 'Error backing up config file'; +var EMPTY_GET_VALUE = 'Empty parameter sent to getConfigValue'; +var DEFAULT_CONFIG_FILE_PATH = path.join(PACKAGE_ROOT, 'static', hdbTerms.HDB_DEFAULT_CONFIG_FILE); -const CONFIGURE_SUCCESS_RESPONSE = +var CONFIGURE_SUCCESS_RESPONSE = 'Configuration successfully set. You must restart Harper for new config settings to take effect.'; -const DEPRECATED_CONFIG = { +var DEPRECATED_CONFIG = { logging_rotation_retain: 'logging.rotation.retain', logging_rotation_rotate: 'logging.rotation.rotate', logging_rotation_rotateinterval: 'logging.rotation.rotateInterval', @@ -37,35 +43,33 @@ const DEPRECATED_CONFIG = { logging_rotation_workerinterval: 'logging.rotation.workerInterval', }; -let flatDefaultConfigObj; -let flatConfigObj; -let configObj; - -exports.createConfigFile = createConfigFile; -exports.getDefaultConfig = getDefaultConfig; -exports.getConfigValue = getConfigValue; -exports.initConfig = initConfig; -exports.flattenConfig = flattenConfig; -exports.updateConfigValue = updateConfigValue; -exports.updateConfigObject = updateConfigObject; -exports.getConfiguration = getConfiguration; -exports.setConfiguration = setConfiguration; -exports.readConfigFile = readConfigFile; -exports.initOldConfig = initOldConfig; -exports.getConfigFromFile = getConfigFromFile; -exports.getConfigFilePath = getConfigFilePath; -exports.addConfig = addConfig; -exports.deleteConfigFromFile = deleteConfigFromFile; -exports.getConfigObj = getConfigObj; -exports.resolvePath = resolvePath; -exports.getFlatConfigObj = getFlatConfigObj; -exports.getConfigPath = getConfigPath; - +var flatDefaultConfigObj; +var flatConfigObj; +var configObj; + +export { createConfigFile }; +export { getDefaultConfig }; +export { getConfigValue }; +export { initConfig }; +export { flattenConfig }; +export { updateConfigValue }; +export { updateConfigObject }; +export { getConfiguration }; +export { setConfiguration }; +export { readConfigFile }; +export { initOldConfig }; +export { getConfigFromFile }; +export { getConfigFilePath }; +export { addConfig }; +export { deleteConfigFromFile }; +export { getConfigObj }; +export { resolvePath }; +export { getFlatConfigObj }; +export { getConfigPath }; function resolvePath(relativePath) { if (relativePath?.startsWith('~/')) { return path.join(hdbUtils.getHomeDir(), relativePath.slice(1)); } - const env = require('../utility/environment/environmentManager.ts'); try { return path.resolve(env.getHdbBasePath(), relativePath); } catch (error) { @@ -79,7 +83,6 @@ function resolvePath(relativePath) { * @param param */ function getConfigPath(param) { - const env = require('../utility/environment/environmentManager.ts'); const value = env.get(param); if (!value || typeof value !== 'string') return value; if (value.startsWith('~/')) { @@ -184,7 +187,7 @@ function createConfigFile(args, skipFsValidation = false) { flatConfigObj = flattenConfig(configObj); // Create new config file and write config doc to it. - const hdbRoot = configDoc.getIn(['rootPath']); + const hdbRoot = configDoc.getIn(['rootPath']) as string; const configFilePath = path.join(hdbRoot, hdbTerms.HARPER_CONFIG_FILE); fs.createFileSync(configFilePath); if (configDoc.errors?.length > 0) { @@ -358,7 +361,7 @@ function initConfig(force = false) { // Validates config doc and if required sets default values for some parameters. validateConfig(configDoc); const configObj = configDoc.toJSON(); - server.config = configObj; + (server as any).config = configObj; flatConfigObj = flattenConfig(configObj); // If config has old version of logrotate enabled let user know it has been deprecated. @@ -615,7 +618,7 @@ function updateConfigValue( } if (configParam?.startsWith('threads_')) { // if threads was a number, recreate the threads object - const threadCount = configDoc.getIn(['threads']); + const threadCount = configDoc.getIn(['threads']) as any; if (threadCount >= 0) { configDoc.deleteIn(['threads']); configDoc.setIn(['threads', 'count'], threadCount); @@ -654,7 +657,7 @@ function updateConfigValue( // Validates config doc and if required sets default values for some parameters. validateConfig(configDoc); - const hdbRoot = configDoc.getIn(['rootPath']); + const hdbRoot = configDoc.getIn(['rootPath']) as string; let configFileLocation = path.join(hdbRoot, hdbTerms.HARPER_CONFIG_FILE); if (!fs.existsSync(configFileLocation) && fs.existsSync(path.join(hdbRoot, hdbTerms.HDB_CONFIG_FILE))) { configFileLocation = path.join(hdbRoot, hdbTerms.HDB_CONFIG_FILE); @@ -693,7 +696,7 @@ function backupConfigFile(configPath, hdbRoot) { } } -const PRESERVED_PROPERTIES = ['databases']; +var PRESERVED_PROPERTIES = ['databases']; /** * Flattens the JSON version of Harper config with underscores separating each parent/child key. * @param obj @@ -844,6 +847,7 @@ function readConfigFile() { } function parseYamlDoc(filePath) { + // @ts-expect-error - simpleKeys is a valid yaml option but not in the type defs return YAML.parseDocument(fs.readFileSync(filePath, 'utf8'), { simpleKeys: true }); } @@ -874,8 +878,6 @@ function applyRuntimeEnvVarConfig(configDoc, configFilePath, options = {}) { // No env vars set, skip entirely (zero overhead) if (!defaultEnvValue && !setEnvValue) return; - const { applyRuntimeEnvConfig } = require('./harperConfigEnvVars.ts'); - // Get rootPath for state file location const rootPath = configDoc.getIn(['rootPath']); if (!rootPath) { @@ -903,6 +905,7 @@ function applyRuntimeEnvVarConfig(configDoc, configFilePath, options = {}) { // Update the YAML document's contents // We update only the 'contents' property to preserve the Document instance and its methods + // @ts-expect-error - simpleKeys is a valid yaml option but not in the type defs const mergedDoc = YAML.parseDocument(YAML.stringify(configObj), { simpleKeys: true }); // Check for YAML parsing errors @@ -1002,7 +1005,7 @@ function deleteConfigFromFile(param) { const configFilePath = getConfigFilePath(hdbUtils.getPropsFilePath()); const configDoc = parseYamlDoc(configFilePath); configDoc.deleteIn(param); - const hdbRoot = configDoc.getIn(['rootPath']); + const hdbRoot = configDoc.getIn(['rootPath']) as string; const configFileLocation = path.join(hdbRoot, hdbTerms.HARPER_CONFIG_FILE); atomicWriteFile(configFileLocation, String(configDoc)); } diff --git a/config/harperConfigEnvVars.ts b/config/harperConfigEnvVars.ts index 80b0f8433..bf5d3f4b1 100644 --- a/config/harperConfigEnvVars.ts +++ b/config/harperConfigEnvVars.ts @@ -12,20 +12,17 @@ */ import type { Logger } from '../utility/logging/logger.ts'; -import * as fs from 'fs-extra'; +import fs from 'fs-extra'; import * as path from 'node:path'; import * as crypto from 'node:crypto'; -import { cloneDeep } from 'lodash'; +import _lodash from 'lodash'; +const { cloneDeep } = _lodash; import { getBackupDirPath } from './configHelpers.ts'; +import { loggerWithTag } from '../utility/logging/harper_logger.ts'; const STATE_FILE_NAME = '.harper-config-state.json'; -/** - * Get logger instance with tag - lazy loaded to avoid circular dependencies - * and ensure logger is initialized before use - */ function getLogger(): Logger { - const { loggerWithTag } = require('../utility/logging/harper_logger'); return loggerWithTag('env-config'); } diff --git a/dataLayer/SQLSearch.ts b/dataLayer/SQLSearch.ts index 3c7389e99..68270ab14 100644 --- a/dataLayer/SQLSearch.ts +++ b/dataLayer/SQLSearch.ts @@ -6,7 +6,7 @@ * process and return results by passing the raw values into the alasql SQL parser */ -import * as _ from 'lodash'; +import _ from 'lodash'; import * as alasql from 'alasql'; alasql.options.cache = false; import alasqlFunctionImporter from '../sqlTranslator/alasqlFunctionImporter.ts'; @@ -14,7 +14,7 @@ import clone from 'clone'; import RecursiveIterator from 'recursive-iterator'; import log from '../utility/logging/harper_logger.ts'; import * as commonUtils from '../utility/common_utils.ts'; -const harperBridge = require('./harperBridge/harperBridge').default; +import harperBridge from './harperBridge/harperBridge.ts'; import * as hdbTerms from '../utility/hdbTerms.ts'; import { hdbErrors } from '../utility/errors/hdbError.ts'; import { getDatabases } from '../resources/databases.ts'; diff --git a/dataLayer/SearchObject.ts b/dataLayer/SearchObject.ts index cfdd1e8b8..d45bbaa5e 100644 --- a/dataLayer/SearchObject.ts +++ b/dataLayer/SearchObject.ts @@ -25,7 +25,7 @@ class SearchObject { value, hash_attribute, get_attributes, - endValue, + endValue = undefined, reverse = false, limit = undefined, offset = undefined diff --git a/dataLayer/bulkLoad.ts b/dataLayer/bulkLoad.ts index 9a5a97d2b..21d1f780a 100644 --- a/dataLayer/bulkLoad.ts +++ b/dataLayer/bulkLoad.ts @@ -1,26 +1,32 @@ import * as insert from './insert.ts'; -import * as validator from '../validation/fileLoadValidator.ts'; -import needle from 'needle'; +import * as _validator from '../validation/fileLoadValidator.ts'; +const validator = _validator; +import _needle from 'needle'; +const needle = _needle; import * as hdbTerms from '../utility/hdbTerms.ts'; import * as hdbUtils from '../utility/common_utils.ts'; -import { handleHDBError, hdbErrors } from '../utility/errors/hdbError.ts'; +import { handleHDBError as _handleHDBError, hdbErrors } from '../utility/errors/hdbError.ts'; +const handleHDBError = _handleHDBError; import { HTTP_STATUS_CODES, HDB_ERROR_MSGS, CHECK_LOGS_WRAPPER } from '../utility/errors/commonErrors.ts'; import logger from '../utility/logging/harper_logger.ts'; import * as papaParse from 'papaparse'; -import * as fs from 'fs-extra'; +import _fs from 'fs-extra'; +const fs = _fs; import * as path from 'path'; -import { chain } from 'stream-chain'; -import StreamArray from 'stream-json/streamers/StreamArray'; -import Batch from 'stream-json/utils/Batch'; -import comp from 'stream-chain/utils/comp'; +import streamChain from 'stream-chain'; +const chain = (streamChain as any).chain ?? streamChain; +import StreamArray from 'stream-json/streamers/StreamArray.js'; +import Batch from 'stream-json/utils/Batch.js'; +import comp from 'stream-chain/utils/comp.js'; import { finished } from 'stream'; import * as env from '../utility/environment/environmentManager.ts'; import * as opFuncCaller from '../utility/OperationFunctionCaller.ts'; import * as AWSConnector from '../utility/AWS/AWSConnector.js'; import { BulkLoadFileObject, BulkLoadDataObject } from './dataObjects/BulkLoadObjects.js'; import PermissionResponseObject from '../security/data_objects/PermissionResponseObject.ts'; -import { verifyBulkLoadAttributePerms } from '../utility/operation_authorization.ts'; +import { verifyBulkLoadAttributePerms as _verifyBulkLoadAttributePerms } from '../utility/operation_authorization.ts'; +const verifyBulkLoadAttributePerms = _verifyBulkLoadAttributePerms; import { databases } from '../resources/databases.ts'; import { coerceType } from '../resources/Table.ts'; @@ -574,7 +580,7 @@ async function callPapaParse(jsonMessage) { } function createTransformMap(schema, table) { - const attributes = databases[schema][table].attributes; + const attributes = (databases[schema][table] as any).attributes; let mapOfTransforms = new Map(); // I don't know if this should be a Map, but this just makes a map of attributes with type coercions that we want for (let attribute of attributes) { if (attribute.type && !attribute.computed && !attribute.relationship) diff --git a/dataLayer/dataObjects/UpsertObject.js b/dataLayer/dataObjects/UpsertObject.ts similarity index 69% rename from dataLayer/dataObjects/UpsertObject.js rename to dataLayer/dataObjects/UpsertObject.ts index c318882c5..618be2415 100644 --- a/dataLayer/dataObjects/UpsertObject.js +++ b/dataLayer/dataObjects/UpsertObject.ts @@ -1,10 +1,13 @@ -'use strict'; -const OPERATIONS_ENUM = require('../../utility/hdbTerms.ts').OPERATIONS_ENUM; - +import { OPERATIONS_ENUM } from '../../utility/hdbTerms.ts'; /** * object representing an UPSERT operation */ class UpsertObject { + operation: string; + schema: string; + table: string; + records: any[]; + __origin: any; /** * @param {String} schema * @param {string} table @@ -20,4 +23,4 @@ class UpsertObject { } } -module.exports = UpsertObject; +export default UpsertObject; diff --git a/dataLayer/delete.ts b/dataLayer/delete.ts index 99fa6f273..96c368249 100644 --- a/dataLayer/delete.ts +++ b/dataLayer/delete.ts @@ -9,7 +9,9 @@ import { promisify, callbackify } from 'util'; import * as terms from '../utility/hdbTerms.ts'; import * as globalSchema from '../utility/globalSchema.ts'; const pGlobalSchema = promisify(globalSchema.getTableSchema); -const harperBridge = require('./harperBridge/harperBridge').default; +import _harperBridge from './harperBridge/harperBridge.ts'; +// Lazy access to handle import cycle (insert.ts and harperBridge.ts both import each other transitively). +const harperBridge: any = new Proxy({}, { get: (_, p) => (_harperBridge as any)[p] }); import { DeleteResponseObject } from './DataLayerObjects.ts'; import { handleHDBError } from '../utility/errors/hdbError.ts'; import { HDB_ERROR_MSGS, HTTP_STATUS_CODES } from '../utility/errors/commonErrors.ts'; diff --git a/dataLayer/export.ts b/dataLayer/export.ts index 1eb31c638..cd79f59ac 100644 --- a/dataLayer/export.ts +++ b/dataLayer/export.ts @@ -4,7 +4,7 @@ import * as search from './search.ts'; import * as AWSConnector from '../utility/AWS/AWSConnector.js'; import * as stream from 'stream'; import * as hdbUtils from '../utility/common_utils.ts'; -import * as fs from 'fs-extra'; +import fs from 'fs-extra'; import * as path from 'path'; import hdbLogger from '../utility/logging/harper_logger.ts'; import { promisify } from 'util'; @@ -13,7 +13,8 @@ import { handleHDBError } from '../utility/errors/hdbError.ts'; import { HDB_ERROR_MSGS, HTTP_STATUS_CODES } from '../utility/errors/commonErrors.ts'; import { streamAsJSON } from '../server/serverHelpers/JSONStream.ts'; -let { Upload } = require('@aws-sdk/lib-storage'); +import { Upload as _Upload } from '@aws-sdk/lib-storage'; +const Upload = _Upload; import { toCsvStream } from '../server/serverHelpers/contentTypes.ts'; const VALID_SEARCH_OPERATIONS = ['search_by_value', 'search_by_hash', 'sql', 'search_by_conditions']; diff --git a/dataLayer/getBackup.ts b/dataLayer/getBackup.ts index ed8b339ef..a952135e7 100644 --- a/dataLayer/getBackup.ts +++ b/dataLayer/getBackup.ts @@ -4,7 +4,7 @@ 'use strict'; -const harperBridge = require('./harperBridge/harperBridge').default; +import harperBridge from './harperBridge/harperBridge.ts'; // eslint-disable-next-line no-unused-vars import GetBackupObject from './GetBackupObject.ts'; import * as hdbUtils from '../utility/common_utils.ts'; diff --git a/dataLayer/harperBridge/ResourceBridge.ts b/dataLayer/harperBridge/ResourceBridge.ts index 7a50ddd5b..531ba8f18 100644 --- a/dataLayer/harperBridge/ResourceBridge.ts +++ b/dataLayer/harperBridge/ResourceBridge.ts @@ -1,7 +1,7 @@ import searchValidator from '../../validation/searchValidator.ts'; import { handleHDBError, ClientError, hdbErrors } from '../../utility/errors/hdbError.ts'; import { table, getDatabases, database, dropDatabase, type Table } from '../../resources/databases.ts'; -import insertUpdateValidate from './bridgeUtility/insertUpdateValidate.js'; +import insertUpdateValidate from './bridgeUtility/insertUpdateValidate.ts'; import SearchObject from '../SearchObject.ts'; import { OPERATIONS_ENUM, @@ -10,7 +10,7 @@ import { READ_AUDIT_LOG_SEARCH_TYPES_ENUM, } from '../../utility/hdbTerms.ts'; import * as signalling from '../../utility/signalling.ts'; -import { SchemaEventMsg } from '../../server/threads/itc.js'; +import { SchemaEventMsg } from '../../server/threads/itc.ts'; import { asyncSetTimeout } from '../../utility/common_utils.ts'; import { transaction } from '../../resources/transaction.ts'; import type { @@ -26,7 +26,7 @@ import { collapseData } from '../../resources/tracked.ts'; import { errorToString } from '../../utility/logging/harper_logger.ts'; import { RocksDatabase } from '@harperfast/rocksdb-js'; import { BridgeMethods } from './BridgeMethods.ts'; -import lmdbGetBackup from './lmdbBridge/lmdbMethods/lmdbGetBackup.js'; +import lmdbGetBackup from './lmdbBridge/lmdbMethods/lmdbGetBackup.ts'; import { DeleteTransactionLogsBeforeResults } from './DeleteTransactionLogsBeforeResults.ts'; import type { Readable } from 'node:stream'; @@ -202,7 +202,7 @@ export class ResourceBridge extends BridgeMethods { const { attributes } = insertUpdateValidate(upsertObj); let new_attributes; - const Table = getDatabases()[upsertObj.schema][upsertObj.table]; + const Table: any = getDatabases()[upsertObj.schema][upsertObj.table]; const context: Context = { user: upsertObj.hdb_user, expiresAt: upsertObj.expiresAt, @@ -275,7 +275,7 @@ export class ResourceBridge extends BridgeMethods { } async deleteRecords(deleteObj) { - const Table = getDatabases()[deleteObj.schema][deleteObj.table]; + const Table: any = getDatabases()[deleteObj.schema][deleteObj.table]; const context: Context = { user: deleteObj.hdb_user }; if (deleteObj.replicateTo) context.replicateTo = deleteObj.replicateTo; if (deleteObj.replicatedConfirmation) context.replicatedConfirmation = deleteObj.replicatedConfirmation; @@ -304,7 +304,7 @@ export class ResourceBridge extends BridgeMethods { * @returns {undefined} */ async deleteRecordsBefore(deleteObj) { - const Table = getDatabases()[deleteObj.schema][deleteObj.table]; + const Table: any = getDatabases()[deleteObj.schema][deleteObj.table]; if (!Table.createdTimeProperty) { throw new ClientError( `Table must have a '__createdtime__' attribute or @createdTime timestamp defined to perform this operation` diff --git a/dataLayer/harperBridge/bridgeUtility/insertUpdateValidate.js b/dataLayer/harperBridge/bridgeUtility/insertUpdateValidate.ts similarity index 87% rename from dataLayer/harperBridge/bridgeUtility/insertUpdateValidate.js rename to dataLayer/harperBridge/bridgeUtility/insertUpdateValidate.ts index 8fd159da1..ba8e693dd 100644 --- a/dataLayer/harperBridge/bridgeUtility/insertUpdateValidate.js +++ b/dataLayer/harperBridge/bridgeUtility/insertUpdateValidate.ts @@ -1,11 +1,9 @@ -'use strict'; +import * as hdbUtils from '../../../utility/common_utils.ts'; +import log from '../../../utility/logging/harper_logger.ts'; +import { getDatabases } from '../../../resources/databases.ts'; +import { ClientError } from '../../../utility/errors/hdbError.ts'; -const hdbUtils = require('../../../utility/common_utils.ts'); -const log = require('../../../utility/logging/harper_logger.ts'); -const { getDatabases } = require('../../../resources/databases.ts'); -const { ClientError } = require('../../../utility/errors/hdbError.ts'); - -module.exports = insertUpdateValidate; +export default insertUpdateValidate; //IMPORTANT - This code is the same code as the async validation() function in dataLayer/insert - make sure any changes // below are also made there. This is to resolve a circular dependency. @@ -32,7 +30,7 @@ function insertUpdateValidate(writeObject) { throw new ClientError('records must be an array'); } - let schemaTable = getDatabases()[writeObject.schema]?.[writeObject.table]; + let schemaTable: any = getDatabases()[writeObject.schema]?.[writeObject.table]; if (hdbUtils.isEmpty(schemaTable)) { throw new ClientError(`could not retrieve schema:${writeObject.schema} and table ${writeObject.table}`); } diff --git a/dataLayer/harperBridge/harperBridge.ts b/dataLayer/harperBridge/harperBridge.ts index bb697a182..051910d69 100644 --- a/dataLayer/harperBridge/harperBridge.ts +++ b/dataLayer/harperBridge/harperBridge.ts @@ -2,8 +2,11 @@ import { ResourceBridge } from './ResourceBridge.ts'; import * as envMngr from '../../utility/environment/environmentManager.ts'; -envMngr.initSync(); - +try { + envMngr.initSync(); +} catch { + /* tolerate ESM cycle TDZ; bin entry will re-call later */ +} let harperBridge; // ResourceBridge /** diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateAttribute.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateAttribute.ts similarity index 75% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateAttribute.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateAttribute.ts index 513f6df3b..474173508 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateAttribute.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateAttribute.ts @@ -1,22 +1,17 @@ -'use strict'; - -const hdbTerms = require('../../../../utility/hdbTerms.ts'); -const environmentUtility = require('../../../../utility/lmdb/environmentUtility.ts'); -const writeUtility = require('../../../../utility/lmdb/writeUtility.ts'); -const { getSystemSchemaPath, getSchemaPath } = require('../lmdbUtility/initializePaths.js'); -const { validateBySchema } = require('../../../../validation/validationWrapper.ts'); -const Joi = require('joi'); -const LMDBCreateAttributeObject = - require('../lmdbUtility/LMDBCreateAttributeObject.js').default || - require('../lmdbUtility/LMDBCreateAttributeObject.js'); -const returnObject = require('../../bridgeUtility/insertUpdateReturnObj.js'); -const { handleHDBError, hdbErrors, ClientError } = require('../../../../utility/errors/hdbError.ts'); -const hdbUtils = require('../../../../utility/common_utils.ts'); +import * as hdbTerms from '../../../../utility/hdbTerms.ts'; +import * as environmentUtility from '../../../../utility/lmdb/environmentUtility.ts'; +import * as writeUtility from '../../../../utility/lmdb/writeUtility.ts'; +import { getSystemSchemaPath, getSchemaPath } from '../lmdbUtility/initializePaths.ts'; +import { validateBySchema } from '../../../../validation/validationWrapper.ts'; +import Joi from 'joi'; +import LMDBCreateAttributeObject from '../lmdbUtility/LMDBCreateAttributeObject.ts'; +import returnObject from '../../bridgeUtility/insertUpdateReturnObj.js'; +import { handleHDBError, hdbErrors, ClientError } from '../../../../utility/errors/hdbError.ts'; +import * as hdbUtils from '../../../../utility/common_utils.ts'; const { HTTP_STATUS_CODES } = hdbErrors; - const ACTION = 'inserted'; -module.exports = lmdbCreateAttribute; +export default lmdbCreateAttribute; /** * First adds the attribute to the system attribute table, then creates the dbi. @@ -100,10 +95,10 @@ async function lmdbCreateAttribute(createAttributeObj) { hdbAttributeEnv, // I'm not sure what else to do with these for now, but I do want to eslint to check the rest of the codebase // for undefined vars. - WSM 2025-11-26 - // eslint-disable-next-line no-undef - HDB_TABLE_INFO.hash_attribute, - // eslint-disable-next-line no-undef - hdbAttributeAttributes, + // @ts-expect-error - HDB_TABLE_INFO is a legacy global injected at runtime + HDB_TABLE_INFO.hash_attribute, // eslint-disable-line no-undef + // @ts-expect-error - hdbAttributeAttributes is a legacy global injected at runtime + hdbAttributeAttributes, // eslint-disable-line no-undef [record] ); diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateRecords.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateRecords.ts similarity index 62% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateRecords.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateRecords.ts index e9a4a085d..7913cb513 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateRecords.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateRecords.ts @@ -1,19 +1,17 @@ -'use strict'; - -const insertUpdateValidate = require('../../bridgeUtility/insertUpdateValidate.js'); +import insertUpdateValidate from '../../bridgeUtility/insertUpdateValidate.ts'; // eslint-disable-next-line no-unused-vars -const InsertObject = require('../../../InsertObject.ts').default || require('../../../InsertObject.ts'); -const hdbTerms = require('../../../../utility/hdbTerms.ts'); -const lmdbProcessRows = require('../lmdbUtility/lmdbProcessRows.js'); -const lmdbInsertRecords = require('../../../../utility/lmdb/writeUtility.ts').insertRecords; -const environmentUtility = require('../../../../utility/lmdb/environmentUtility.ts'); -const logger = require('../../../../utility/logging/harper_logger.ts'); +import InsertObject from '../../../InsertObject.ts'; +import * as hdbTerms from '../../../../utility/hdbTerms.ts'; +import lmdbProcessRows from '../lmdbUtility/lmdbProcessRows.ts'; +import { insertRecords as lmdbInsertRecords } from '../../../../utility/lmdb/writeUtility.ts'; +import * as environmentUtility from '../../../../utility/lmdb/environmentUtility.ts'; +import logger from '../../../../utility/logging/harper_logger.ts'; -const lmdbCheckNewAttributes = require('../lmdbUtility/lmdbCheckForNewAttributes.js'); -const { getSchemaPath } = require('../lmdbUtility/initializePaths.js'); -const writeTransaction = require('../lmdbUtility/lmdbWriteTransaction.js'); +import lmdbCheckNewAttributes from '../lmdbUtility/lmdbCheckForNewAttributes.ts'; +import { getSchemaPath } from '../lmdbUtility/initializePaths.ts'; +import writeTransaction from '../lmdbUtility/lmdbWriteTransaction.ts'; -module.exports = lmdbCreateRecords; +export default lmdbCreateRecords; /** * Orchestrates the insertion of data into LMDB and the creation of new attributes/dbis @@ -23,7 +21,7 @@ module.exports = lmdbCreateRecords; */ async function lmdbCreateRecords(insertObj) { try { - let { schemaTable, attributes } = insertUpdateValidate(insertObj); + let { schema_table: schemaTable, attributes } = insertUpdateValidate(insertObj); lmdbProcessRows(insertObj, attributes, schemaTable.hash_attribute); diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateSchema.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateSchema.ts similarity index 54% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateSchema.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateSchema.ts index 501a7200c..dc3404367 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateSchema.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateSchema.ts @@ -1,12 +1,11 @@ -'use strict'; +import * as hdbTerms from '../../../../utility/hdbTerms.ts'; +import _lmdbCreateRecords from './lmdbCreateRecords.ts'; +const lmdbCreateRecords = _lmdbCreateRecords; +import InsertObject from '../../../InsertObject.ts'; +import fs from 'fs-extra'; +import { getSchemaPath } from '../lmdbUtility/initializePaths.ts'; -const hdbTerms = require('../../../../utility/hdbTerms.ts'); -const lmdbCreateRecords = require('./lmdbCreateRecords.js'); -const InsertObject = require('../../../InsertObject.ts').default || require('../../../InsertObject.ts'); -const fs = require('fs-extra'); -const { getSchemaPath } = require('../lmdbUtility/initializePaths.js'); - -module.exports = lmdbCreateSchema; +export default lmdbCreateSchema; /** * creates the meta data for the schema diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateTable.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateTable.ts similarity index 70% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateTable.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateTable.ts index 1f95a6970..5ede0ab25 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateTable.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbCreateTable.ts @@ -1,17 +1,14 @@ -'use strict'; +import * as hdbTerms from '../../../../utility/hdbTerms.ts'; +import * as environmentUtility from '../../../../utility/lmdb/environmentUtility.ts'; +import * as _writeUtility from '../../../../utility/lmdb/writeUtility.ts'; +const writeUtility = _writeUtility; +import { getSystemSchemaPath, getSchemaPath } from '../lmdbUtility/initializePaths.ts'; +import lmdbCreateAttribute from './lmdbCreateAttribute.ts'; +import LMDBCreateAttributeObject from '../lmdbUtility/LMDBCreateAttributeObject.ts'; +import log from '../../../../utility/logging/harper_logger.ts'; +import createTxnEnvironments from '../lmdbUtility/lmdbCreateTransactionsAuditEnvironment.ts'; -const hdbTerms = require('../../../../utility/hdbTerms.ts'); -const environmentUtility = require('../../../../utility/lmdb/environmentUtility.ts'); -const writeUtility = require('../../../../utility/lmdb/writeUtility.ts'); -const { getSystemSchemaPath, getSchemaPath } = require('../lmdbUtility/initializePaths.js'); -const lmdbCreateAttribute = require('./lmdbCreateAttribute.js'); -const LMDBCreateAttributeObject = - require('../lmdbUtility/LMDBCreateAttributeObject.js').default || - require('../lmdbUtility/LMDBCreateAttributeObject.js'); -const log = require('../../../../utility/logging/harper_logger.ts'); -const createTxnEnvironments = require('../lmdbUtility/lmdbCreateTransactionsAuditEnvironment.js'); - -module.exports = lmdbCreateTable; +export default lmdbCreateTable; /** * Writes new table data to the system tables creates the environment file and creates two datastores to track created and updated @@ -60,10 +57,10 @@ async function lmdbCreateTable(tableSystemData, tableCreateObj) { hdbTableEnv, // I'm not sure what else to do with these for now, but I do want to eslint to check the rest of the codebase // for undefined vars. - WSM 2025-11-26 - // eslint-disable-next-line no-undef - HDB_TABLE_INFO.hash_attribute, - // eslint-disable-next-line no-undef - hdbTableAttributes, + // @ts-expect-error - HDB_TABLE_INFO is a legacy global injected at runtime + HDB_TABLE_INFO.hash_attribute, // eslint-disable-line no-undef + // @ts-expect-error - hdbTableAttributes is a legacy global injected at runtime + hdbTableAttributes, // eslint-disable-line no-undef [tableSystemData] ); //create attributes for hash attribute created/updated time stamps diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteAuditLogsBefore.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteAuditLogsBefore.ts similarity index 81% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteAuditLogsBefore.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteAuditLogsBefore.ts index 0d2f14df3..5d4024876 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteAuditLogsBefore.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteAuditLogsBefore.ts @@ -1,20 +1,17 @@ -'use strict'; - -const environmentUtility = require('../../../../utility/lmdb/environmentUtility.ts'); -const { getTransactionAuditStorePath } = require('../lmdbUtility/initializePaths.js'); +import * as environmentUtility from '../../../../utility/lmdb/environmentUtility.ts'; +import { getTransactionAuditStorePath } from '../lmdbUtility/initializePaths.ts'; // eslint-disable-next-line no-unused-vars -const DeleteBeforeObject = - require('../../../DeleteBeforeObject.ts').default || require('../../../DeleteBeforeObject.ts'); -const lmdbTerms = require('../../../../utility/lmdb/terms.ts'); -const hdbUtils = require('../../../../utility/common_utils.ts'); -const DeleteAuditLogsBeforeResults = require('./DeleteAuditLogsBeforeResults.js'); -const promisify = require('util').promisify; +import DeleteBeforeObject from '../../../DeleteBeforeObject.ts'; +import * as lmdbTerms from '../../../../utility/lmdb/terms.ts'; +import * as hdbUtils from '../../../../utility/common_utils.ts'; +import DeleteAuditLogsBeforeResults from './DeleteAuditLogsBeforeResults.js'; +import { promisify } from 'util'; const pSettimeout = promisify(setTimeout); const BATCH_SIZE = 10000; const SLEEP_TIME_MS = 100; -module.exports = deleteAuditLogsBefore; +export default deleteAuditLogsBefore; /** * diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteRecords.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteRecords.ts similarity index 83% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteRecords.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteRecords.ts index fc9c872e3..1f771cb1a 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteRecords.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDeleteRecords.ts @@ -1,13 +1,11 @@ -'use strict'; +import * as hdbUtils from '../../../../utility/common_utils.ts'; +import * as deleteUtility from '../../../../utility/lmdb/deleteUtility.ts'; +import * as environmentUtility from '../../../../utility/lmdb/environmentUtility.ts'; +import { getSchemaPath } from '../lmdbUtility/initializePaths.ts'; +import writeTransaction from '../lmdbUtility/lmdbWriteTransaction.ts'; +import logger from '../../../../utility/logging/harper_logger.ts'; -const hdbUtils = require('../../../../utility/common_utils.ts'); -const deleteUtility = require('../../../../utility/lmdb/deleteUtility.ts'); -const environmentUtility = require('../../../../utility/lmdb/environmentUtility.ts'); -const { getSchemaPath } = require('../lmdbUtility/initializePaths.js'); -const writeTransaction = require('../lmdbUtility/lmdbWriteTransaction.js'); -const logger = require('../../../../utility/logging/harper_logger.ts'); - -module.exports = lmdbDeleteRecords; +export default lmdbDeleteRecords; /** * Deletes a full table row at a certain hash. @@ -76,7 +74,7 @@ async function lmdbDeleteRecords(deleteObj, writeToTxnLog = true) { * @param {number} txnTime - the transaction timestamp * @returns {{skipped_hashes: [], deleted_hashes: [], message: string}} */ -function createDeleteResponse(deleted, skipped, txnTime) { +function createDeleteResponse(deleted, skipped, txnTime?) { let total = deleted.length + skipped.length; let plural = total === 1 ? 'record' : 'records'; diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropAttribute.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropAttribute.ts similarity index 81% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropAttribute.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropAttribute.ts index de7445992..30d344780 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropAttribute.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropAttribute.ts @@ -1,19 +1,21 @@ -'use strict'; - -const SearchObject = require('../../../SearchObject.ts').default || require('../../../SearchObject.ts'); -const DeleteObject = require('../../../DeleteObject.ts').default || require('../../../DeleteObject.ts'); +import SearchObject from '../../../SearchObject.ts'; +import DeleteObject from '../../../DeleteObject.ts'; // eslint-disable-next-line no-unused-vars -const DropAttributeObject = - require('../../../DropAttributeObject.ts').default || require('../../../DropAttributeObject.ts'); -const hdbTerms = require('../../../../utility/hdbTerms.ts'); -const commonUtils = require('../../../../utility/common_utils.ts'); -const environmentUtility = require('../../../../utility/lmdb/environmentUtility.ts'); -const systemSchema = require('../../../../json/systemSchema.json'); -const searchByValue = require('./lmdbSearchByValue.js'); -const deleteRecords = require('./lmdbDeleteRecords.js'); -const { getSchemaPath } = require('../lmdbUtility/initializePaths.js'); +import DropAttributeObject from '../../../DropAttributeObject.ts'; +import * as hdbTerms from '../../../../utility/hdbTerms.ts'; +import * as commonUtils from '../../../../utility/common_utils.ts'; +import * as environmentUtility from '../../../../utility/lmdb/environmentUtility.ts'; +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { PACKAGE_ROOT } from '../../../../utility/packageUtils.js'; +const systemSchema: Record = JSON.parse( + readFileSync(join(PACKAGE_ROOT, 'json/systemSchema.json'), 'utf-8') +); +import searchByValue from './lmdbSearchByValue.ts'; +import deleteRecords from './lmdbDeleteRecords.ts'; +import { getSchemaPath } from '../lmdbUtility/initializePaths.ts'; -module.exports = lmdbDropAttribute; +export default lmdbDropAttribute; /** * First deletes the attribute/dbi from lmdb then removes its record from system table diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropSchema.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropSchema.ts similarity index 71% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropSchema.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropSchema.ts index 7191f91f1..5e206c2cb 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropSchema.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropSchema.ts @@ -1,20 +1,16 @@ -'use strict'; - -const fs = require('fs-extra'); -const SearchObject = require('../../../SearchObject.ts').default || require('../../../SearchObject.ts'); -const SearchByHashObject = - require('../../../SearchByHashObject.ts').default || require('../../../SearchByHashObject.ts'); -const DeleteObject = require('../../../DeleteObject.ts').default || require('../../../DeleteObject.ts'); -const dropTable = require('./lmdbDropTable.js'); -const deleteRecords = require('./lmdbDeleteRecords.js'); -const getDataByHash = require('./lmdbGetDataByHash.js'); -const searchDataByValue = require('./lmdbSearchByValue.js'); -const hdbTerms = require('../../../../utility/hdbTerms.ts'); -const { getSchemaPath } = require('../lmdbUtility/initializePaths.js'); -const { handleHDBError, hdbErrors } = require('../../../../utility/errors/hdbError.ts'); +import fs from 'fs-extra'; +import SearchObject from '../../../SearchObject.ts'; +import SearchByHashObject from '../../../SearchByHashObject.ts'; +import DeleteObject from '../../../DeleteObject.ts'; +import dropTable from './lmdbDropTable.ts'; +import deleteRecords from './lmdbDeleteRecords.ts'; +import getDataByHash from './lmdbGetDataByHash.ts'; +import searchDataByValue from './lmdbSearchByValue.ts'; +import * as hdbTerms from '../../../../utility/hdbTerms.ts'; +import { getSchemaPath } from '../lmdbUtility/initializePaths.ts'; +import { handleHDBError, hdbErrors } from '../../../../utility/errors/hdbError.ts'; const { HDB_ERROR_MSGS, HTTP_STATUS_CODES } = hdbErrors; - -module.exports = lmdbDropSchema; +export default lmdbDropSchema; /** * deletes all environment files under the schema folder, deletes all schema/table/attribute meta data from system @@ -36,7 +32,7 @@ async function lmdbDropSchema(dropSchemaObj) { [hdbTerms.SYSTEM_DEFAULT_ATTRIBUTE_NAMES.ATTR_NAME_KEY] ); - let tables = Array.from(await searchDataByValue(tableSearchObj)); + let tables: any[] = Array.from(await searchDataByValue(tableSearchObj)); for (let x = 0; x < tables.length; x++) { const deleteTableObj = { diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropTable.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropTable.ts similarity index 79% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropTable.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropTable.ts index 8573271d1..fca3ae35d 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropTable.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbDropTable.ts @@ -1,16 +1,16 @@ -'use strict'; +import SearchObject from '../../../SearchObject.ts'; +import DeleteObject from '../../../DeleteObject.ts'; +import _searchByValue from './lmdbSearchByValue.ts'; +const searchByValue = _searchByValue; +import _deleteRecords from './lmdbDeleteRecords.ts'; +const deleteRecords = _deleteRecords; +import * as hdbTerms from '../../../../utility/hdbTerms.ts'; +import * as hdbUtils from '../../../../utility/common_utils.ts'; +import * as environmentUtility from '../../../../utility/lmdb/environmentUtility.ts'; +import { getTransactionAuditStorePath, getSchemaPath } from '../lmdbUtility/initializePaths.ts'; +import log from '../../../../utility/logging/harper_logger.ts'; -const SearchObject = require('../../../SearchObject.ts').default || require('../../../SearchObject.ts'); -const DeleteObject = require('../../../DeleteObject.ts').default || require('../../../DeleteObject.ts'); -const searchByValue = require('./lmdbSearchByValue.js'); -const deleteRecords = require('./lmdbDeleteRecords.js'); -const hdbTerms = require('../../../../utility/hdbTerms.ts'); -const hdbUtils = require('../../../../utility/common_utils.ts'); -const environmentUtility = require('../../../../utility/lmdb/environmentUtility.ts'); -const { getTransactionAuditStorePath, getSchemaPath } = require('../lmdbUtility/initializePaths.js'); -const log = require('../../../../utility/logging/harper_logger.ts'); - -module.exports = lmdbDropTable; +export default lmdbDropTable; /** * Calls drops the table, all of it's attribute & deletes the environment @@ -68,7 +68,7 @@ async function deleteAttributesFromSystem(dropTableObj) { [hdbTerms.SYSTEM_DEFAULT_ATTRIBUTE_NAMES.ATTR_ID_KEY] ); - let searchResult = Array.from(await searchByValue(searchObj)); + let searchResult: any[] = Array.from(await searchByValue(searchObj)); let deleteIds = []; for (let x = 0; x < searchResult.length; x++) { @@ -109,7 +109,7 @@ async function dropTableFromSystem(dropTableObj) { let searchResult; let deleteTable; try { - searchResult = Array.from(await searchByValue(searchObj)); + searchResult = Array.from(await searchByValue(searchObj)) as any[]; } catch (err) { throw err; } diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbFlush.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbFlush.ts similarity index 76% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbFlush.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbFlush.ts index 3ca34e726..d0aa6713c 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbFlush.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbFlush.ts @@ -1,13 +1,6 @@ -'use strict'; - -const { getSchemaPath } = require('../lmdbUtility/initializePaths.js'); -const environmentUtility = require('../../../../utility/lmdb/environmentUtility.ts'); - -module.exports = { - flush, - resetReadTxn, -}; - +import { getSchemaPath } from '../lmdbUtility/initializePaths.ts'; +import * as environmentUtility from '../../../../utility/lmdb/environmentUtility.ts'; +export { flush, resetReadTxn }; /** * This is wrapper for sync/flush to disk * @param schema diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetBackup.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetBackup.ts similarity index 78% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetBackup.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetBackup.ts index e95a0d651..8f31829f6 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetBackup.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetBackup.ts @@ -1,15 +1,13 @@ -'use strict'; +import { Readable } from 'stream'; +import { getDatabases } from '../../../../resources/databases.ts'; +import { readSync, openSync, createReadStream } from 'fs'; +import { open } from 'lmdb'; +import { OpenDBIObject } from '../../../../utility/lmdb/OpenDBIObject.ts'; +import OpenEnvironmentObject from '../../../../utility/lmdb/OpenEnvironmentObject.ts'; +import { AUDIT_STORE_OPTIONS } from '../../../../resources/auditStore.ts'; +import { INTERNAL_DBIS_NAME, AUDIT_STORE_NAME } from '../../../../utility/lmdb/terms.ts'; -const { Readable } = require('stream'); -const { getDatabases } = require('../../../../resources/databases.ts'); -const { readSync, openSync, createReadStream } = require('fs'); -const { open } = require('lmdb'); -const { OpenDBIObject } = require('../../../../utility/lmdb/OpenDBIObject.ts'); -const OpenEnvironmentObject = require('../../../../utility/lmdb/OpenEnvironmentObject.ts'); -const { AUDIT_STORE_OPTIONS } = require('../../../../resources/auditStore.ts'); -const { INTERNAL_DBIS_NAME, AUDIT_STORE_NAME } = require('../../../../utility/lmdb/terms.ts'); - -module.exports = getBackup; +export default getBackup; const META_SIZE = 32768; const DELAY_ITERATIONS = 100; /** @@ -24,7 +22,7 @@ async function getBackup(getBackupObj) { let tables = getBackupObj.tables || (getBackupObj.table && [getBackupObj.table]); if (tables) { // if tables are specified, we have to copy the database with just the specified tables and then stream that - let tableClass = database[tables[0]]; + let tableClass: any = database[tables[0]]; if (!tableClass) throw new Error(`Can not find table ${tables[0]}`); // we use the attribute store to drive this process, finding the right stores to duplicate let attributeStore = tableClass.dbisDB; @@ -66,7 +64,7 @@ async function getBackup(getBackupObj) { await copyDatabase(AUDIT_STORE_NAME, { ...AUDIT_STORE_OPTIONS }); } await resolution; - let stream = createReadStream(backupRoot.path); + let stream: any = createReadStream((backupRoot as any).path); stream.headers = getHeaders(); stream.on('close', () => { readTxn.done(); @@ -74,21 +72,21 @@ async function getBackup(getBackupObj) { }); return stream; } - const firstTable = database[Object.keys(database)[0]]; - const store = firstTable.primaryStore; + const firstTable: any = database[Object.keys(database)[0]]; + const store: any = firstTable.primaryStore; - let fd = openSync(store.path); + let fd = openSync(store.path, 'r'); return store.transaction(() => { let metaBuffers = Buffer.alloc(META_SIZE); - readSync(fd, metaBuffers, 0, META_SIZE); // sync, need to do this as fast as possible since we are in a write txn + readSync(fd, metaBuffers, 0, META_SIZE, null); // sync, need to do this as fast as possible since we are in a write txn store.resetReadTxn(); // make sure we are not using a cached read transaction, force a fresh one let readTxn = store.useReadTransaction(); // this guarantees the current transaction is preserved in the backup // renew is necessary because normally renew is actually lazily called on the next db operation, but // we are not performing any db operations readTxn.renew(); // create a file stream that starts after the meta area - let fileStream = createReadStream(null, { fd, start: META_SIZE }); - let stream = new Readable.from( + let fileStream = createReadStream(null as any, { fd, start: META_SIZE }); + let stream: any = Readable.from( (async function* () { yield metaBuffers; // return the meta area that was frozen inside the write transaction for await (const chunk of fileStream) { diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByHash.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByHash.ts similarity index 75% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByHash.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByHash.ts index aa6d05980..cb0766229 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByHash.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByHash.ts @@ -1,9 +1,7 @@ -'use strict'; +import * as searchUtility from '../../../../utility/lmdb/searchUtility.ts'; +import hashSearchInit from '../lmdbUtility/initializeHashSearch.ts'; -const searchUtility = require('../../../../utility/lmdb/searchUtility.ts'); -const hashSearchInit = require('../lmdbUtility/initializeHashSearch.js'); - -module.exports = lmdbGetDataByHash; +export default lmdbGetDataByHash; /** * fetches records by their hash values and returns a map of the results diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByValue.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByValue.ts similarity index 64% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByValue.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByValue.ts index e5b6abd26..03531c93f 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByValue.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbGetDataByValue.ts @@ -1,12 +1,8 @@ -'use strict'; - -const searchValidator = - require('../../../../validation/searchValidator.ts').default || require('../../../../validation/searchValidator.ts'); -const commonUtils = require('../../../../utility/common_utils.ts'); -const hdbTerms = require('../../../../utility/hdbTerms.ts'); -const lmdbSearch = require('../lmdbUtility/lmdbSearch.js'); - -module.exports = lmdbGetDataByValue; +import searchValidator from '../../../../validation/searchValidator.ts'; +import * as commonUtils from '../../../../utility/common_utils.ts'; +import * as hdbTerms from '../../../../utility/hdbTerms.ts'; +import * as lmdbSearch from '../lmdbUtility/lmdbSearch.ts'; +export default lmdbGetDataByValue; /** * gets records by value returns a map of objects diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbReadAuditLog.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbReadAuditLog.ts similarity index 88% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbReadAuditLog.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbReadAuditLog.ts index 30f91654f..b514326ce 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbReadAuditLog.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbReadAuditLog.ts @@ -1,15 +1,13 @@ -'use strict'; +import * as environmentUtility from '../../../../utility/lmdb/environmentUtility.ts'; +import * as lmdbTerms from '../../../../utility/lmdb/terms.ts'; +import * as hdbTerms from '../../../../utility/hdbTerms.ts'; +import * as hdbUtils from '../../../../utility/common_utils.ts'; +import { getTransactionAuditStorePath } from '../lmdbUtility/initializePaths.ts'; +import * as searchUtility from '../../../../utility/lmdb/searchUtility.ts'; +import LMDBTransactionObject from '../lmdbUtility/LMDBTransactionObject.js'; +import log from '../../../../utility/logging/harper_logger.ts'; -const environmentUtility = require('../../../../utility/lmdb/environmentUtility.ts'); -const lmdbTerms = require('../../../../utility/lmdb/terms.ts'); -const hdbTerms = require('../../../../utility/hdbTerms.ts'); -const hdbUtils = require('../../../../utility/common_utils.ts'); -const { getTransactionAuditStorePath } = require('../lmdbUtility/initializePaths.js'); -const searchUtility = require('../../../../utility/lmdb/searchUtility.ts'); -const LMDBTransactionObject = require('../lmdbUtility/LMDBTransactionObject.js'); -const log = require('../../../../utility/logging/harper_logger.ts'); - -module.exports = readAuditLog; +export default readAuditLog; /** * function execute the readTransactionLog operation @@ -67,7 +65,7 @@ function searchTransactionsByTimestamp(env, timestamps = [0, Date.now()]) { return timestampDbi .getRange({ start: timestamps[0], end: nextValue }) - .map(({ value }) => Object.assign(new LMDBTransactionObject(), value)); + .map(({ value }) => Object.assign(Object.create(LMDBTransactionObject.prototype), value)); } /** @@ -193,7 +191,7 @@ function batchSearchTransactions(env, ids) { try { let value = timestampDbi.get(ids[x]); if (value) { - let txnRecord = Object.assign(new LMDBTransactionObject(), value); + let txnRecord = Object.assign(Object.create(LMDBTransactionObject.prototype), value); results.push(txnRecord); } } catch (e) { diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByConditions.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByConditions.ts similarity index 83% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByConditions.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByConditions.ts index 512c78f2f..965dbec48 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByConditions.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByConditions.ts @@ -1,23 +1,17 @@ -'use strict'; - -// eslint-disable-next-line no-unused-vars -const { SearchByConditionsObject, SearchCondition } = - require('../../../SearchByConditionsObject.ts').default || require('../../../SearchByConditionsObject.ts'); -const SearchObject = require('../../../SearchObject.ts').default || require('../../../SearchObject.ts'); -const searchValidator = - require('../../../../validation/searchValidator.ts').default || require('../../../../validation/searchValidator.ts'); -const searchUtility = require('../../../../utility/lmdb/searchUtility.ts'); -const lmdbTerms = require('../../../../utility/lmdb/terms.ts'); -const lmdb_search = require('../lmdbUtility/lmdbSearch.js'); -const cursorFunctions = require('../../../../utility/lmdb/searchCursorFunctions.ts'); -const _ = require('lodash'); -const { getSchemaPath } = require('../lmdbUtility/initializePaths.js'); -const environmentUtility = require('../../../../utility/lmdb/environmentUtility.ts'); -const { handleHDBError, hdbErrors } = require('../../../../utility/errors/hdbError.ts'); +import SearchObject from '../../../SearchObject.ts'; +import searchValidator from '../../../../validation/searchValidator.ts'; +import * as searchUtility from '../../../../utility/lmdb/searchUtility.ts'; +import * as lmdbTerms from '../../../../utility/lmdb/terms.ts'; +import * as lmdb_search from '../lmdbUtility/lmdbSearch.ts'; +import * as cursorFunctions from '../../../../utility/lmdb/searchCursorFunctions.ts'; +import _ from 'lodash'; +import { getSchemaPath } from '../lmdbUtility/initializePaths.ts'; +import * as environmentUtility from '../../../../utility/lmdb/environmentUtility.ts'; +import { handleHDBError, hdbErrors } from '../../../../utility/errors/hdbError.ts'; const { HTTP_STATUS_CODES } = hdbErrors; const RANGE_ESTIMATE = 100000000; -module.exports = lmdbSearchByConditions; +export default lmdbSearchByConditions; /** * gets records by conditions - returns array of Objects @@ -81,7 +75,7 @@ async function lmdbSearchByConditions(searchObject) { // get the intersection of condition searches by using the indexed query for the first condition // and then filtering by all subsequent conditions let primaryDbi = env.dbis[tableInfo.hash_attribute]; - let filters = sortedConditions.slice(1).map(lmdb_search.filterByType); + let filters: any[] = sortedConditions.slice(1).map(lmdb_search.filterByType as any); let filtersLength = filters.length; let fetchAttributes = searchUtility.setGetWholeRowAttributes(env, searchObject.get_attributes); records = ids.map((id) => primaryDbi.get(id, { transaction, lazy: true })); diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByHash.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByHash.ts similarity index 69% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByHash.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByHash.ts index 3be46a425..f28aa8f41 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByHash.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByHash.ts @@ -1,9 +1,7 @@ -'use strict'; +import * as searchUtility from '../../../../utility/lmdb/searchUtility.ts'; +import hashSearchInit from '../lmdbUtility/initializeHashSearch.ts'; -const searchUtility = require('../../../../utility/lmdb/searchUtility.ts'); -const hashSearchInit = require('../lmdbUtility/initializeHashSearch.js'); - -module.exports = lmdbSearchByHash; +export default lmdbSearchByHash; /** * fetches records by their hash values and returns an Array of the results diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByValue.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByValue.ts similarity index 54% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByValue.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByValue.ts index 45d0303b1..a9e4f2260 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByValue.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbSearchByValue.ts @@ -1,14 +1,10 @@ -'use strict'; - // eslint-disable-next-line no-unused-vars -const SearchObject = require('../../../SearchObject.ts').default || require('../../../SearchObject.ts'); -const searchValidator = - require('../../../../validation/searchValidator.ts').default || require('../../../../validation/searchValidator.ts'); -const commonUtils = require('../../../../utility/common_utils.ts'); -const hdbTerms = require('../../../../utility/hdbTerms.ts'); -const lmdb_search = require('../lmdbUtility/lmdbSearch.js'); - -module.exports = lmdbSearchByValue; +import SearchObject from '../../../SearchObject.ts'; +import searchValidator from '../../../../validation/searchValidator.ts'; +import * as commonUtils from '../../../../utility/common_utils.ts'; +import * as hdbTerms from '../../../../utility/hdbTerms.ts'; +import * as lmdb_search from '../lmdbUtility/lmdbSearch.ts'; +export default lmdbSearchByValue; /** * gets records by value - returns array of Objects @@ -16,7 +12,7 @@ module.exports = lmdbSearchByValue; * @param {hdbTerms.VALUE_SEARCH_COMPARATORS} [comparator] * @returns {Promise<{}|{}[]>} */ -async function lmdbSearchByValue(searchObject, comparator) { +async function lmdbSearchByValue(searchObject, comparator?) { let comparatorSearch = !commonUtils.isEmpty(comparator); if (comparatorSearch && hdbTerms.VALUE_SEARCH_COMPARATORS_REVERSE_LOOKUP[comparator] === undefined) { throw new Error(`Value search comparator - ${comparator} - is not valid`); diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbTransaction.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbTransaction.ts similarity index 74% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbTransaction.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbTransaction.ts index 9541156ea..d2c885596 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbTransaction.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbTransaction.ts @@ -1,11 +1,6 @@ -'use strict'; - -const { database } = require('../../../../resources/databases.ts'); - -module.exports = { - writeTransaction, -}; +import { database } from '../../../../resources/databases.ts'; +export { writeTransaction }; /** * This is wrapper for write transactions, ensuring that all reads and writes within the callback occur atomically * @param schema diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpdateRecords.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpdateRecords.ts similarity index 64% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpdateRecords.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpdateRecords.ts index 2b2dfa8c4..1e0141242 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpdateRecords.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpdateRecords.ts @@ -1,16 +1,14 @@ -'use strict'; - -const insertUpdateValidate = require('../../bridgeUtility/insertUpdateValidate.js'); -const lmdbProcessRows = require('../lmdbUtility/lmdbProcessRows.js'); -const lmdbCheckNewAttributes = require('../lmdbUtility/lmdbCheckForNewAttributes.js'); -const hdbTerms = require('../../../../utility/hdbTerms.ts'); -const lmdb_update_records = require('../../../../utility/lmdb/writeUtility.ts').updateRecords; -const environmentUtility = require('../../../../utility/lmdb/environmentUtility.ts'); -const { getSchemaPath } = require('../lmdbUtility/initializePaths.js'); -const writeTransaction = require('../lmdbUtility/lmdbWriteTransaction.js'); -const logger = require('../../../../utility/logging/harper_logger.ts'); - -module.exports = lmdbUpdateRecords; +import insertUpdateValidate from '../../bridgeUtility/insertUpdateValidate.ts'; +import lmdbProcessRows from '../lmdbUtility/lmdbProcessRows.ts'; +import lmdbCheckNewAttributes from '../lmdbUtility/lmdbCheckForNewAttributes.ts'; +import * as hdbTerms from '../../../../utility/hdbTerms.ts'; +import { updateRecords as lmdb_update_records } from '../../../../utility/lmdb/writeUtility.ts'; +import * as environmentUtility from '../../../../utility/lmdb/environmentUtility.ts'; +import { getSchemaPath } from '../lmdbUtility/initializePaths.ts'; +import writeTransaction from '../lmdbUtility/lmdbWriteTransaction.ts'; +import logger from '../../../../utility/logging/harper_logger.ts'; + +export default lmdbUpdateRecords; /** * Orchestrates the update of data in LMDB and the creation of new attributes/dbis @@ -20,7 +18,7 @@ module.exports = lmdbUpdateRecords; */ async function lmdbUpdateRecords(updateObj) { try { - let { schemaTable, attributes } = insertUpdateValidate(updateObj); + let { schema_table: schemaTable, attributes } = insertUpdateValidate(updateObj); lmdbProcessRows(updateObj, attributes, schemaTable.hash_attribute); diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpsertRecords.js b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpsertRecords.ts similarity index 64% rename from dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpsertRecords.js rename to dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpsertRecords.ts index f84db3a9f..775675284 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpsertRecords.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbMethods/lmdbUpsertRecords.ts @@ -1,21 +1,19 @@ -'use strict'; - // eslint-disable-next-line no-unused-vars -const UpsertObject = - require('../../../dataObjects/UpsertObject.js').default || require('../../../dataObjects/UpsertObject.js'); -const insertUpdateValidate = require('../../bridgeUtility/insertUpdateValidate.js'); -const lmdbProcessRows = require('../lmdbUtility/lmdbProcessRows.js'); -const lmdbCheckNewAttributes = require('../lmdbUtility/lmdbCheckForNewAttributes.js'); -const hdbTerms = require('../../../../utility/hdbTerms.ts'); -const lmdb_upsert_records = require('../../../../utility/lmdb/writeUtility.ts').upsertRecords; -const environmentUtility = require('../../../../utility/lmdb/environmentUtility.ts'); -const { getSchemaPath } = require('../lmdbUtility/initializePaths.js'); -const writeTransaction = require('../lmdbUtility/lmdbWriteTransaction.js'); +import UpsertObject from '../../../dataObjects/UpsertObject.ts'; +import insertUpdateValidate from '../../bridgeUtility/insertUpdateValidate.ts'; +import _lmdbProcessRows from '../lmdbUtility/lmdbProcessRows.ts'; +const lmdbProcessRows = _lmdbProcessRows; +import lmdbCheckNewAttributes from '../lmdbUtility/lmdbCheckForNewAttributes.ts'; +import * as hdbTerms from '../../../../utility/hdbTerms.ts'; +import { upsertRecords as lmdb_upsert_records } from '../../../../utility/lmdb/writeUtility.ts'; +import * as environmentUtility from '../../../../utility/lmdb/environmentUtility.ts'; +import { getSchemaPath } from '../lmdbUtility/initializePaths.ts'; +import writeTransaction from '../lmdbUtility/lmdbWriteTransaction.ts'; -const logger = require('../../../../utility/logging/harper_logger.ts'); -const { handleHDBError, hdbErrors } = require('../../../../utility/errors/hdbError.ts'); +import logger from '../../../../utility/logging/harper_logger.ts'; +import { handleHDBError, hdbErrors } from '../../../../utility/errors/hdbError.ts'; -module.exports = lmdbUpsertRecords; +export default lmdbUpsertRecords; /** * Orchestrates the UPSERT of data in LMDB and the creation of new attributes/dbis diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBCreateAttributeObject.js b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBCreateAttributeObject.ts similarity index 75% rename from dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBCreateAttributeObject.js rename to dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBCreateAttributeObject.ts index e109a0845..a27399c69 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBCreateAttributeObject.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBCreateAttributeObject.ts @@ -1,8 +1,4 @@ -'use strict'; - -const CreateAttributeObject = - require('../../../CreateAttributeObject.ts').default || require('../../../CreateAttributeObject.ts'); - +import CreateAttributeObject from '../../../CreateAttributeObject.ts'; class LMDBCreateAttributeObject extends CreateAttributeObject { /** * @@ -20,4 +16,4 @@ class LMDBCreateAttributeObject extends CreateAttributeObject { } } -module.exports = LMDBCreateAttributeObject; +export default LMDBCreateAttributeObject; diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBDeleteTransactionObject.js b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBDeleteTransactionObject.ts similarity index 75% rename from dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBDeleteTransactionObject.js rename to dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBDeleteTransactionObject.ts index 5f08bbe76..b28282efd 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBDeleteTransactionObject.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBDeleteTransactionObject.ts @@ -1,11 +1,10 @@ -'use strict'; -const LMDBTransactionObject = require('./LMDBTransactionObject.js'); -const OPERATIONS_ENUM = require('../../../../utility/hdbTerms.ts').OPERATIONS_ENUM; - +import LMDBTransactionObject from './LMDBTransactionObject.js'; +import { OPERATIONS_ENUM } from '../../../../utility/hdbTerms.ts'; /** * class to define a delete transaction */ class LMDBDeleteTransactionObject extends LMDBTransactionObject { + original_records: any; /** * @param {Array.} hash_values - hash values of deleted records * @param {Array.} originalRecords - original records prior to delete @@ -20,4 +19,4 @@ class LMDBDeleteTransactionObject extends LMDBTransactionObject { } } -module.exports = LMDBDeleteTransactionObject; +export default LMDBDeleteTransactionObject; diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBInsertTransactionObject.js b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBInsertTransactionObject.ts similarity index 72% rename from dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBInsertTransactionObject.js rename to dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBInsertTransactionObject.ts index 24d5c3271..287437b85 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBInsertTransactionObject.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBInsertTransactionObject.ts @@ -1,11 +1,10 @@ -'use strict'; -const LMDBTransactionObject = require('./LMDBTransactionObject.js'); -const OPERATIONS_ENUM = require('../../../../utility/hdbTerms.ts').OPERATIONS_ENUM; - +import LMDBTransactionObject from './LMDBTransactionObject.js'; +import { OPERATIONS_ENUM } from '../../../../utility/hdbTerms.ts'; /** * class to define an insert transaction */ class LMDBInsertTransactionObject extends LMDBTransactionObject { + records: any; /** * @param {Array.} records - inserted records * @param {string} userName - username that executed trasaction @@ -19,4 +18,4 @@ class LMDBInsertTransactionObject extends LMDBTransactionObject { } } -module.exports = LMDBInsertTransactionObject; +export default LMDBInsertTransactionObject; diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpdateTransactionObject.js b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpdateTransactionObject.ts similarity index 76% rename from dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpdateTransactionObject.js rename to dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpdateTransactionObject.ts index d10d8e736..290aed96f 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpdateTransactionObject.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpdateTransactionObject.ts @@ -1,11 +1,11 @@ -'use strict'; -const LMDBTransactionObject = require('./LMDBTransactionObject.js'); -const OPERATIONS_ENUM = require('../../../../utility/hdbTerms.ts').OPERATIONS_ENUM; - +import LMDBTransactionObject from './LMDBTransactionObject.js'; +import { OPERATIONS_ENUM } from '../../../../utility/hdbTerms.ts'; /** * class to define an update transaction */ class LMDBUpdateTransactionObject extends LMDBTransactionObject { + records: any; + original_records: any; /** * @param {Array.} records - records updated * @param {Array.} originalRecords - original state of records that were updated @@ -21,4 +21,4 @@ class LMDBUpdateTransactionObject extends LMDBTransactionObject { } } -module.exports = LMDBUpdateTransactionObject; +export default LMDBUpdateTransactionObject; diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpsertTransactionObject.js b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpsertTransactionObject.ts similarity index 76% rename from dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpsertTransactionObject.js rename to dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpsertTransactionObject.ts index 5e39e3f5b..1e42435ec 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpsertTransactionObject.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/LMDBUpsertTransactionObject.ts @@ -1,11 +1,11 @@ -'use strict'; -const LMDBTransactionObject = require('./LMDBTransactionObject.js'); -const OPERATIONS_ENUM = require('../../../../utility/hdbTerms.ts').OPERATIONS_ENUM; - +import LMDBTransactionObject from './LMDBTransactionObject.js'; +import { OPERATIONS_ENUM } from '../../../../utility/hdbTerms.ts'; /** * class to define an update transaction */ class LMDBUpsertTransactionObject extends LMDBTransactionObject { + records: any; + original_records: any; /** * @param {Array.} records - records updated * @param {Array.} originalRecords - original state of records that were updated @@ -21,4 +21,4 @@ class LMDBUpsertTransactionObject extends LMDBTransactionObject { } } -module.exports = LMDBUpsertTransactionObject; +export default LMDBUpsertTransactionObject; diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializeHashSearch.js b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializeHashSearch.ts similarity index 52% rename from dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializeHashSearch.js rename to dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializeHashSearch.ts index 9f1492513..79f67e72b 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializeHashSearch.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializeHashSearch.ts @@ -1,11 +1,8 @@ -'use strict'; +import * as environmentUtility from '../../../../utility/lmdb/environmentUtility.ts'; +import searchValidator from '../../../../validation/searchValidator.ts'; +import { getSchemaPath } from './initializePaths.ts'; -const environmentUtility = require('../../../../utility/lmdb/environmentUtility.ts'); -const searchValidator = - require('../../../../validation/searchValidator.ts').default || require('../../../../validation/searchValidator.ts'); -const { getSchemaPath } = require('./initializePaths.js'); - -module.exports = initialize; +export default initialize; /** * diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.ts similarity index 88% rename from dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js rename to dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.ts index 032f001c0..21d3bceaa 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.ts @@ -1,14 +1,13 @@ -'use strict'; - -const hdbTerms = require('../../../../utility/hdbTerms.ts'); -const hdbUtils = require('../../../../utility/common_utils.ts'); -const env = require('../../../../utility/environment/environmentManager.ts'); -const path = require('path'); -const minimist = require('minimist'); -const fs = require('fs-extra'); -const _ = require('lodash'); -const { getConfigPath } = require('../../../../config/configUtils.js'); -env.initSync(); +import * as hdbTerms from '../../../../utility/hdbTerms.ts'; +import * as hdbUtils from '../../../../utility/common_utils.ts'; +import * as env from '../../../../utility/environment/environmentManager.ts'; +import path from 'path'; +import minimist from 'minimist'; +import fs from 'fs-extra'; +import _ from 'lodash'; +import { getConfigPath } from '../../../../config/configUtils.ts'; +// env.initSync() is called by the CLI entry points (bin/run.ts, bin/cliOperations.ts); +// calling it here at module load would TDZ during ESM cycle evaluation. const { CONFIG_PARAMS, DATABASES_PARAM_CONFIG, SYSTEM_SCHEMA_NAME } = hdbTerms; let BASE_SCHEMA_PATH = undefined; @@ -68,7 +67,7 @@ function getTransactionAuditStorePath(schema, table) { ); } -function getSchemaPath(schema, table) { +function getSchemaPath(schema, table?: string | null) { schema = schema.toString(); table = table ? table.toString() : table; let schemaConfig = env.get(hdbTerms.CONFIG_PARAMS.DATABASES)?.[schema]; @@ -147,7 +146,7 @@ function resetPaths() { SYSTEM_SCHEMA_PATH = undefined; TRANSACTION_STORE_PATH = undefined; } -module.exports = { +export { getBaseSchemaPath, getSystemSchemaPath, getTransactionAuditStorePath, diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCheckForNewAttributes.js b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCheckForNewAttributes.ts similarity index 79% rename from dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCheckForNewAttributes.js rename to dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCheckForNewAttributes.ts index e519e55e9..ecbd04dcc 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCheckForNewAttributes.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCheckForNewAttributes.ts @@ -1,17 +1,14 @@ -'use strict'; - -const hUtils = require('../../../../utility/common_utils.ts'); -const hdbTerms = require('../../../../utility/hdbTerms.ts'); -const logger = require('../../../../utility/logging/harper_logger.ts'); -const lmdbCreateAttribute = require('../lmdbMethods/lmdbCreateAttribute.js'); -const LMDBCreateAttributeObject = - require('./LMDBCreateAttributeObject.js').default || require('./LMDBCreateAttributeObject.js'); -const signalling = require('../../../../utility/signalling.ts'); -const { SchemaEventMsg } = require('../../../../server/threads/itc.js'); +import * as hUtils from '../../../../utility/common_utils.ts'; +import * as hdbTerms from '../../../../utility/hdbTerms.ts'; +import logger from '../../../../utility/logging/harper_logger.ts'; +import lmdbCreateAttribute from '../lmdbMethods/lmdbCreateAttribute.ts'; +import LMDBCreateAttributeObject from './LMDBCreateAttributeObject.ts'; +import * as signalling from '../../../../utility/signalling.ts'; +import { SchemaEventMsg } from '../../../../server/threads/itc.ts'; const ATTRIBUTE_ALREADY_EXISTS = 'already exists in'; -module.exports = lmdbCheckForNewAttributes; +export default lmdbCheckForNewAttributes; /** * Uses a utility function to check if there are any new attributes that dont exist. Utility function diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCreateTransactionsAuditEnvironment.js b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCreateTransactionsAuditEnvironment.ts similarity index 70% rename from dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCreateTransactionsAuditEnvironment.js rename to dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCreateTransactionsAuditEnvironment.ts index 41c4d8c2c..8b723c3c7 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCreateTransactionsAuditEnvironment.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbCreateTransactionsAuditEnvironment.ts @@ -1,16 +1,11 @@ -'use strict'; - -const fs = require('fs-extra'); -const environmentUtility = require('../../../../utility/lmdb/environmentUtility.ts'); -const { getTransactionAuditStorePath } = require('../lmdbUtility/initializePaths.js'); -const lmdbTerms = require('../../../../utility/lmdb/terms.ts'); +import fs from 'fs-extra'; +import * as environmentUtility from '../../../../utility/lmdb/environmentUtility.ts'; +import { getTransactionAuditStorePath } from '../lmdbUtility/initializePaths.ts'; +import * as lmdbTerms from '../../../../utility/lmdb/terms.ts'; // eslint-disable-next-line no-unused-vars -const CreateTableObject = - require('../../../CreateTableObject.ts').default || - require('../../../CreateTableObject.ts').default || - require('../../../CreateTableObject.ts'); +import CreateTableObject from '../../../CreateTableObject.ts'; -module.exports = createTransactionsAuditEnvironment; +export default createTransactionsAuditEnvironment; /** * Creates the environment to hold transactions diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbProcessRows.js b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbProcessRows.ts similarity index 83% rename from dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbProcessRows.js rename to dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbProcessRows.ts index 837a526e8..81994b5f0 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbProcessRows.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbProcessRows.ts @@ -1,15 +1,12 @@ -'use strict'; - // eslint-disable-next-line no-unused-vars -const InsertObject = require('../../../InsertObject.ts').default || require('../../../InsertObject.ts'); -const hdbTerms = require('../../../../utility/hdbTerms.ts'); -const hdbUtils = require('../../../../utility/common_utils.ts'); -const log = require('../../../../utility/logging/harper_logger.ts'); -const uuid = require('uuid'); -const { handleHDBError, hdbErrors } = require('../../../../utility/errors/hdbError.ts'); +import InsertObject from '../../../InsertObject.ts'; +import * as hdbTerms from '../../../../utility/hdbTerms.ts'; +import * as hdbUtils from '../../../../utility/common_utils.ts'; +import log from '../../../../utility/logging/harper_logger.ts'; +import { v4 as uuidv4 } from 'uuid'; +import { handleHDBError, hdbErrors } from '../../../../utility/errors/hdbError.ts'; const { HDB_ERROR_MSGS, HTTP_STATUS_CODES } = hdbErrors; - -module.exports = processRows; +export default processRows; /** * parses the records and validates the hash value for each row as well as adding updated/created time stamps @@ -70,7 +67,7 @@ function validateAttribute(attribute) { function validateHash(record, hash_attribute, operation) { if (!record.hasOwnProperty(hash_attribute) || hdbUtils.isEmptyOrZeroLength(record[hash_attribute])) { if (operation === hdbTerms.OPERATIONS_ENUM.INSERT || operation === hdbTerms.OPERATIONS_ENUM.UPSERT) { - record[hash_attribute] = uuid.v4(); + record[hash_attribute] = uuidv4(); //return here since the rest of the validations do not apply return; } diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbSearch.js b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbSearch.ts similarity index 91% rename from dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbSearch.js rename to dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbSearch.ts index e5af231e8..8b3f3ccb0 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbSearch.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbSearch.ts @@ -1,13 +1,16 @@ -'use strict'; - -const searchUtility = require('../../../../utility/lmdb/searchUtility.ts'); -const environmentUtility = require('../../../../utility/lmdb/environmentUtility.ts'); -const commonUtils = require('../../../../utility/common_utils.ts'); -const lmdbTerms = require('../../../../utility/lmdb/terms.ts'); -const hdbTerms = require('../../../../utility/hdbTerms.ts'); -const systemSchema = require('../../../../json/systemSchema.json'); -const LMDB_ERRORS = require('../../../../utility/errors/commonErrors.ts').LMDB_ERRORS_ENUM; -const { getSchemaPath } = require('./initializePaths.js'); +import * as searchUtility from '../../../../utility/lmdb/searchUtility.ts'; +import * as environmentUtility from '../../../../utility/lmdb/environmentUtility.ts'; +import * as commonUtils from '../../../../utility/common_utils.ts'; +import * as lmdbTerms from '../../../../utility/lmdb/terms.ts'; +import * as hdbTerms from '../../../../utility/hdbTerms.ts'; +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { PACKAGE_ROOT } from '../../../../utility/packageUtils.js'; +const systemSchema: Record = JSON.parse( + readFileSync(join(PACKAGE_ROOT, 'json/systemSchema.json'), 'utf-8') +); +import { LMDB_ERRORS_ENUM as LMDB_ERRORS } from '../../../../utility/errors/commonErrors.ts'; +import { getSchemaPath } from './initializePaths.ts'; const WILDCARDS = hdbTerms.SEARCH_WILDCARDS; @@ -362,10 +365,5 @@ function createSearchTypeFromSearchObject(searchObject, hash_attribute, returnMa } } -module.exports = { - executeSearch, - createSearchTypeFromSearchObject, - prepSearch, - searchByType, - // filterByType, -}; +export { executeSearch, createSearchTypeFromSearchObject, prepSearch, searchByType }; +export const filterByType: any = undefined; // placeholder: filterByType is not yet implemented diff --git a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbWriteTransaction.js b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbWriteTransaction.ts similarity index 74% rename from dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbWriteTransaction.js rename to dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbWriteTransaction.ts index 0889ca63f..47d424015 100644 --- a/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbWriteTransaction.js +++ b/dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbWriteTransaction.ts @@ -1,21 +1,18 @@ -'use strict'; +import * as environmentUtil from '../../../../utility/lmdb/environmentUtility.ts'; +import LMDBInsertTransactionObject from './LMDBInsertTransactionObject.ts'; +import LMDBUpdateTransactionObject from './LMDBUpdateTransactionObject.ts'; +import LMDBUpsertTransactionObject from './LMDBUpsertTransactionObject.ts'; +import LMDBDeleteTransactionObject from './LMDBDeleteTransactionObject.ts'; -const environmentUtil = require('../../../../utility/lmdb/environmentUtility.ts'); -const LMDBInsertTransactionObject = require('./LMDBInsertTransactionObject.js'); -const LMDBUpdateTransactionObject = require('./LMDBUpdateTransactionObject.js'); -const LMDBUpsertTransactionObject = require('./LMDBUpsertTransactionObject.js'); -const LMDBDeleteTransactionObject = require('./LMDBDeleteTransactionObject.js'); +import * as lmdbTerms from '../../../../utility/lmdb/terms.ts'; +import * as hdbUtil from '../../../../utility/common_utils.ts'; +import { CONFIG_PARAMS } from '../../../../utility/hdbTerms.ts'; +import * as envMngr from '../../../../utility/environment/environmentManager.ts'; -const lmdbTerms = require('../../../../utility/lmdb/terms.ts'); -const hdbUtil = require('../../../../utility/common_utils.ts'); -const { CONFIG_PARAMS } = require('../../../../utility/hdbTerms.ts'); -const envMngr = require('../../../../utility/environment/environmentManager.ts'); -envMngr.initSync(); +import { OPERATIONS_ENUM } from '../../../../utility/hdbTerms.ts'; +import { getTransactionAuditStorePath } from './initializePaths.ts'; -const OPERATIONS_ENUM = require('../../../../utility/hdbTerms.ts').OPERATIONS_ENUM; -const { getTransactionAuditStorePath } = require('./initializePaths.js'); - -module.exports = writeTransaction; +export default writeTransaction; /** * @@ -33,7 +30,7 @@ async function writeTransaction(hdbOperation, lmdbResponse) { let txnObject = createTransactionObject(hdbOperation, lmdbResponse); - if (txnObject === undefined || txnObject.hash_values.length === 0) { + if (txnObject === undefined || (txnObject as any).hash_values.length === 0) { return; } diff --git a/dataLayer/insert.ts b/dataLayer/insert.ts index 4efc8331d..07634d5b8 100644 --- a/dataLayer/insert.ts +++ b/dataLayer/insert.ts @@ -7,10 +7,13 @@ * as the update module is meant to be used in more specific circumstances. */ import insertValidator from '../validation/insertValidator.ts'; -import * as hdbUtils from '../utility/common_utils.ts'; +import * as _hdbUtils from '../utility/common_utils.ts'; +const hdbUtils = _hdbUtils; import * as util from 'util'; // Leave this unused signalling import here. Due to circular dependencies we bring it in early to load it before the bridge -const harperBridge = require('./harperBridge/harperBridge').default; +import _harperBridge from './harperBridge/harperBridge.ts'; +// Lazy access to handle import cycle (insert.ts and harperBridge.ts both import each other transitively). +const harperBridge: any = new Proxy({}, { get: (_, p) => (_harperBridge as any)[p] }); import * as globalSchema from '../utility/globalSchema.ts'; import log from '../utility/logging/harper_logger.ts'; import { handleHDBError } from '../utility/errors/hdbError.ts'; diff --git a/dataLayer/readAuditLog.ts b/dataLayer/readAuditLog.ts index b0a4dd217..601b2e3bd 100644 --- a/dataLayer/readAuditLog.ts +++ b/dataLayer/readAuditLog.ts @@ -1,6 +1,8 @@ 'use strict'; -const harperBridge = require('./harperBridge/harperBridge').default; +import _harperBridge from './harperBridge/harperBridge.ts'; +// Lazy access to handle import cycle (insert.ts and harperBridge.ts both import each other transitively). +const harperBridge: any = new Proxy({}, { get: (_, p) => (_harperBridge as any)[p] }); // eslint-disable-next-line no-unused-vars import ReadAuditLogObject from './ReadAuditLogObject.ts'; import * as hdbUtils from '../utility/common_utils.ts'; diff --git a/dataLayer/schema.ts b/dataLayer/schema.ts index 9d3357d5a..7f85f9a08 100644 --- a/dataLayer/schema.ts +++ b/dataLayer/schema.ts @@ -1,6 +1,7 @@ 'use strict'; -import * as schemaMetadataValidator from '../validation/schemaMetadataValidator.ts'; +import * as _schemaMetadataValidator from '../validation/schemaMetadataValidator.ts'; +const schemaMetadataValidator = _schemaMetadataValidator; import { validateBySchema } from '../validation/validationWrapper.ts'; import { commonValidators, schemaRegex } from '../validation/common_validators.ts'; import Joi from 'joi'; @@ -9,12 +10,13 @@ import { v4 as uuidV4 } from 'uuid'; import * as signalling from '../utility/signalling.ts'; import * as hdbTerms from '../utility/hdbTerms.ts'; import * as util from 'util'; -const harperBridge = require('./harperBridge/harperBridge').default; +import harperBridge from './harperBridge/harperBridge.ts'; import { handleHDBError, ClientError } from '../utility/errors/hdbError.ts'; import { HDB_ERROR_MSGS, HTTP_STATUS_CODES } from '../utility/errors/commonErrors.ts'; -import { SchemaEventMsg } from '../server/threads/itc.js'; -import { getDatabases, dropTableMeta } from '../resources/databases.ts'; +import { SchemaEventMsg } from '../server/threads/itc.ts'; +import { getDatabases as _getDatabases, dropTableMeta } from '../resources/databases.ts'; +const getDatabases = _getDatabases; import { transformReq } from '../utility/common_utils.ts'; import { server } from '../server/Server.ts'; import { cleanupOrphans } from '../resources/blob.ts'; @@ -312,7 +314,7 @@ function dropAttributeFromGlobal(dropAttributeObject) { export async function createAttribute(createAttributeObject: any) { transformReq(createAttributeObject); - const tableAttr = getDatabases()[createAttributeObject.schema][createAttributeObject.table].attributes; + const tableAttr = (getDatabases()[createAttributeObject.schema][createAttributeObject.table] as any).attributes; for (const { name } of tableAttr) { if (name === createAttributeObject.attribute) { throw handleHDBError( diff --git a/dataLayer/schemaDescribe.ts b/dataLayer/schemaDescribe.ts index 88b260e85..7648b9048 100644 --- a/dataLayer/schemaDescribe.ts +++ b/dataLayer/schemaDescribe.ts @@ -8,11 +8,15 @@ import * as hdbUtils from '../utility/common_utils.ts'; import { handleHDBError, ClientError } from '../utility/errors/hdbError.ts'; import { HDB_ERROR_MSGS, HTTP_STATUS_CODES } from '../utility/errors/commonErrors.ts'; +import { getDatabases as _getDatabases } from '../resources/databases.ts'; +const getDatabases = _getDatabases; +import fs from 'fs-extra'; import * as envMngr from '../utility/environment/environmentManager.ts'; -envMngr.initSync(); -import { getDatabases } from '../resources/databases.ts'; -import * as fs from 'fs-extra'; - +try { + envMngr.initSync(); +} catch { + /* tolerate ESM cycle TDZ; bin entry will re-call later */ +} /** * This method is exposed to the API and internally for system operations. If the op is being made internally, the `opObj` * argument is not passed and, therefore, no permissions are used to filter the final schema metadata results. @@ -138,7 +142,7 @@ async function descTable(describeTableObject: any, attrPerms?: any) { HTTP_STATUS_CODES.NOT_FOUND ); } - let tableObj = tables[table]; + let tableObj: any = tables[table]; if (!tableObj) throw handleHDBError( new Error(), diff --git a/dataLayer/search.ts b/dataLayer/search.ts index 2edd0f4ee..7fd27b6f0 100644 --- a/dataLayer/search.ts +++ b/dataLayer/search.ts @@ -1,6 +1,6 @@ 'use strict'; -const harperBridge = require('./harperBridge/harperBridge').default; +import harperBridge from './harperBridge/harperBridge.ts'; import { transformReq } from '../utility/common_utils.ts'; export async function searchByConditions(searchObject: any) { diff --git a/dataLayer/transaction.ts b/dataLayer/transaction.ts deleted file mode 100644 index c2a23f91e..000000000 --- a/dataLayer/transaction.ts +++ /dev/null @@ -1,14 +0,0 @@ -'use strict'; - -const harperBridge = require('./harperBridge/harperBridge').default; - -/** - * This is wrapper for write transactions, ensuring that all reads and writes within the callback occur atomically - * @param schema - * @param table - * @param callback - * @returns {Promise} - */ -export function writeTransaction(schema: string, table: string, callback: any) { - return harperBridge.writeTransaction(schema, table, callback); -} diff --git a/dataLayer/update.ts b/dataLayer/update.ts index 974d5bfa6..45c987c24 100644 --- a/dataLayer/update.ts +++ b/dataLayer/update.ts @@ -3,7 +3,8 @@ import * as search from './search.ts'; import * as globalSchema from '../utility/globalSchema.ts'; import logger from '../utility/logging/harper_logger.ts'; -import * as write from './insert.ts'; +import * as _write from './insert.ts'; +const write = _write; import clone from 'clone'; import * as alasql from 'alasql'; import alasqlFunctionImporter from '../sqlTranslator/alasqlFunctionImporter.ts'; diff --git a/index.ts b/index.ts index c8802b281..0590db95d 100644 --- a/index.ts +++ b/index.ts @@ -67,7 +67,10 @@ import type { operation as OperationImport } from './server/serverHelpers/server import type { Resource as ResourceImport } from './resources/Resource.ts'; import type { server as ServerImport } from './server/Server.ts'; import type { tables as TablesImport } from './resources/databases.ts'; -type ThreadsImport = unknown[]; // TODO: figure out actual type for this +type ThreadsImport = any[] & { + sendToThread?: (threadId: number, message: any) => boolean; + onMessageByType?: (type: string, listener: (...args: any[]) => any) => void; +}; import type { transaction as TransactionImport } from './resources/transaction.ts'; declare global { @@ -113,6 +116,6 @@ exports.transaction = undefined; // And finally assign globals to exports. // These values are populated at runtime by `_assignPackageExport()` in their respective modules // (e.g. Resource.ts, databases.ts, Server.ts, etc.) -import { globals } from './server/threads/threadServer.js'; +import { globals } from './server/threads/threadServer.ts'; Object.assign(exports, globals); diff --git a/integrationTests/server/operations-server.test.ts b/integrationTests/server/operations-server.test.ts index 1fedb271d..145188f59 100644 --- a/integrationTests/server/operations-server.test.ts +++ b/integrationTests/server/operations-server.test.ts @@ -11,7 +11,6 @@ import { suite, test, before, after } from 'node:test'; import { ok, strictEqual } from 'node:assert/strict'; import { pack, unpack } from 'msgpackr'; import { encode, decode } from 'cbor-x'; - import { startHarper, teardownHarper, type ContextWithHarper } from '@harperfast/integration-testing'; suite('Operations Server', (ctx: ContextWithHarper) => { diff --git a/launchServiceScripts/launchHarperDB.js b/launchServiceScripts/launchHarperDB.js deleted file mode 100644 index 7bc839255..000000000 --- a/launchServiceScripts/launchHarperDB.js +++ /dev/null @@ -1,3 +0,0 @@ -'use strict'; - -require('../server/operationsServer.ts').hdbServer(); diff --git a/launchServiceScripts/launchHarperDB.ts b/launchServiceScripts/launchHarperDB.ts new file mode 100644 index 000000000..8e032f986 --- /dev/null +++ b/launchServiceScripts/launchHarperDB.ts @@ -0,0 +1 @@ +import('../server/operationsServer.ts').then((m) => m.hdbServer()); diff --git a/resources/DatabaseTransaction.ts b/resources/DatabaseTransaction.ts index 79e3a025b..ee351e459 100644 --- a/resources/DatabaseTransaction.ts +++ b/resources/DatabaseTransaction.ts @@ -1,5 +1,5 @@ import { cleanupUnusedBlobs } from './blob.ts'; -import { Transaction as LMDBTransaction } from 'lmdb'; +import { type Transaction as LMDBTransaction } from 'lmdb'; import { getNextMonotonicTime } from '../utility/lmdb/commonUtility.ts'; import { ServerError } from '../utility/errors/hdbError.ts'; import * as harperLogger from '../utility/logging/harper_logger.ts'; diff --git a/resources/ErrorResource.ts b/resources/ErrorResource.ts index 07b86001f..09119a9a3 100644 --- a/resources/ErrorResource.ts +++ b/resources/ErrorResource.ts @@ -1,58 +1,86 @@ import { Resource } from './Resource.ts'; import type { Context } from './ResourceInterface.ts'; + /** * ErrorResource is a Resource that throws an error on any request, communicating to the client when attempts are made * to access endpoints/resources that had an internal error in their configuration or setup. This helps ensure that * if there is a problem with a resource, it is immediately apparent and can be fixed. + * + * The class is constructed lazily on first use, not at module load. ErrorResource + * sits inside Resource.ts's own static-graph SCC, so a class-extends declaration + * at module-top would TDZ on `Resource`. The Proxy here lets `new ErrorResource(x)` + * and `ErrorResource.staticMember` work whenever they're called, even when this + * module is loaded directly (unit tests, scripts) without the lifecycle hooks running. */ -export class ErrorResource extends Resource { - error: Error; - constructor(error: Error) { - super(null as any, null); - this.error = error; - } - isError = true; - allowRead(): never { - throw this.error; - } - allowUpdate(): never { - throw this.error; - } - allowCreate(): never { - throw this.error; - } - allowDelete(): never { - throw this.error; - } - getId(): never { - throw this.error; - } - getContext(): Context { - throw this.error; - } - get(): never { - throw this.error; - } - post(): never { - throw this.error; - } - put(): never { - throw this.error; - } - delete(): never { - throw this.error; - } - connect(): never { - throw this.error; - } - getResource() { - // all child paths resolve back to reporting this error - return this; - } - publish(): never { - throw this.error; - } - subscribe(): never { - throw this.error; +let _ErrorResource: any; +function getErrorResource(): any { + if (!_ErrorResource) { + _ErrorResource = class ErrorResource extends Resource { + error: Error; + constructor(error: Error) { + super(null as any, null); + this.error = error; + } + isError = true; + allowRead(): never { + throw this.error; + } + allowUpdate(): never { + throw this.error; + } + allowCreate(): never { + throw this.error; + } + allowDelete(): never { + throw this.error; + } + getId(): never { + throw this.error; + } + getContext(): Context { + throw this.error; + } + get(): never { + throw this.error; + } + post(): never { + throw this.error; + } + put(): never { + throw this.error; + } + delete(): never { + throw this.error; + } + connect(): never { + throw this.error; + } + getResource() { + // all child paths resolve back to reporting this error + return this; + } + publish(): never { + throw this.error; + } + subscribe(): never { + throw this.error; + } + }; } + return _ErrorResource; } + +export const ErrorResource: any = new Proxy(function () {} as any, { + construct(_target, args) { + return Reflect.construct(getErrorResource(), args); + }, + get(_target, prop) { + return getErrorResource()[prop]; + }, + has(_target, prop) { + return prop in getErrorResource(); + }, + getPrototypeOf() { + return getErrorResource().prototype; + }, +}); diff --git a/resources/LMDBTransaction.ts b/resources/LMDBTransaction.ts index 179752182..5dbd8bf5c 100644 --- a/resources/LMDBTransaction.ts +++ b/resources/LMDBTransaction.ts @@ -3,7 +3,7 @@ import { type CommitOptions, type TransactionWrite, type CommitResolution, -} from './DatabaseTransaction'; +} from './DatabaseTransaction.ts'; import { cleanupUnusedBlobs } from './blob.ts'; import { getNextMonotonicTime } from '../utility/lmdb/commonUtility.ts'; import * as harperLogger from '../utility/logging/harper_logger.ts'; diff --git a/resources/Resource.ts b/resources/Resource.ts index 9b64354d2..1d36c1abb 100644 --- a/resources/Resource.ts +++ b/resources/Resource.ts @@ -1,13 +1,13 @@ import type { User } from '../security/user.ts'; import type { RecordObject } from './RecordEncoder.ts'; import { - ResourceInterface, - SubscriptionRequest, - Id, - Context, - Query, - SourceContext, - RequestTargetOrId, + type ResourceInterface, + type SubscriptionRequest, + type Id, + type Context, + type Query, + type SourceContext, + type RequestTargetOrId, } from './ResourceInterface.ts'; import { randomUUID } from 'crypto'; import { DatabaseTransaction, type Transaction } from './DatabaseTransaction.ts'; diff --git a/resources/Resources.ts b/resources/Resources.ts index 4f121cf22..169068663 100644 --- a/resources/Resources.ts +++ b/resources/Resources.ts @@ -2,6 +2,7 @@ import { transaction } from './transaction.ts'; import logger from '../utility/logging/harper_logger.ts'; import { ServerError } from '../utility/errors/hdbError.ts'; import { server } from '../server/Server.ts'; +import { ErrorResource } from './ErrorResource.ts'; interface ResourceEntry { Resource: any; @@ -44,7 +45,6 @@ export class Resources extends Map { // don't provide anything more descriptive. const error = new ServerError(`Conflicting paths for ${path}`); logger.error(error); - const { ErrorResource } = require('./ErrorResource'); entry.Resource = new ErrorResource(error); } super.set(path, entry); diff --git a/resources/RocksIndexStore.ts b/resources/RocksIndexStore.ts index c223f8409..49ad4c58a 100644 --- a/resources/RocksIndexStore.ts +++ b/resources/RocksIndexStore.ts @@ -5,9 +5,8 @@ import { type StoreRemoveOptions, RocksDatabase, } from '@harperfast/rocksdb-js'; -import { Id } from './ResourceInterface.ts'; +import { type Id } from './ResourceInterface.ts'; import { MAXIMUM_KEY } from 'ordered-binary'; - declare module '@harperfast/rocksdb-js' { interface DBI { getValuesCount(indexedValue: any): number; diff --git a/resources/RocksTransactionLogStore.ts b/resources/RocksTransactionLogStore.ts index 8cd899dfb..3c7f8ebe3 100644 --- a/resources/RocksTransactionLogStore.ts +++ b/resources/RocksTransactionLogStore.ts @@ -1,7 +1,7 @@ import { TransactionLog, RocksDatabase, shutdown, type TransactionEntry } from '@harperfast/rocksdb-js'; import { ExtendedIterable } from '@harperfast/extended-iterable'; import { getIdOfRemoteNode } from './nodeIdMapping.ts'; -import { Decoder, readAuditEntry, ENTRY_DATAVIEW, AuditRecord, createAuditEntry } from './auditStore.ts'; +import { Decoder, readAuditEntry, ENTRY_DATAVIEW, type AuditRecord, createAuditEntry } from './auditStore.ts'; import { isMainThread } from 'node:worker_threads'; import { EventEmitter } from 'node:events'; import { asBinary } from 'lmdb'; @@ -317,7 +317,8 @@ export class RocksTransactionLogStore extends EventEmitter { // downstream consumers already skip records with no `tableId`/`type`. try { const decoder = new Decoder(data.buffer, data.byteOffset, data.byteLength); - (data as any).dataView = decoder; + // @ts-expect-error - dataView is attached to data dynamically for downstream consumers + data.dataView = decoder; // This represents the data that shouldn't be transferred for replication let structureVersion = decoder.getUint32(0); let position = 4; diff --git a/resources/Table.ts b/resources/Table.ts index 03cfdf4e5..339260e3d 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -22,7 +22,7 @@ import type { RequestTargetOrId, } from './ResourceInterface.ts'; import type { User } from '../security/user.ts'; -import lmdbProcessRows from '../dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbProcessRows.js'; +import lmdbProcessRows from '../dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbProcessRows.ts'; import { Resource, transformForSelect } from './Resource.ts'; import { when, promiseNormalize } from '../utility/when.ts'; import { DatabaseTransaction, ImmediateTransaction, TRANSACTION_STATE } from './DatabaseTransaction.ts'; @@ -30,7 +30,7 @@ import * as envMngr from '../utility/environment/environmentManager.ts'; import { addSubscription } from './transactionBroadcast.ts'; import { handleHDBError, ClientError, ServerError, AccessViolation } from '../utility/errors/hdbError.ts'; import * as signalling from '../utility/signalling.ts'; -import { SchemaEventMsg, UserEventMsg } from '../server/threads/itc.js'; +import { SchemaEventMsg, UserEventMsg } from '../server/threads/itc.ts'; import { databases, table } from './databases.ts'; import { searchByIndex, @@ -45,7 +45,7 @@ import { logger } from '../utility/logging/logger.ts'; import { Addition, assignTrackedAccessors, updateAndFreeze, hasChanges, GenericTrackedObject } from './tracked.ts'; import { transaction, contextStorage } from './transaction.ts'; import { MAXIMUM_KEY, writeKey, compareKeys } from 'ordered-binary'; -import { getWorkerIndex, getWorkerCount } from '../server/threads/manageThreads.js'; +import { getWorkerIndex, getWorkerCount } from '../server/threads/manageThreads.ts'; import { HAS_BLOBS, auditRetention, removeAuditEntry } from './auditStore.ts'; import { autoCast, autoCastBooleanStrict } from '../utility/common_utils.ts'; import { @@ -66,12 +66,11 @@ import { RequestTarget } from './RequestTarget.ts'; import harperLogger from '../utility/logging/harper_logger.ts'; import { throttle } from '../server/throttle.ts'; import { RocksDatabase } from '@harperfast/rocksdb-js'; -import { LMDBTransaction, ImmediateTransaction as ImmediateLMDBTransaction } from './LMDBTransaction'; -import { contentTypes } from '../server/serverHelpers/contentTypes'; +import { LMDBTransaction, ImmediateTransaction as ImmediateLMDBTransaction } from './LMDBTransaction.ts'; +import { contentTypes } from '../server/serverHelpers/contentTypes.ts'; const { sortBy } = lodash; const { validateAttribute } = lmdbProcessRows; - export type Attribute = { name: string; type: 'ID' | 'Int' | 'Float' | 'Long' | 'String' | 'Boolean' | 'Date' | 'Bytes' | 'Any' | 'BigInt' | 'Blob' | string; @@ -102,7 +101,11 @@ NULL_WITH_TIMESTAMP[8] = 0xc0; // null const UNCACHEABLE_TIMESTAMP = Infinity; // we use this when dynamic content is accessed that we can't safely cache, and this prevents earlier timestamps from change the "last" modification const RECORD_PRUNING_INTERVAL = 60000; // one minute const CACHEABLE_STATUS_CODES = new Set([200, 203, 204, 206, 300, 301, 308, 404, 405, 410, 414, 501]); -envMngr.initSync(); +try { + envMngr.initSync(); +} catch { + /* tolerate ESM cycle TDZ; bin entry will re-call later */ +} const LMDB_PREFETCH_WRITES = envMngr.get(CONFIG_PARAMS.STORAGE_PREFETCHWRITES); const LOCK_TIMEOUT = 10000; export const INVALIDATED = 1; @@ -309,7 +312,7 @@ export function makeTable(options) { // perform the write of an individual write event const writeUpdate = async (event, context) => { const value = event.value; - const Table = event.table ? databases[databaseName][event.table] : TableResource; + const Table: any = event.table ? databases[databaseName][event.table] : TableResource; if ( databaseName === SYSTEM_SCHEMA_NAME && (event.table === SYSTEM_TABLE_NAMES.ROLE_TABLE_NAME || event.table === SYSTEM_TABLE_NAMES.USER_TABLE_NAME) diff --git a/resources/analytics/write.ts b/resources/analytics/write.ts index abe0ba5c0..522d63d9d 100644 --- a/resources/analytics/write.ts +++ b/resources/analytics/write.ts @@ -1,5 +1,5 @@ import { parentPort, threadId } from 'worker_threads'; -import { onMessageByType } from '../../server/threads/manageThreads.js'; +import { onMessageByType } from '../../server/threads/manageThreads.ts'; import { getDatabases, table, isReadOnlyMode } from '../databases.ts'; import type { Databases, Table, Tables } from '../databases.ts'; import harperLogger from '../../utility/logging/harper_logger.ts'; @@ -8,19 +8,18 @@ const { getLogFilePath, forComponent } = harperLogger; import { dirname, join } from 'path'; import { open } from 'fs/promises'; import { getNextMonotonicTime } from '../../utility/lmdb/commonUtility.ts'; -import { get as envGet, getHdbBasePath, initSync } from '../../utility/environment/environmentManager.ts'; +import { get as envGet, getHdbBasePath } from '../../utility/environment/environmentManager.ts'; import { CONFIG_PARAMS } from '../../utility/hdbTerms.ts'; import { server } from '../../server/Server.ts'; import * as fs from 'node:fs'; import { getAnalyticsHostnameTable, nodeIds, stableNodeId } from './hostnames.ts'; import { METRIC } from './metadata.ts'; import { RocksDatabase } from '@harperfast/rocksdb-js'; +import { onStartup } from '../../utility/lifecycle.ts'; const log = forComponent('analytics').conditional; const isBun = typeof globalThis.Bun !== 'undefined'; -initSync(); - type ActionCallback = (action: Action) => void; export type Value = number | boolean | ActionCallback; interface Action { @@ -128,8 +127,6 @@ export function recordAction(value: Value, metric: string, path?: string, method if (!sendAnalyticsTimeout) sendAnalytics(); } -server.recordAnalytics = recordAction; - export function recordActionBinary(value, metric, path?, method?, type?) { recordAction(Boolean(value), metric, path, method, type); } @@ -708,7 +705,7 @@ function getAnalyticsTable() { ); } -if (!parentPort) onMessageByType(ANALYTICS_REPORT_TYPE, recordAnalytics); +if (!parentPort) setImmediate(() => onMessageByType(ANALYTICS_REPORT_TYPE, recordAnalytics)); let scheduledTasksRunning; function startScheduledTasks() { scheduledTasksRunning = true; @@ -867,3 +864,8 @@ function rebalance({ counts, values, totalCount }, resetCounts: boolean) { else counts.set(targetCounts); } */ + +// Wire server singletons during the startup phase +onStartup(() => { + server.recordAnalytics = recordAction; +}); diff --git a/resources/auditStore.ts b/resources/auditStore.ts index cecc34dbc..ba1a55629 100644 --- a/resources/auditStore.ts +++ b/resources/auditStore.ts @@ -1,8 +1,8 @@ import { readKey, writeKey } from 'ordered-binary'; -import { initSync, get as envGet } from '../utility/environment/environmentManager.ts'; +import { get as envGet } from '../utility/environment/environmentManager.ts'; import { AUDIT_STORE_NAME } from '../utility/lmdb/terms.ts'; import { CONFIG_PARAMS } from '../utility/hdbTerms.ts'; -import { getWorkerIndex, getWorkerCount } from '../server/threads/manageThreads.js'; +import { getWorkerIndex, getWorkerCount } from '../server/threads/manageThreads.ts'; import { convertToMS } from '../utility/common_utils.ts'; import { PREVIOUS_TIMESTAMP_PLACEHOLDER, LAST_TIMESTAMP_PLACEHOLDER } from './RecordEncoder.ts'; import * as harperLogger from '../utility/logging/harper_logger.ts'; @@ -30,7 +30,6 @@ import { isReadOnlyMode } from './databases.ts'; * username * remaining bytes (optional, not included for deletes/invalidation): the record itself, using the same encoding as its primary store */ -initSync(); export type AuditRecord = { version: number; diff --git a/resources/blob.ts b/resources/blob.ts index a4f036ae7..ddbb689d6 100644 --- a/resources/blob.ts +++ b/resources/blob.ts @@ -33,7 +33,8 @@ import { import type { StatsFs } from 'node:fs'; import { createDeflate, deflate } from 'node:zlib'; import { Readable, pipeline } from 'node:stream'; -import { ensureDirSync } from 'fs-extra'; +import _fs_extra from 'fs-extra'; +const { ensureDirSync } = _fs_extra; import { get as envGet, getHdbBasePath } from '../utility/environment/environmentManager.ts'; import { CONFIG_PARAMS } from '../utility/hdbTerms.ts'; import { join, dirname } from 'path'; diff --git a/resources/dataLoader.ts b/resources/dataLoader.ts index e10c803db..937e5276e 100644 --- a/resources/dataLoader.ts +++ b/resources/dataLoader.ts @@ -1,15 +1,31 @@ import { basename, extname } from 'node:path'; import { createHash } from 'node:crypto'; import { parseDocument } from 'yaml'; -import { Databases, databases, table, Tables, tables } from './databases.ts'; -import { getWorkerIndex } from '../server/threads/manageThreads'; +import { type Databases, databases, table, type Tables, tables } from './databases.ts'; +import { getWorkerIndex } from '../server/threads/manageThreads.ts'; import { HTTP_STATUS_CODES } from '../utility/errors/commonErrors.ts'; import { ClientError } from '../utility/errors/hdbError.ts'; import harperLogger from '../utility/logging/harper_logger.ts'; -import { Attribute } from './Table.ts'; -import { FileEntry } from '../components/EntryHandler.ts'; - -const dataLoaderLogger = harperLogger.forComponent('dataLoader'); +import { type Attribute } from './Table.ts'; +import { type FileEntry } from '../components/EntryHandler.ts'; + +// Resolve the logger lazily so unit tests can stub `forComponent` between +// module load and first use (otherwise transitive importers — componentLoader +// pulled in via http.ts / threadServer.ts / operations.ts — capture the real +// logger before dataLoader.test.js installs its stub). +let _dataLoaderLogger: any; +function getDataLoaderLogger() { + if (!_dataLoaderLogger) _dataLoaderLogger = harperLogger.forComponent('dataLoader'); + return _dataLoaderLogger; +} +const dataLoaderLogger = new Proxy( + {}, + { + get(_t, prop) { + return getDataLoaderLogger()[prop]; + }, + } +) as any; /** System table name for storing data loader hashes */ const DATA_LOADER_HASH_TABLE = 'hdb_dataloader_hash'; diff --git a/resources/databases.ts b/resources/databases.ts index 44f185ad3..cbc3fa295 100644 --- a/resources/databases.ts +++ b/resources/databases.ts @@ -8,19 +8,19 @@ import { unlink } from 'node:fs/promises'; import { getBaseSchemaPath, getTransactionAuditStoreBasePath, -} from '../dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js'; +} from '../dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.ts'; import { makeTable } from './Table.ts'; import OpenEnvironmentObject from '../utility/lmdb/OpenEnvironmentObject.ts'; import { CONFIG_PARAMS, LEGACY_DATABASES_DIR_NAME, DATABASES_DIR_NAME } from '../utility/hdbTerms.ts'; -import { getConfigPath } from '../config/configUtils.js'; +import { getConfigPath } from '../config/configUtils.ts'; import { _assignPackageExport } from '../globals.js'; import { getIndexedValues } from '../utility/lmdb/commonUtility.ts'; import * as signalling from '../utility/signalling.ts'; -import { SchemaEventMsg } from '../server/threads/itc.js'; +import { SchemaEventMsg } from '../server/threads/itc.ts'; import { workerData } from 'worker_threads'; import harperLogger from '../utility/logging/harper_logger.ts'; const { forComponent } = harperLogger; -import * as manageThreads from '../server/threads/manageThreads.js'; +import * as manageThreads from '../server/threads/manageThreads.ts'; import { openAuditStore, readAuditEntry, createAuditEntry, type AuditRecord } from './auditStore.ts'; import { handleLocalTimeForGets } from './RecordEncoder.ts'; import { deleteRootBlobPathsForDB } from './blob.ts'; @@ -31,7 +31,7 @@ import { replayLogs } from './replayLogs.ts'; import { totalmem } from 'node:os'; import { RocksIndexStore } from './RocksIndexStore.ts'; import { when } from '../utility/when.ts'; -import { isProcessRunning } from '../utility/processManagement/processManagement.js'; +import { isProcessRunning } from '../utility/processManagement/processManagement.ts'; /** * Check if Harper is running in read-only mode. @@ -40,7 +40,7 @@ import { isProcessRunning } from '../utility/processManagement/processManagement * - --readonly CLI flag * - storage.readOnly config setting */ -let _isReadOnlyMode: boolean | undefined; +var _isReadOnlyMode: boolean | undefined; export function isReadOnlyMode(): boolean { if (_isReadOnlyMode !== undefined) return _isReadOnlyMode; // Check environment variable @@ -66,15 +66,24 @@ export function isReadOnlyMode(): boolean { function createOpenDBIObject(dupSort = false, isPrimary = false) { return new OpenDBIObject(dupSort, isPrimary); } -const logger = forComponent('storage'); +var logger = forComponent('storage'); -const DEFAULT_DATABASE_NAME = 'data'; -const DEFINED_TABLES = Symbol('defined-tables'); -const DEFAULT_COMPRESSION_THRESHOLD = (envGet(CONFIG_PARAMS.STORAGE_PAGESIZE) || 4096) - 60; // larger than this requires multiple pages -initSync(); +var DEFAULT_DATABASE_NAME = 'data'; +var DEFINED_TABLES = Symbol('defined-tables'); +var DEFAULT_COMPRESSION_THRESHOLD = (envGet(CONFIG_PARAMS.STORAGE_PAGESIZE) || 4096) - 60; // larger than this requires multiple pages +// Initialise env on module load to mirror main-branch behaviour. Tests +// (and the api-test setup in particular) reach env.get(...) through +// configUtils without otherwise calling initSync, so without this the +// flat config object stays uninitialised and downstream code paths +// (e.g. installApplications -> getConfigPath(COMPONENTSROOT)) see null. +try { + initSync(); +} catch { + /* tolerate ESM cycle TDZ; bin entry will re-call later */ +} // I don't know if this is the best place for this, but somewhere we need to specify which tables // replicate by default: -export const NON_REPLICATING_SYSTEM_TABLES = [ +export var NON_REPLICATING_SYSTEM_TABLES = [ 'hdb_temp', 'hdb_certificate', 'hdb_raw_analytics', @@ -145,10 +154,10 @@ export type DatabaseWatcherEventMap = { dropDatabase: [databaseName: string]; }; -export const databaseEventsEmitter = new EventEmitter(); +export var databaseEventsEmitter = new EventEmitter(); -export const tables: Tables = Object.create(null); -export const databases: Databases = Object.create(null); +export var tables: Tables = Object.create(null); +export var databases: Databases = Object.create(null); function openRocksDatabase(path: string, options: RocksDatabaseOptions & { dupSort?: boolean }) { options.disableWAL ??= true; @@ -204,19 +213,19 @@ function openRocksDatabase(path: string, options: RocksDatabaseOptions & { dupSo return db; } -const lmdbDatabaseEnvs = new Map(); -const rocksdbDatabaseEnvs = new Map(); +var lmdbDatabaseEnvs = new Map(); +var rocksdbDatabaseEnvs = new Map(); // set the following in both global and exports _assignPackageExport('databases', databases); _assignPackageExport('tables', tables); -const NEXT_TABLE_ID = Symbol.for('next-table-id'); -let loadedDatabases; // indicates if we have loaded databases from the file system yet +var NEXT_TABLE_ID = Symbol.for('next-table-id'); +var loadedDatabases; // indicates if we have loaded databases from the file system yet // This is used to track all the databases that are found when iterating through the file system so that anything that is missing // can be removed: -let definedDatabases: Map>; +var definedDatabases: Map>; /** * This gets the set of tables from the default database ("data"). @@ -1231,8 +1240,8 @@ export function table(tableDefinition: TableDefinition): Tabl } } } -const MAX_OUTSTANDING_INDEXING = 1000; -const MIN_OUTSTANDING_INDEXING = 10; +var MAX_OUTSTANDING_INDEXING = 1000; +var MIN_OUTSTANDING_INDEXING = 10; async function runIndexing(Table, attributes, indicesToRemove) { try { logger.info(`Indexing ${Table.tableName} attributes`, attributes); diff --git a/resources/graphql.ts b/resources/graphql.ts index f70672838..cb1ed2250 100644 --- a/resources/graphql.ts +++ b/resources/graphql.ts @@ -1,7 +1,7 @@ import { dirname } from 'path'; import { Script } from 'node:vm'; import { table } from './databases.ts'; -import { getWorkerIndex } from '../server/threads/manageThreads.js'; +import { getWorkerIndex } from '../server/threads/manageThreads.ts'; import { Resources } from './Resources.ts'; import type { NamedTypeNode, StringValueNode } from 'graphql'; import { once } from 'node:events'; diff --git a/resources/login.ts b/resources/login.ts index 842c6882f..a93c58c5e 100644 --- a/resources/login.ts +++ b/resources/login.ts @@ -1,20 +1,30 @@ import { Resource } from './Resource.ts'; -import { Scope } from '../components/Scope.ts'; +import type { Scope } from '../components/Scope.ts'; + +// Login class is created lazily because login.ts is loaded transitively from +// Resource.ts's own static graph; declaring `class Login extends Resource` at +// module-load would hit Resource's TDZ under that ESM cycle. +let LoginClass: any; +function getLoginClass() { + if (LoginClass) return LoginClass; + // @ts-ignore + LoginClass = class Login extends Resource { + static async get(_id, _body, _request) { + // TODO: Return a login page + } + static async post(_id, body, request) { + const { username, password } = body; + return { + data: await request.login(username, password), + }; + } + }; + return LoginClass; +} + export function handleApplication(scope: Scope) { - scope.resources.set('login', Login); + scope.resources.set('login', getLoginClass()); scope.resources.loginPath = (request) => { return '/login?redirect=' + encodeURIComponent(request.url); }; } -// @ts-ignore -class Login extends Resource { - static async get(_id, _body, _request) { - // TODO: Return a login page - } - static async post(_id, body, request) { - const { username, password } = body; - return { - data: await request.login(username, password), - }; - } -} diff --git a/resources/replayLogs.ts b/resources/replayLogs.ts index a3c605336..0220abec6 100644 --- a/resources/replayLogs.ts +++ b/resources/replayLogs.ts @@ -1,7 +1,7 @@ import { RocksDatabase, Transaction as RocksTransaction } from '@harperfast/rocksdb-js'; import { Resource } from './Resource.ts'; import type { Context } from './ResourceInterface.ts'; -import * as logger from '../utility/logging/harper_logger.js'; +import * as logger from '../utility/logging/harper_logger.ts'; import { DatabaseTransaction } from './DatabaseTransaction.ts'; import { RocksTransactionLogStore } from './RocksTransactionLogStore.ts'; import { isMainThread } from 'node:worker_threads'; diff --git a/resources/roles.ts b/resources/roles.ts index 117ed0f17..23e608b1c 100644 --- a/resources/roles.ts +++ b/resources/roles.ts @@ -1,8 +1,8 @@ import { getDatabases } from './databases.ts'; import { alterRole, addRole } from '../security/role.ts'; import { parseDocument } from 'yaml'; -import { isEqual } from 'lodash'; - +import _lodash from 'lodash'; +const { isEqual } = _lodash; const USERS_NOT_DBS = ['super_user', 'structure_user']; /** diff --git a/resources/search.ts b/resources/search.ts index 75b761bba..dd2ba2f3e 100644 --- a/resources/search.ts +++ b/resources/search.ts @@ -6,7 +6,7 @@ import { INVALIDATED, EVICTED } from './Table.ts'; import type { DirectCondition, Id } from './ResourceInterface.ts'; import { RequestTarget } from './RequestTarget.ts'; import { lastMetadata } from './RecordEncoder.ts'; -import { recordAction } from './analytics/write'; +import { recordAction } from './analytics/write.ts'; import { RocksDatabase } from '@harperfast/rocksdb-js'; // these are ratios/percentages of overall table size diff --git a/resources/transactionBroadcast.ts b/resources/transactionBroadcast.ts index 71f01e255..cc606a195 100644 --- a/resources/transactionBroadcast.ts +++ b/resources/transactionBroadcast.ts @@ -1,4 +1,4 @@ -import { warn } from '../utility/logging/harper_logger.js'; +import { warn } from '../utility/logging/harper_logger.ts'; import { IterableEventQueue } from './IterableEventQueue.ts'; import { keyArrayToString } from './Resources.ts'; import type { Id } from './ResourceInterface.ts'; diff --git a/security/auth.ts b/security/auth.ts index f286bd64e..88fd8dec8 100644 --- a/security/auth.ts +++ b/security/auth.ts @@ -8,45 +8,63 @@ import * as env from '../utility/environment/environmentManager.ts'; import { CONFIG_PARAMS, AUTH_AUDIT_STATUS, AUTH_AUDIT_TYPES } from '../utility/hdbTerms.ts'; import harperLogger from '../utility/logging/harper_logger.ts'; const { forComponent, AuthAuditLog } = harperLogger; -import serverHandlers from '../server/itc/serverHandlers.js'; +import serverHandlers from '../server/itc/serverHandlers.ts'; const { user } = serverHandlers; import { Headers } from '../server/serverHelpers/Headers.ts'; import { convertToMS } from '../utility/common_utils.ts'; import { verifyCertificate } from './certificateVerification/index.ts'; import { serializeMessage } from '../server/serverHelpers/contentTypes.ts'; +import { onStartup } from '../utility/lifecycle.ts'; const authLogger = forComponent('authentication'); const { debug } = authLogger; const authEventLog = authLogger.withTag('auth-event'); -env.initSync(); - -const appsCorsAccesslist = env.get(CONFIG_PARAMS.HTTP_CORSACCESSLIST); -const appsCors = env.get(CONFIG_PARAMS.HTTP_CORS); -const operationsCorsAccesslist = env.get(CONFIG_PARAMS.OPERATIONSAPI_NETWORK_CORSACCESSLIST); -const operationsCors = env.get(CONFIG_PARAMS.OPERATIONSAPI_NETWORK_CORS); +try { + env.initSync(); +} catch { + /* tolerate ESM cycle TDZ; bin entry will re-call later */ +} +// Config-derived state is populated during startup, after the environment is +// initialized and the module graph is fully linked. Reading config here at +// module-load would either TDZ inside an ESM cycle or pick up stale defaults. +let appsCorsAccesslist: any; +let appsCors: any; +let operationsCorsAccesslist: any; +let operationsCors: any; +let _sessionTable: Table | undefined; +let ENABLE_SESSIONS: boolean = true; +let AUTHORIZE_LOCAL: any = process.env.AUTHENTICATION_AUTHORIZELOCAL ?? process.env.DEV_MODE; +let LOG_AUTH_SUCCESSFUL: boolean = false; +let LOG_AUTH_FAILED: boolean = false; -const _sessionTable = table({ - table: 'hdb_session', - database: 'system', - attributes: [{ name: 'id', isPrimaryKey: true }, { name: 'user' }], -}); function getSessionTable() { return _sessionTable; } -const ENABLE_SESSIONS = env.get(CONFIG_PARAMS.AUTHENTICATION_ENABLESESSIONS) ?? true; -// check the environment for a flag to bypass authentication (for testing) since it doesn't necessarily get set on child threads -let AUTHORIZE_LOCAL = - process.env.AUTHENTICATION_AUTHORIZELOCAL ?? - env.get(CONFIG_PARAMS.AUTHENTICATION_AUTHORIZELOCAL) ?? - process.env.DEV_MODE; -const LOG_AUTH_SUCCESSFUL = env.get(CONFIG_PARAMS.LOGGING_AUDITAUTHEVENTS_LOGSUCCESSFUL) ?? false; -const LOG_AUTH_FAILED = env.get(CONFIG_PARAMS.LOGGING_AUDITAUTHEVENTS_LOGFAILED) ?? false; const DEFAULT_COOKIE_EXPIRES = 'Tue, 01 Oct 8307 19:33:20 GMT'; let authorizationCache = new Map(); -server.onInvalidatedUser(() => { - // TODO: Eventually we probably want to be able to invalidate individual users - authorizationCache = new Map(); + +onStartup(() => { + appsCorsAccesslist = env.get(CONFIG_PARAMS.HTTP_CORSACCESSLIST); + appsCors = env.get(CONFIG_PARAMS.HTTP_CORS); + operationsCorsAccesslist = env.get(CONFIG_PARAMS.OPERATIONSAPI_NETWORK_CORSACCESSLIST); + operationsCors = env.get(CONFIG_PARAMS.OPERATIONSAPI_NETWORK_CORS); + ENABLE_SESSIONS = env.get(CONFIG_PARAMS.AUTHENTICATION_ENABLESESSIONS) ?? true; + AUTHORIZE_LOCAL = + process.env.AUTHENTICATION_AUTHORIZELOCAL ?? + env.get(CONFIG_PARAMS.AUTHENTICATION_AUTHORIZELOCAL) ?? + process.env.DEV_MODE; + LOG_AUTH_SUCCESSFUL = env.get(CONFIG_PARAMS.LOGGING_AUDITAUTHEVENTS_LOGSUCCESSFUL) ?? false; + LOG_AUTH_FAILED = env.get(CONFIG_PARAMS.LOGGING_AUDITAUTHEVENTS_LOGFAILED) ?? false; + _sessionTable = table
({ + table: 'hdb_session', + database: 'system', + attributes: [{ name: 'id', isPrimaryKey: true }, { name: 'user' }], + }); + server.onInvalidatedUser(() => { + // TODO: Eventually we probably want to be able to invalidate individual users + authorizationCache = new Map(); + }); }); let bypassUser: any; export function bypassAuth() { diff --git a/security/certificateVerification/certificateVerificationSource.ts b/security/certificateVerification/certificateVerificationSource.ts index 9f304c2c3..2202d1879 100644 --- a/security/certificateVerification/certificateVerificationSource.ts +++ b/security/certificateVerification/certificateVerificationSource.ts @@ -23,62 +23,88 @@ async function loadVerificationFunctions() { } /** - * Certificate Verification Source that can handle both CRL and OCSP + * Certificate Verification Source that can handle both CRL and OCSP. + * + * Late-bound via a Proxy because this module is loaded inside Resource.ts's + * own static-graph SCC (via the auth.ts → server-utilities chain), so a + * class-extends declaration at module-top would TDZ on `Resource`. The Proxy + * defers the `extends Resource` evaluation to first construct/access. */ -export class CertificateVerificationSource extends Resource { - async get(query: Query) { - const id = query.id as string; +let _CertificateVerificationSource: any; +function getCertificateVerificationSource(): any { + if (!_CertificateVerificationSource) { + _CertificateVerificationSource = class CertificateVerificationSource extends Resource { + async get(query: Query) { + const id = query.id as string; - // Get the certificate data from requestContext - const context = this.getContext() as SourceContext; - const requestContext = context?.requestContext; + // Get the certificate data from requestContext + const context = this.getContext() as SourceContext; + const requestContext = context?.requestContext; - if (!requestContext || !requestContext.certPem || !requestContext.issuerPem) { - // Likely a source request for an expired entry - we can't verify without cert and issuer data - return null; - } + if (!requestContext || !requestContext.certPem || !requestContext.issuerPem) { + // Likely a source request for an expired entry - we can't verify without cert and issuer data + return null; + } - const { certPem: certPemStr, issuerPem: issuerPemStr, ocspUrls, config } = requestContext; + const { certPem: certPemStr, issuerPem: issuerPemStr, ocspUrls, config } = requestContext; - // Determine method from cache key - let method: string; - if (id.startsWith('crl:')) { - method = 'crl'; - } else if (id.startsWith('ocsp:')) { - method = 'ocsp'; - } else { - method = 'unknown'; - } + // Determine method from cache key + let method: string; + if (id.startsWith('crl:')) { + method = 'crl'; + } else if (id.startsWith('ocsp:')) { + method = 'ocsp'; + } else { + method = 'unknown'; + } - // Load verification functions - await loadVerificationFunctions(); + // Load verification functions + await loadVerificationFunctions(); - // Perform verification based on method - let result; - let methodConfig; + // Perform verification based on method + let result; + let methodConfig; - if (method === 'crl') { - methodConfig = config.crl; - // Pass distributionPoint as an array if available (for CRL fetch) - const crlUrls = requestContext.distributionPoint ? [requestContext.distributionPoint] : undefined; - result = await performCRLCheck(certPemStr, issuerPemStr, methodConfig, crlUrls); - } else if (method === 'ocsp') { - methodConfig = config.ocsp; - result = await performOCSPCheck(certPemStr, issuerPemStr, methodConfig, ocspUrls); - } else { - throw new Error(`Unsupported verification method: ${method} for ID: ${id}`); - } + if (method === 'crl') { + methodConfig = config.crl; + // Pass distributionPoint as an array if available (for CRL fetch) + const crlUrls = requestContext.distributionPoint ? [requestContext.distributionPoint] : undefined; + result = await performCRLCheck(certPemStr, issuerPemStr, methodConfig, crlUrls); + } else if (method === 'ocsp') { + methodConfig = config.ocsp; + result = await performOCSPCheck(certPemStr, issuerPemStr, methodConfig, ocspUrls); + } else { + throw new Error(`Unsupported verification method: ${method} for ID: ${id}`); + } - // Handle result consistently - const expiresAt = Date.now() + methodConfig.cacheTtl; + // Handle result consistently + const expiresAt = Date.now() + methodConfig.cacheTtl; - return { - certificate_id: id, - status: result.status, - reason: result.reason, - checked_at: Date.now(), - expiresAt, - method, + return { + certificate_id: id, + status: result.status, + reason: result.reason, + checked_at: Date.now(), + expiresAt, + method, + }; + } }; } + return _CertificateVerificationSource; } + +export const CertificateVerificationSource: any = new Proxy(function () {} as any, { + construct(_target, args) { + return Reflect.construct(getCertificateVerificationSource(), args); + }, + get(_target, prop) { + return getCertificateVerificationSource()[prop]; + }, + has(_target, prop) { + return prop in getCertificateVerificationSource(); + }, + getPrototypeOf() { + return getCertificateVerificationSource().prototype; + }, +}); diff --git a/security/certificateVerification/crlVerification.ts b/security/certificateVerification/crlVerification.ts index decef6a2c..6a270599e 100644 --- a/security/certificateVerification/crlVerification.ts +++ b/security/certificateVerification/crlVerification.ts @@ -54,62 +54,89 @@ function getCertificateCacheTable() { } /** - * CRL fetching and validation source + * CRL fetching and validation source. + * + * Late-bound via a Proxy because this module sits inside Resource.ts's + * static-graph SCC, so a class-extends declaration at module-top would TDZ on + * `Resource`. The Proxy defers the `extends Resource` evaluation to first + * construct/access — same pattern as resources/ErrorResource.ts and + * security/certificateVerification/certificateVerificationSource.ts. */ -class CertificateRevocationListSource extends Resource { - async get(id: string) { - const context = this.getContext() as SourceContext; - const requestContext = context?.requestContext; - - if (!requestContext?.distributionPoint || !requestContext?.issuerPem) { - throw new Error(`No CRL data provided for cache key: ${id}`); - } +let _CertificateRevocationListSource: any; +function getCertificateRevocationListSource(): any { + if (!_CertificateRevocationListSource) { + _CertificateRevocationListSource = class CertificateRevocationListSource extends Resource { + async get(id: string) { + const context = this.getContext() as SourceContext; + const requestContext = context?.requestContext; + + if (!requestContext?.distributionPoint || !requestContext?.issuerPem) { + throw new Error(`No CRL data provided for cache key: ${id}`); + } - const { distributionPoint, issuerPem: issuerPemStr, config } = requestContext; + const { distributionPoint, issuerPem: issuerPemStr, config } = requestContext; - try { - const result = await downloadAndParseCRL(distributionPoint, issuerPemStr, config.timeout); + try { + const result = await downloadAndParseCRL(distributionPoint, issuerPemStr, config.timeout); - // Set expiration - use the CRL's nextUpdate time or configured TTL, whichever is sooner - const crlExpiry = result.next_update; - const configExpiry = Date.now() + config.cacheTtl; - const expiresAt = Math.min(crlExpiry, configExpiry); + // Set expiration - use the CRL's nextUpdate time or configured TTL, whichever is sooner + const crlExpiry = result.next_update; + const configExpiry = Date.now() + config.cacheTtl; + const expiresAt = Math.min(crlExpiry, configExpiry); - return { - ...result, - expiresAt, - }; - } catch (error) { - logger.error?.(`CRL fetch error for: ${distributionPoint} - ${error}`); + return { + ...result, + expiresAt, + }; + } catch (error) { + logger.error?.(`CRL fetch error for: ${distributionPoint} - ${error}`); - if (error instanceof CRLSignatureVerificationError) { - throw error; - } + if (error instanceof CRLSignatureVerificationError) { + throw error; + } - // Check failure mode - if (config.failureMode === 'fail-closed') { - // Cache the error for faster recovery - const expiresAt = Date.now() + ERROR_CACHE_TTL; + // Check failure mode + if (config.failureMode === 'fail-closed') { + // Cache the error for faster recovery + const expiresAt = Date.now() + ERROR_CACHE_TTL; + + return { + crl_id: id, + distribution_point: distributionPoint, + issuer_dn: 'unknown', + crl_blob: Buffer.alloc(0), + this_update: Date.now(), + next_update: expiresAt, + signature_valid: false, + expiresAt, + }; + } - return { - crl_id: id, - distribution_point: distributionPoint, - issuer_dn: 'unknown', - crl_blob: Buffer.alloc(0), - this_update: Date.now(), - next_update: expiresAt, - signature_valid: false, - expiresAt, - }; + // Fail open - return null to not cache + logger.warn?.('CRL fetch failed, not caching (fail-open mode)'); + return null; + } } - - // Fail open - return null to not cache - logger.warn?.('CRL fetch failed, not caching (fail-open mode)'); - return null; - } + }; } + return _CertificateRevocationListSource; } +const CertificateRevocationListSource: any = new Proxy(function () {} as any, { + construct(_target, args) { + return Reflect.construct(getCertificateRevocationListSource(), args); + }, + get(_target, prop) { + return getCertificateRevocationListSource()[prop]; + }, + has(_target, prop) { + return prop in getCertificateRevocationListSource(); + }, + getPrototypeOf() { + return getCertificateRevocationListSource().prototype; + }, +}); + // Lazy-load Harper tables let crlCacheTable: ReturnType; let revokedCertificateTable: ReturnType; diff --git a/security/fastifyAuth.ts b/security/fastifyAuth.ts index 004d353ee..adbdd5be3 100644 --- a/security/fastifyAuth.ts +++ b/security/fastifyAuth.ts @@ -8,7 +8,8 @@ import * as util from 'util'; import * as userFunctions from './user.ts'; const cbFindValidateUsers = util.callbackify(userFunctions.findAndValidateUser); import * as hdbTerms from '../utility/hdbTerms.ts'; -import * as tokenAuthentication from './tokenAuthentication.ts'; +import * as _tokenAuthentication from './tokenAuthentication.ts'; +const tokenAuthentication = _tokenAuthentication; import { AccessViolation } from '../utility/errors/hdbError.ts'; import { authentication } from './auth.ts'; diff --git a/security/jsLoader.ts b/security/jsLoader.ts index 35287114b..044fe11ce 100644 --- a/security/jsLoader.ts +++ b/security/jsLoader.ts @@ -6,11 +6,18 @@ import { models as harperModelsSingleton } from '../resources/models/Models.ts'; import { readFile } from 'node:fs/promises'; import { dirname, isAbsolute } from 'node:path'; import { pathToFileURL, fileURLToPath } from 'node:url'; -import { SourceTextModule, SyntheticModule, createContext, runInContext, runInThisContext } from 'node:vm'; +import * as _vm from 'node:vm'; +import { createContext, runInContext, runInThisContext } from 'node:vm'; +// SourceTextModule and SyntheticModule require `--experimental-vm-modules`. Pull +// them off the vm namespace at runtime so the named ESM import doesn't fail when +// the flag isn't passed (e.g. CLI paths that never touch the JS loader). +const { SourceTextModule, SyntheticModule } = _vm as any; +type SourceTextModule = any; +type SyntheticModule = any; import { ApplicationScope } from '../components/ApplicationScope.ts'; import logger from '../utility/logging/harper_logger.ts'; import { createRequire } from 'node:module'; -import * as env from '../utility/environment/environmentManager'; +import * as env from '../utility/environment/environmentManager.ts'; import * as child_process from 'node:child_process'; import { CONFIG_PARAMS } from '../utility/hdbTerms.ts'; import { contentTypes } from '../server/serverHelpers/contentTypes.ts'; @@ -27,7 +34,7 @@ import { } from 'node:fs'; import { join } from 'node:path'; import { EventEmitter } from 'node:events'; -import { whenComponentsLoaded } from '../server/threads/threadServer.js'; +import { whenComponentsLoaded } from '../server/threads/threadServer.ts'; type Lockdown = 'none' | 'freeze' | 'ses' | 'freeze-after-load'; const APPLICATIONS_LOCKDOWN: Lockdown = env.get(CONFIG_PARAMS.APPLICATIONS_LOCKDOWN); @@ -826,7 +833,6 @@ const ALLOWED_NODE_BUILTIN_MODULES = env.get(CONFIG_PARAMS.APPLICATIONS_ALLOWEDB return true; }, }; -const ALLOWED_COMMANDS = new Set(env.get(CONFIG_PARAMS.APPLICATIONS_ALLOWEDSPAWNCOMMANDS) ?? []); const child_processConstrained: any = { exec: createSpawn(child_process.exec), execFile: createSpawn(child_process.execFile), @@ -982,8 +988,14 @@ function acquirePidFileLock( } function createSpawn(spawnFunction: (...args: any) => child_process.ChildProcess, alwaysAllow?: boolean) { - const basePath = env.getHdbBasePath(); return function (command: string, args?: any, options?: any, callback?: (...args: any[]) => void) { + // Resolved lazily because jsLoader may be evaluated before the environment + // is initialized (e.g. component loading paths in unit tests). Mirror + // defaultConfig.yaml's applications.allowedSpawnCommands as the + // pre-config fallback so the allow-list isn't empty in that window. + const basePath = env.getHdbBasePath(); + const allowedSpawn = env.get(CONFIG_PARAMS.APPLICATIONS_ALLOWEDSPAWNCOMMANDS) ?? ['npm', 'node']; + const ALLOWED_COMMANDS = new Set(allowedSpawn); if (!ALLOWED_COMMANDS.has(command.split(' ')[0]) && !alwaysAllow) { throw new Error(`Command ${command} is not allowed`); } diff --git a/security/keys.ts b/security/keys.ts index 4027e72b1..34b8c7394 100644 --- a/security/keys.ts +++ b/security/keys.ts @@ -2,7 +2,7 @@ import * as path from 'path'; import { watch } from 'chokidar'; -import * as fs from 'fs-extra'; +import fs from 'fs-extra'; import * as forge from 'node-forge'; import * as net from 'net'; import { generateKeyPair as generateKeyPairOrig, X509Certificate, createPrivateKey, randomBytes } from 'node:crypto'; @@ -17,11 +17,12 @@ import * as envManager from '../utility/environment/environmentManager.ts'; import * as hdbTerms from '../utility/hdbTerms.ts'; import * as certificatesTerms from '../utility/terms/certificates.js'; -const tls = require('node:tls'); +import tls from 'node:tls'; import { relative, join } from 'node:path'; import assignCmdenvVars from '../utility/assignCmdEnvVariables.ts'; -import * as configUtils from '../config/configUtils.js'; +import * as configUtils from '../config/configUtils.ts'; +import { filterArgsAgainstRuntimeConfig } from '../config/harperConfigEnvVars.ts'; import { table, getDatabases, databases } from '../resources/databases.ts'; const logger = forComponent('tls').conditional; const { CONFIG_PARAMS } = hdbTerms; @@ -31,7 +32,7 @@ import { getThisNodeName, getThisNodeUrl, urlToNodeName, clearThisNodeName } fro export const getPrivateKeys = () => privateKeys; import { readFileSync, statSync } from 'node:fs'; -import { getTicketKeys, onMessageFromWorkers } from '../server/threads/manageThreads.js'; +import { getTicketKeys, onMessageFromWorkers } from '../server/threads/manageThreads.ts'; import { isMainThread } from 'worker_threads'; import { TLSSocket } from 'node:tls'; @@ -58,12 +59,15 @@ export function generateSerialNumber() { return bytes.toString('hex'); } -onMessageFromWorkers(async (message) => { - if (message.type === hdbTerms.ITC_EVENT_TYPES.RESTART) { - envManager.initSync(true); - // This will also call loadCertificates - await reviewSelfSignedCert(); - } +// Defer registration to setImmediate so manageThreads internal state is initialized +setImmediate(() => { + onMessageFromWorkers(async (message) => { + if (message.type === hdbTerms.ITC_EVENT_TYPES.RESTART) { + envManager.initSync(true); + // This will also call loadCertificates + await reviewSelfSignedCert(); + } + }); }); let certificateTable; @@ -652,7 +656,6 @@ export function updateConfigCert() { // Filter out any cert config keys already set by HARPER_SET_CONFIG so we don't overwrite them // with defaults. On first boot, HARPER_SET_CONFIG values are written to the config file during // createConfigFile(), but updateConfigCert() runs afterward without re-applying HARPER_SET_CONFIG. - const { filterArgsAgainstRuntimeConfig } = require('../config/harperConfigEnvVars'); const filteredCerts = filterArgsAgainstRuntimeConfig(newCerts); configUtils.updateConfigValue(undefined, undefined, filteredCerts, false, true); diff --git a/security/permissionsTranslator.js b/security/permissionsTranslator.ts similarity index 97% rename from security/permissionsTranslator.js rename to security/permissionsTranslator.ts index d921d4ec7..6b9a72da0 100644 --- a/security/permissionsTranslator.js +++ b/security/permissionsTranslator.ts @@ -1,15 +1,10 @@ -'use strict'; - -const _ = require('lodash'); -const terms = require('../utility/hdbTerms.ts'); -const { handleHDBError, hdbErrors } = require('../utility/errors/hdbError.ts'); +import _ from 'lodash'; +import * as terms from '../utility/hdbTerms.ts'; +import { handleHDBError, hdbErrors } from '../utility/errors/hdbError.ts'; const { HDB_ERROR_MSGS, HTTP_STATUS_CODES } = hdbErrors; -const logger = require('../utility/logging/harper_logger.ts'); - -module.exports = { - getRolePermissions, -}; +import logger from '../utility/logging/harper_logger.ts'; +export { getRolePermissions }; const rolePermsMap = Object.create(null); const permsTemplateObj = (permsKey) => ({ key: permsKey, perms: {} }); diff --git a/security/role.ts b/security/role.ts index 53eac59ca..0c8ad80d1 100644 --- a/security/role.ts +++ b/security/role.ts @@ -17,7 +17,7 @@ import SearchByHashObject from '../dataLayer/SearchByHashObject.ts'; import { handleHDBError } from '../utility/errors/hdbError.ts'; import { HDB_ERROR_MSGS, HTTP_STATUS_CODES } from '../utility/errors/commonErrors.ts'; -import { UserEventMsg } from '../server/threads/itc.js'; +import { UserEventMsg } from '../server/threads/itc.ts'; function scrubRoleDetails(role) { try { diff --git a/security/tokenAuthentication.ts b/security/tokenAuthentication.ts index 9c1b8826b..b4b69dcdc 100644 --- a/security/tokenAuthentication.ts +++ b/security/tokenAuthentication.ts @@ -18,10 +18,13 @@ import { findAndValidateUser, type User } from './user.ts'; import { update } from '../dataLayer/insert.ts'; import UpdateObject from '../dataLayer/UpdateObject.ts'; import * as signalling from '../utility/signalling.ts'; -import { UserEventMsg } from '../server/threads/itc.js'; +import { UserEventMsg } from '../server/threads/itc.ts'; import * as env from '../utility/environment/environmentManager.ts'; -env.initSync(); - +try { + env.initSync(); +} catch { + /* tolerate ESM cycle TDZ; bin entry will re-call later */ +} type StringValue = SignOptions['expiresIn']; const OPERATION_TOKEN_TIMEOUT: StringValue = env.get(CONFIG_PARAMS.AUTHENTICATION_OPERATIONTOKENTIMEOUT) || '1d'; const REFRESH_TOKEN_TIMEOUT: StringValue = env.get(CONFIG_PARAMS.AUTHENTICATION_REFRESHTOKENTIMEOUT) || '30d'; diff --git a/security/user.ts b/security/user.ts index 7013dfe52..d310c51e8 100644 --- a/security/user.ts +++ b/security/user.ts @@ -92,11 +92,16 @@ import * as validate from 'validate.js'; import * as logger from '../utility/logging/harper_logger.ts'; import { promisify } from 'util'; import * as env from '../utility/environment/environmentManager.ts'; -import systemSchema from '../json/systemSchema.json'; +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { PACKAGE_ROOT } from '../utility/packageUtils.js'; +const systemSchema: Record = JSON.parse( + readFileSync(join(PACKAGE_ROOT, 'json/systemSchema.json'), 'utf-8') +); import { hdbErrors, ClientError } from '../utility/errors/hdbError.ts'; const { HTTP_STATUS_CODES, AUTHENTICATION_ERROR_MSGS, HDB_ERROR_MSGS } = hdbErrors; -const { UserEventMsg } = require('../server/threads/itc.js'); -import * as _ from 'lodash'; +import { UserEventMsg } from '../server/threads/itc.ts'; +import _ from 'lodash'; import * as harperLogger from '../utility/logging/harper_logger.ts'; // Need to use `.js` even for other TS files since TS compiler won't replace requires. @@ -105,14 +110,15 @@ import * as password from '../utility/password.ts'; import { server } from '../server/Server.ts'; import * as terms from '../utility/hdbTerms.ts'; import { expandOperationsPerms } from '../utility/operationPermissions.ts'; +import { onStartup } from '../utility/lifecycle.ts'; -server.getUser = (username: string, password?: string | null): Promise => { +function getUserImpl(username: string, password?: string | null): Promise { return findAndValidateUser(username, password, password != null); -}; +} -server.authenticateUser = (username: string, password?: string | null): Promise => { +function authenticateUserImpl(username: string, password?: string | null): Promise { return findAndValidateUser(username, password); -}; +} const USER_ATTRIBUTE_ALLOWLIST = { username: true, @@ -440,7 +446,7 @@ async function getSuperUser(): Promise { } let invalidateCallbacks = []; -(server as any).invalidateUser = function (user: User | any) { +function invalidateUserImpl(user: User | any) { for (let callback of invalidateCallbacks) { try { callback(user); @@ -448,8 +454,17 @@ let invalidateCallbacks = []; harperLogger.error('Error invalidating user', error); } } -}; +} -server.onInvalidatedUser = function (callback) { +function onInvalidatedUserImpl(callback) { invalidateCallbacks.push(callback); -}; +} + +// Wire server singletons during the startup phase +onStartup(() => { + server.getUser = getUserImpl; + server.authenticateUser = authenticateUserImpl; + server.onInvalidatedUser = onInvalidatedUserImpl; + // @ts-expect-error - invalidateUser is wired here as a server-singleton extension; not declared on the Server type + server.invalidateUser = invalidateUserImpl; +}); diff --git a/server/DurableSubscriptionsSession.ts b/server/DurableSubscriptionsSession.ts index ea8f58241..285ab818d 100644 --- a/server/DurableSubscriptionsSession.ts +++ b/server/DurableSubscriptionsSession.ts @@ -1,14 +1,15 @@ import { table } from '../resources/databases.ts'; +import { onStartup } from '../utility/lifecycle.ts'; import { keyArrayToString, resources } from '../resources/Resources.ts'; import { getNextMonotonicTime } from '../utility/lmdb/commonUtility.ts'; import { warn, trace } from '../utility/logging/harper_logger.ts'; import { transaction } from '../resources/transaction.ts'; -import { getWorkerIndex } from '../server/threads/manageThreads.js'; -import { whenComponentsLoaded } from '../server/threads/threadServer.js'; +import { getWorkerIndex } from '../server/threads/manageThreads.ts'; +import { whenComponentsLoaded } from '../server/threads/threadServer.ts'; import { server } from '../server/Server.ts'; -import { RequestTarget } from '../resources/RequestTarget'; -import { cloneDeep } from 'lodash'; - +import { RequestTarget } from '../resources/RequestTarget.ts'; +import _lodash from 'lodash'; +const { cloneDeep } = _lodash; const AWAITING_ACKS_HIGH_WATER_MARK = 100; let _DurableSession: any; function getDurableSession() { @@ -49,7 +50,10 @@ function getLastWill() { } return _LastWill; } -if (getWorkerIndex() === 0) { +// Defer to startup so `whenComponentsLoaded` (an `export const` in threadServer.ts) +// is past TDZ — this module is loaded inside threadServer's static-graph SCC. +onStartup(() => { + if (getWorkerIndex() !== 0) return; (async () => { await whenComponentsLoaded; await new Promise((resolve) => setTimeout(resolve, 2000)); @@ -65,7 +69,7 @@ if (getWorkerIndex() === 0) { getLastWill().delete(will.id); } })(); -} +}); /** * This is used for durable sessions, that is sessions in MQTT that are not "clean" sessions (and with QoS >= 1 diff --git a/server/REST.ts b/server/REST.ts index e079f421f..65f2ac5ee 100644 --- a/server/REST.ts +++ b/server/REST.ts @@ -10,7 +10,7 @@ import { Headers, mergeHeaders } from '../server/serverHelpers/Headers.ts'; import { generateJsonApi } from '../resources/openApi.ts'; import { Request } from '../server/serverHelpers/Request.ts'; -import { RequestTarget } from '../resources/RequestTarget'; +import { RequestTarget } from '../resources/RequestTarget.ts'; const { errorToString } = harperLogger; const etagBytes = new Uint8Array(8); diff --git a/server/Server.ts b/server/Server.ts index fc2e55ab9..ed10a242a 100644 --- a/server/Server.ts +++ b/server/Server.ts @@ -2,7 +2,7 @@ import { Socket } from 'net'; import { _assignPackageExport } from '../globals.js'; import type { Value } from '../resources/analytics/write.ts'; import type { Resources } from '../resources/Resources.ts'; -import { OperationDefinition } from './serverHelpers/serverUtilities.ts'; +import { type OperationDefinition } from './serverHelpers/serverUtilities.ts'; import { Duplex } from 'stream'; import { Request } from './serverHelpers/Request.ts'; diff --git a/server/fastifyRoutes.ts b/server/fastifyRoutes.ts index 88db39fbc..01d36b580 100644 --- a/server/fastifyRoutes.ts +++ b/server/fastifyRoutes.ts @@ -2,18 +2,18 @@ import { dirname, basename } from 'path'; import { existsSync } from 'fs'; import fastify from 'fastify'; import fastifyCors from '@fastify/cors'; -import requestTimePlugin from './serverHelpers/requestTimePlugin.js'; +import requestTimePlugin from './serverHelpers/requestTimePlugin.ts'; import autoload from '@fastify/autoload'; import * as env from '../utility/environment/environmentManager.ts'; import { CONFIG_PARAMS } from '../utility/hdbTerms.ts'; import * as harperLogger from '../utility/logging/harper_logger.ts'; import { realExit } from './threads/workerProcessGuard.ts'; -import * as hdbCore from './fastifyRoutes/plugins/hdbCore.js'; +import * as hdbCore from './fastifyRoutes/plugins/hdbCore.ts'; import * as userSchema from '../security/user.ts'; -import getServerOptions from './fastifyRoutes/helpers/getServerOptions.js'; -import getCORSOptions from './fastifyRoutes/helpers/getCORSOptions.js'; -import getHeaderTimeoutConfig from './fastifyRoutes/helpers/getHeaderTimeoutConfig.js'; -import { serverErrorHandler } from '../server/serverHelpers/serverHandlers.js'; +import getServerOptions from './fastifyRoutes/helpers/getServerOptions.ts'; +import getCORSOptions from './fastifyRoutes/helpers/getCORSOptions.ts'; +import getHeaderTimeoutConfig from './fastifyRoutes/helpers/getHeaderTimeoutConfig.ts'; +import { serverErrorHandler } from '../server/serverHelpers/serverHandlers.ts'; import { registerContentHandlers } from '../server/serverHelpers/contentTypes.ts'; import { server } from './Server.ts'; diff --git a/server/fastifyRoutes/helpers/getCORSOptions.js b/server/fastifyRoutes/helpers/getCORSOptions.ts similarity index 79% rename from server/fastifyRoutes/helpers/getCORSOptions.js rename to server/fastifyRoutes/helpers/getCORSOptions.ts index de4a205b0..222afd8b7 100644 --- a/server/fastifyRoutes/helpers/getCORSOptions.js +++ b/server/fastifyRoutes/helpers/getCORSOptions.ts @@ -1,8 +1,5 @@ -'use strict'; - -const env = require('../../../utility/environment/environmentManager.ts'); -env.initSync(); -const { CONFIG_PARAMS } = require('../../../utility/hdbTerms.ts'); +import * as env from '../../../utility/environment/environmentManager.ts'; +import { CONFIG_PARAMS } from '../../../utility/hdbTerms.ts'; /** * Builds CORS options object to pass to cors plugin when/if it needs to be registered with Fastify @@ -33,4 +30,4 @@ function getCORSOptions() { return corsOptions; } -module.exports = getCORSOptions; +export default getCORSOptions; diff --git a/server/fastifyRoutes/helpers/getHeaderTimeoutConfig.js b/server/fastifyRoutes/helpers/getHeaderTimeoutConfig.js deleted file mode 100644 index ed4e4400c..000000000 --- a/server/fastifyRoutes/helpers/getHeaderTimeoutConfig.js +++ /dev/null @@ -1,15 +0,0 @@ -'use strict'; - -const env = require('../../../utility/environment/environmentManager.ts'); -env.initSync(); -const terms = require('../../../utility/hdbTerms.ts'); - -/** - * Returns header timeout value from config file - * @returns {*} - */ -function getHeaderTimeoutConfig() { - return env.get(terms.CONFIG_PARAMS.HTTP_HEADERSTIMEOUT) ?? 60000; -} - -module.exports = getHeaderTimeoutConfig; diff --git a/server/fastifyRoutes/helpers/getHeaderTimeoutConfig.ts b/server/fastifyRoutes/helpers/getHeaderTimeoutConfig.ts new file mode 100644 index 000000000..55bd21cdb --- /dev/null +++ b/server/fastifyRoutes/helpers/getHeaderTimeoutConfig.ts @@ -0,0 +1,12 @@ +import * as env from '../../../utility/environment/environmentManager.ts'; +import * as terms from '../../../utility/hdbTerms.ts'; + +/** + * Returns header timeout value from config file + * @returns {*} + */ +function getHeaderTimeoutConfig() { + return env.get(terms.CONFIG_PARAMS.HTTP_HEADERSTIMEOUT) ?? 60000; +} + +export default getHeaderTimeoutConfig; diff --git a/server/fastifyRoutes/helpers/getServerOptions.js b/server/fastifyRoutes/helpers/getServerOptions.ts similarity index 81% rename from server/fastifyRoutes/helpers/getServerOptions.js rename to server/fastifyRoutes/helpers/getServerOptions.ts index f0ae0226a..6346c50cd 100644 --- a/server/fastifyRoutes/helpers/getServerOptions.js +++ b/server/fastifyRoutes/helpers/getServerOptions.ts @@ -1,8 +1,5 @@ -'use strict'; - -const env = require('../../../utility/environment/environmentManager.ts'); -env.initSync(); -const { CONFIG_PARAMS } = require('../../../utility/hdbTerms.ts'); +import * as env from '../../../utility/environment/environmentManager.ts'; +import { CONFIG_PARAMS } from '../../../utility/hdbTerms.ts'; // eslint-disable-next-line no-magic-numbers const REQ_MAX_BODY_SIZE = 1024 * 1024 * 1024; //this is 1GB in bytes @@ -30,4 +27,4 @@ function getServerOptions(isHttps) { }; } -module.exports = getServerOptions; +export default getServerOptions; diff --git a/server/fastifyRoutes/plugins/hdbCore.js b/server/fastifyRoutes/plugins/hdbCore.ts similarity index 86% rename from server/fastifyRoutes/plugins/hdbCore.js rename to server/fastifyRoutes/plugins/hdbCore.ts index 8b5e7965d..4ea45e51f 100644 --- a/server/fastifyRoutes/plugins/hdbCore.js +++ b/server/fastifyRoutes/plugins/hdbCore.ts @@ -1,12 +1,10 @@ -'use strict'; +import fp from 'fastify-plugin'; -const fp = require('fastify-plugin'); - -const { +import { handlePostRequest, authHandler, reqBodyValidationHandler, -} = require('../../../server/serverHelpers/serverHandlers.js'); +} from '../../../server/serverHelpers/serverHandlers.ts'; /** * Generates a fastify plugin containing three core methods @@ -36,4 +34,4 @@ async function convertAsyncIterators(response) { return response; } -module.exports = fp(hdbCore); +export default fp(hdbCore); diff --git a/server/http.ts b/server/http.ts index 9189e9421..dde6dcee9 100644 --- a/server/http.ts +++ b/server/http.ts @@ -10,8 +10,8 @@ import harperLogger from '../utility/logging/harper_logger.ts'; import { parentPort } from 'node:worker_threads'; import * as env from '../utility/environment/environmentManager.ts'; import * as terms from '../utility/hdbTerms.ts'; -import { getConfigPath } from '../config/configUtils.js'; -import { getTicketKeys, getWorkerIndex } from './threads/manageThreads.js'; +import { getConfigPath } from '../config/configUtils.ts'; +import { getTicketKeys, getWorkerIndex } from './threads/manageThreads.ts'; import { createTLSSelector } from '../security/keys.ts'; import { createSecureServer } from 'node:http2'; import { createServer as createSecureServerHttp1 } from 'node:https'; @@ -23,18 +23,15 @@ import { recordAction, recordActionBinary } from '../resources/analytics/write.t import { Readable, Writable } from 'node:stream'; import { mkdirSync, writeFileSync, unlinkSync, readdirSync } from 'node:fs'; import { join } from 'node:path'; -import { server, type ServerOptions, type HttpOptions, type UpgradeOptions, UpgradeListener } from './Server.ts'; +import { server, type ServerOptions, type HttpOptions, type UpgradeOptions, type UpgradeListener } from './Server.ts'; import { setPortServerMap, SERVERS } from './serverRegistry.ts'; import { getComponentName } from '../components/componentLoader.ts'; import { throttle } from './throttle.ts'; import { makeCallbackChain as buildCallbackChain } from './middlewareChain.ts'; import { WebSocketServer } from 'ws'; +import { onStartup } from '../utility/lifecycle.ts'; const { errorToString } = harperLogger; -server.http = httpServer; -server.request = onRequest; -server.ws = onWebSocket; -server.upgrade = onUpgrade; const websocketServers = {}; const httpServers = {}, httpChain = {}, @@ -1101,3 +1098,11 @@ export function getRequestId() { } return Number(Atomics.add(nextRequestId, 0, 1n)); } + +// Wire server singletons during the startup phase +onStartup(() => { + server.http = httpServer; + server.request = onRequest; + server.ws = onWebSocket; + server.upgrade = onUpgrade; +}); diff --git a/server/itc/serverHandlers.js b/server/itc/serverHandlers.ts similarity index 84% rename from server/itc/serverHandlers.js rename to server/itc/serverHandlers.ts index 1cdcc3685..e0a6b64d9 100644 --- a/server/itc/serverHandlers.js +++ b/server/itc/serverHandlers.ts @@ -1,17 +1,16 @@ -'use strict'; - /* global threads */ -const hdbLogger = require('../../utility/logging/harper_logger.ts'); -const hdbTerms = require('../../utility/hdbTerms.ts'); -const cleanLmdbMap = - require('../../utility/lmdb/cleanLMDBMap.ts').default || require('../../utility/lmdb/cleanLMDBMap.ts'); -const userSchema = require('../../security/user.ts'); -const { validateEvent } = require('../threads/itc.js'); -const harperBridge = - require('../../dataLayer/harperBridge/harperBridge.ts').default || - require('../../dataLayer/harperBridge/harperBridge.ts'); -const process = require('process'); -const { resetDatabases } = require('../../resources/databases.ts'); +import hdbLogger from '../../utility/logging/harper_logger.ts'; +import * as hdbTerms from '../../utility/hdbTerms.ts'; +import { internal } from '../../components/status/index.ts'; +import { getWorkerIndex } from '../threads/manageThreads.ts'; +import cleanLmdbMap from '../../utility/lmdb/cleanLMDBMap.ts'; +import * as userSchema from '../../security/user.ts'; +import { validateEvent } from '../threads/itc.ts'; +import harperBridge from '../../dataLayer/harperBridge/harperBridge.ts'; +import process from 'process'; +import { resetDatabases } from '../../resources/databases.ts'; +import { resources } from '../../resources/Resources.ts'; +import { generateJsonApi } from '../../resources/openApi.ts'; /** * This object/functions are passed to the ITC client instance and dynamically added as event handlers. @@ -66,7 +65,7 @@ async function syncSchemaMetadata(msg) { let databases = resetDatabases(); if (msg.table && msg.database) // wait for a write to finish to ensure all writes have been written - await databases[msg.database][msg.table].put(Symbol.for('write-verify'), null); + await (databases[msg.database][msg.table] as any).put(Symbol.for('write-verify'), null); } catch (e) { hdbLogger.error(e); } @@ -93,7 +92,7 @@ async function userHandler(event) { return; } - hdbLogger.trace(`ITC userHandler ${hdbTerms.HDB_ITC_CLIENT_PREFIX}${process.pid} received user event:`, event); + hdbLogger.trace(`ITC userHandler ${process.pid} received user event:`, event); await userSchema.setUsersWithRolesCache(); for (let listener of userListeners) listener(); } catch (err) { @@ -129,8 +128,6 @@ async function componentStatusRequestHandler(event) { hdbLogger.trace(`ITC componentStatusRequestHandler received request:`, event); // Get current thread's component status - const { internal } = require('../../components/status/index.ts'); - const { getWorkerIndex } = require('../threads/manageThreads.js'); const componentStatuses = internal.componentStatusRegistry.getAllStatuses(); // Convert Map to array for serialization @@ -183,13 +180,11 @@ async function resourceOpenApiRequestHandler(event) { hdbLogger.trace(`ITC resourceOpenApiRequestHandler received request:`, event); - const { resources } = require('../../resources/Resources.ts'); // Only respond if this thread has registered resources. Job-type workers with an empty // resources map must stay silent so that an app worker with real resources replies first. // If no worker has resources the main thread gets a 503 after the timeout, which is a // more honest response than silently returning an empty spec. if (!resources || resources.size === 0) return; - const { generateJsonApi } = require('../../resources/openApi.ts'); const openapi = generateJsonApi(resources, event.message.serverHttpURL); const originatorThreadId = event.message.originator; @@ -211,8 +206,7 @@ async function resourceOpenApiRequestHandler(event) { } } -module.exports = serverItcHandlers; +export default serverItcHandlers; // Named exports so consumers (e.g., MCP listChanged) can subscribe via // `userHandler.addListener(fn)` / `schemaHandler.addListener(fn)`. -module.exports.userHandler = userHandler; -module.exports.schemaHandler = schemaHandler; +export { userHandler, schemaHandler }; diff --git a/server/jobs/jobProcess.ts b/server/jobs/jobProcess.ts index 2c9e982e7..9972368ed 100644 --- a/server/jobs/jobProcess.ts +++ b/server/jobs/jobProcess.ts @@ -11,8 +11,8 @@ import * as user from '../../security/user.ts'; import * as serverUtils from '../serverHelpers/serverUtilities.ts'; import moment from 'moment'; import * as jobs from './jobs.ts'; -import { cloneDeep } from 'lodash'; - +import _lodash from 'lodash'; +const { cloneDeep } = _lodash; import { pathToFileURL } from 'node:url'; import { join } from 'node:path'; import { getEnvBuiltInComponents } from './../../components/Application.ts'; diff --git a/server/jobs/jobRunner.ts b/server/jobs/jobRunner.ts index dc1d19249..7f4557d51 100644 --- a/server/jobs/jobRunner.ts +++ b/server/jobs/jobRunner.ts @@ -1,7 +1,5 @@ 'use strict'; -import { join } from 'node:path'; - import * as hdbUtil from '../../utility/common_utils.ts'; import * as hdbTerms from '../../utility/hdbTerms.ts'; import moment from 'moment'; @@ -10,11 +8,11 @@ import log from '../../utility/logging/harper_logger.ts'; import * as jobs from './jobs.ts'; import * as hdbExport from '../../dataLayer/export.ts'; import * as hdbDelete from '../../dataLayer/delete.ts'; -import * as threadsStart from '../threads/manageThreads.js'; +import * as threadsStart from '../threads/manageThreads.ts'; import * as transactionLog from '../../utility/logging/transactionLog.ts'; import * as restart from '../../bin/restart.ts'; import { parentPort, isMainThread } from 'worker_threads'; -import { onMessageByType } from '../threads/manageThreads.js'; +import { onMessageByType } from '../threads/manageThreads.ts'; class RunnerMessage { job: any; @@ -133,7 +131,7 @@ async function runJob(runnerMessage: any, operation: any) { async function launchJobThread(job_id: any) { log.trace('launching job thread:', job_id); if (isMainThread) { - threadsStart.startWorker(join(__dirname, './jobProcess.js'), { + threadsStart.startWorker('server/jobs/jobProcess', { autoRestart: false, name: 'job', env: { ...process.env, [hdbTerms.PROCESS_NAME_ENV_PROP]: `JOB-${job_id}` }, @@ -148,7 +146,7 @@ async function launchJobThread(job_id: any) { if (isMainThread) { onMessageByType(hdbTerms.ITC_EVENT_TYPES.START_JOB, async (message) => { try { - threadsStart.startWorker(join(__dirname, './jobProcess.js'), { + threadsStart.startWorker('server/jobs/jobProcess', { autoRestart: false, name: 'job', env: { ...process.env, [hdbTerms.PROCESS_NAME_ENV_PROP]: `JOB-${message.jobId}` }, diff --git a/server/loadRootComponents.js b/server/loadRootComponents.ts similarity index 66% rename from server/loadRootComponents.js rename to server/loadRootComponents.ts index 1c4d676e2..3e2053601 100644 --- a/server/loadRootComponents.js +++ b/server/loadRootComponents.ts @@ -1,11 +1,11 @@ -const { isMainThread } = require('worker_threads'); -const { getTables } = require('../resources/databases.ts'); -const { loadComponentDirectories, loadComponent } = require('../components/componentLoader.ts'); -const { resetResources } = require('../resources/Resources.ts'); -const configUtils = require('../config/configUtils.js'); -const { dirname } = require('path'); -const { loadCertificates } = require('../security/keys.ts'); -const { installApplications } = require('../components/Application.ts'); +import { isMainThread } from 'worker_threads'; +import { getTables } from '../resources/databases.ts'; +import { loadComponentDirectories, loadComponent } from '../components/componentLoader.ts'; +import { resetResources } from '../resources/Resources.ts'; +import * as configUtils from '../config/configUtils.ts'; +import { dirname } from 'path'; +import { loadCertificates } from '../security/keys.ts'; +import { installApplications } from '../components/Application.ts'; let loadedComponents = new Map(); /** @@ -41,4 +41,4 @@ async function loadRootComponents(isWorkerThread = false) { if (allReady.length > 0) await Promise.all(allReady); } -module.exports.loadRootComponents = loadRootComponents; +export { loadRootComponents }; diff --git a/server/mqtt.ts b/server/mqtt.ts index dfed57504..4c8b10186 100644 --- a/server/mqtt.ts +++ b/server/mqtt.ts @@ -1,6 +1,7 @@ // for now we are using mqtt-packet, but we may implement some of this ourselves, particularly packet generation so that // we can implement more efficient progressive buffer allocation. -import { parser as makeParser, generate } from 'mqtt-packet'; +import _mqtt_packet from 'mqtt-packet'; +const { parser: makeParser, generate } = _mqtt_packet; import { getSession, DurableSubscriptionsSession } from './DurableSubscriptionsSession.ts'; import { getSuperUser } from '../security/user.ts'; import { serializeMessage, getDeserializer } from './serverHelpers/contentTypes.ts'; diff --git a/server/nodeName.ts b/server/nodeName.ts index b7a2cc0c0..06423e234 100644 --- a/server/nodeName.ts +++ b/server/nodeName.ts @@ -5,10 +5,14 @@ import * as env from '../utility/environment/environmentManager.ts'; import { logger } from '../utility/logging/logger.ts'; import { server } from './Server.ts'; -Object.defineProperty(server, 'hostname', { - get() { - return getThisNodeName(); - }, +// Defer to next tick so `server` (from Server.ts) is fully evaluated +// even when modules are loaded via an ESM cycle. +setImmediate(() => { + Object.defineProperty(server, 'hostname', { + get() { + return getThisNodeName(); + }, + }); }); let commonNameFromCert: string | undefined; diff --git a/server/operationsServer.ts b/server/operationsServer.ts index 0400ac59c..d9bfb5f4d 100644 --- a/server/operationsServer.ts +++ b/server/operationsServer.ts @@ -2,15 +2,24 @@ import cluster from 'cluster'; import zlib from 'node:zlib'; import * as env from '../utility/environment/environmentManager.ts'; -env.initSync(); +try { + env.initSync(); +} catch { + /* tolerate ESM cycle TDZ; bin entry will re-call later */ +} import * as terms from '../utility/hdbTerms.ts'; import harperLogger from '../utility/logging/harper_logger.ts'; import { realExit } from './threads/workerProcessGuard.ts'; -import fastify, { FastifyInstance, FastifyReply, FastifyRequest, FastifyServerOptions } from 'fastify'; +import fastify, { + type FastifyInstance, + type FastifyReply, + type FastifyRequest, + type FastifyServerOptions, +} from 'fastify'; import fastifyCors, { type FastifyCorsOptions } from '@fastify/cors'; import fastifyCompress from '@fastify/compress'; import fastifyStatic from '@fastify/static'; -import requestTimePlugin from './serverHelpers/requestTimePlugin.js'; +import requestTimePlugin from './serverHelpers/requestTimePlugin.ts'; import guidePath from 'path'; import { PACKAGE_ROOT } from '../utility/packageUtils.js'; import * as globalSchema from '../utility/globalSchema.ts'; @@ -23,7 +32,7 @@ import { handlePostRequest, serverErrorHandler, reqBodyValidationHandler, -} from './serverHelpers/serverHandlers.js'; +} from './serverHelpers/serverHandlers.ts'; import { registerBunFastifyInstance } from './http.ts'; import { registerContentHandlers } from './serverHelpers/contentTypes.ts'; import { getConfigObj } from '../config/configUtils.js'; @@ -49,7 +58,7 @@ export { operationsServer as startOnMainThread }; /** * Builds a Harper server. */ -async function operationsServer(options: ServerOptions & { resources?: Resources }) { +async function operationsServer(options: ServerOptions & { resources?: Resources } = {}) { try { harperLogger.debug('In Fastify server' + process.cwd()); harperLogger.debug(`Running with NODE_ENV set as: ${process.env.NODE_ENV}`); diff --git a/server/serverHelpers/contentTypes.ts b/server/serverHelpers/contentTypes.ts index d5fcf7f87..92bf1a799 100644 --- a/server/serverHelpers/contentTypes.ts +++ b/server/serverHelpers/contentTypes.ts @@ -14,9 +14,19 @@ import { Blob } from '../../resources/blob.ts'; // TODO: Only load this if fastify is loaded import fp from 'fastify-plugin'; import { parseMultipartRequest } from './multipartParser.ts'; -const SERIALIZATION_BIGINT = envMgr.get(CONFIG_PARAMS.SERIALIZATION_BIGINT) !== false; -const JSONStringify = SERIALIZATION_BIGINT ? stringify : JSON.stringify; -const JSONParse = SERIALIZATION_BIGINT ? parse : JSON.parse; +// Resolve lazily: reading config at module-load time would TDZ under ESM +// because configUtils.ts is mid-evaluation when this module is imported. +let _serializationBigint: boolean | undefined; +function getSerializationBigint(): boolean { + if (_serializationBigint === undefined) { + _serializationBigint = envMgr.get(CONFIG_PARAMS.SERIALIZATION_BIGINT) !== false; + } + return _serializationBigint; +} +const JSONStringify = ((value: any, ...rest: any[]) => + (getSerializationBigint() ? stringify : JSON.stringify)(value, ...rest)) as typeof JSON.stringify; +const JSONParse = ((text: string, ...rest: any[]) => + (getSerializationBigint() ? parse : JSON.parse)(text, ...rest)) as typeof JSON.parse; const PUBLIC_ENCODE_OPTIONS = { useRecords: false, @@ -37,7 +47,12 @@ const mediaTypes = new Map< >(); export const contentTypes = mediaTypes; -server.contentTypes = contentTypes as any; +// Defer attachment to `server` because under ESM cycles Server.ts may still +// be mid-evaluation when this module is loaded. +setImmediate(() => { + // @ts-expect-error - server.contentTypes is wired here as a runtime extension; declared type differs from the Map + server.contentTypes = contentTypes; +}); _assignPackageExport('contentTypes', contentTypes); // TODO: Make these monomorphic for faster access. And use a Map mediaTypes.set('application/json', { @@ -353,8 +368,13 @@ export function findBestSerializer(incomingMessage) { return { serializer: bestSerializer, type: bestType, parameters: bestParameters }; } -// about an average TCP packet size (if headers included) -const COMPRESSION_THRESHOLD = envMgr.get(CONFIG_PARAMS.HTTP_COMPRESSIONTHRESHOLD); +// about an average TCP packet size (if headers included). Read lazily — +// reading config at module-load time would fault under ESM cycles where +// the environment manager hasn't finished initializing. +let _compressionThreshold: number | undefined; +function COMPRESSION_THRESHOLD() { + return (_compressionThreshold ??= envMgr.get(CONFIG_PARAMS.HTTP_COMPRESSIONTHRESHOLD)); +} /** * Serialize a response * @param responseData @@ -365,7 +385,7 @@ const COMPRESSION_THRESHOLD = envMgr.get(CONFIG_PARAMS.HTTP_COMPRESSIONTHRESHOLD export function serialize(responseData, request, responseObject) { // TODO: Maybe support other compression encodings; browsers basically universally support brotli, but Node's HTTP // client itself actually (just) supports gzip/deflate - let canCompress = COMPRESSION_THRESHOLD && request.headers.asObject?.['accept-encoding']?.includes('br'); + let canCompress = COMPRESSION_THRESHOLD() && request.headers.asObject?.['accept-encoding']?.includes('br'); let responseBody; if (responseData?.contentType != null && responseData.data != null) { // we use this as a special marker for blobs of data that are explicitly one content type @@ -421,7 +441,7 @@ export function serialize(responseData, request, responseObject) { } responseBody = serializer.serializer.serialize(responseData, responseObject); } - if (canCompress && responseBody?.length > COMPRESSION_THRESHOLD) { + if (canCompress && responseBody?.length > COMPRESSION_THRESHOLD()) { // TODO: Only do this if the size is large and we can cache the result (otherwise use logic above) responseObject.headers.set('Content-Encoding', 'br'); // if we have a single buffer (or string) we compress in a single async call diff --git a/server/serverHelpers/requestTimePlugin.js b/server/serverHelpers/requestTimePlugin.ts similarity index 86% rename from server/serverHelpers/requestTimePlugin.js rename to server/serverHelpers/requestTimePlugin.ts index c369b48ec..67e956741 100644 --- a/server/serverHelpers/requestTimePlugin.js +++ b/server/serverHelpers/requestTimePlugin.ts @@ -1,10 +1,10 @@ -const { recordAction, recordActionBinary } = require('../../resources/analytics/write.ts'); -const fp = require('fastify-plugin'); +import { recordAction, recordActionBinary } from '../../resources/analytics/write.ts'; +import fp from 'fastify-plugin'; const ESTIMATED_HEADER_SIZE = 200; // it is very expensive to actually measure HTTP response header size (we change it // ourselves) with an unacceptable performance penalty, so we estimate this part -module.exports = fp( +export default fp( function (fastify, opts, done) { // eslint-disable-next-line require-await fastify.addHook('onResponse', async (request, reply) => { @@ -12,15 +12,15 @@ module.exports = fp( let _time = reply.elapsedTime; }); // eslint-disable-next-line require-await - fastify.addHook('onSend', async (request, reply, payload) => { + fastify.addHook('onSend', async (request, reply, payload: any) => { let responseTime = reply.elapsedTime; let startTransfer = performance.now(); - let config = reply.request.routeOptions; + let config: any = reply.request.routeOptions; let action; let type; let method; if (config.config?.isOperation) { - action = request.body?.operation; + action = (request.body as any)?.operation; type = 'operation'; } else { action = config.url; diff --git a/server/serverHelpers/serverHandlers.js b/server/serverHelpers/serverHandlers.ts similarity index 85% rename from server/serverHelpers/serverHandlers.js rename to server/serverHelpers/serverHandlers.ts index a34fd375a..8691b0f3f 100644 --- a/server/serverHelpers/serverHandlers.js +++ b/server/serverHelpers/serverHandlers.ts @@ -1,16 +1,16 @@ -'use strict'; +import * as terms from '../../utility/hdbTerms.ts'; +import * as hdbUtil from '../../utility/common_utils.ts'; +import harperLogger from '../../utility/logging/harper_logger.ts'; +import { handleHDBError, hdbErrors } from '../../utility/errors/hdbError.ts'; +import { isMainThread } from 'worker_threads'; +import { Readable } from 'stream'; -const terms = require('../../utility/hdbTerms.ts'); -const hdbUtil = require('../../utility/common_utils.ts'); -const harperLogger = require('../../utility/logging/harper_logger.ts'); -const { realExit } = require('../threads/workerProcessGuard.ts'); -const { handleHDBError, hdbErrors } = require('../../utility/errors/hdbError.ts'); -const { isMainThread } = require('worker_threads'); -const { Readable } = require('stream'); +import _os from 'os'; +// Rewire-compat alias: unit tests inject `os` by name via rewire __set__ +let os = _os; +import { realExit } from '../threads/workerProcessGuard.ts'; -const os = require('os'); - -const auth = require('../../security/fastifyAuth.ts'); +import * as auth from '../../security/fastifyAuth.ts'; // this is a hack to suppress a deprecation warning and can be removed once `auth.authorize` // is converted to an async function @@ -20,10 +20,10 @@ function pAuthorize(req, resp) { }); } -const serverUtilities = require('./serverUtilities.ts'); -const { applyImpersonation } = require('../../security/impersonation.ts'); -const { createGzip, constants } = require('zlib'); -const { ProgressEmitter, createSSEResponseStream } = require('./progressEmitter.ts'); +import * as serverUtilities from './serverUtilities.ts'; +import { applyImpersonation } from '../../security/impersonation.ts'; +import { createGzip, constants } from 'zlib'; +import { ProgressEmitter, createSSEResponseStream } from './progressEmitter.ts'; // Operations that support `Accept: text/event-stream` for live progress streaming. The // handler attaches a ProgressEmitter as req.body.progress so the operation can emit phase @@ -155,9 +155,9 @@ async function handlePostRequest(req, res, _bypassAuth = false) { return createSSEResponseStream(emitter, () => serverUtilities.processLocalTransaction(req, operation_function)); } - let result = await serverUtilities.processLocalTransaction(req, operation_function); - if (result instanceof Readable && result.headers) { - for (let [name, value] of result.headers) { + let result: any = await serverUtilities.processLocalTransaction(req, operation_function); + if (result instanceof Readable && (result as any).headers) { + for (let [name, value] of (result as any).headers) { res.header(name, value); } // fastify-compress has one job. I don't know why it can't do it. So we compress here to @@ -174,7 +174,7 @@ async function handlePostRequest(req, res, _bypassAuth = false) { } } -module.exports = { +export { authHandler, authAndEnsureUserOnRequest, handlePostRequest, diff --git a/server/serverHelpers/serverUtilities.ts b/server/serverHelpers/serverUtilities.ts index 8cc9fcb4a..6695a83d9 100644 --- a/server/serverHelpers/serverUtilities.ts +++ b/server/serverHelpers/serverUtilities.ts @@ -6,7 +6,7 @@ import * as delete_ from '../../dataLayer/delete.ts'; import readAuditLog from '../../dataLayer/readAuditLog.ts'; import * as user from '../../security/user.ts'; import * as role from '../../security/role.ts'; -import customFunctionOperations from '../../components/operations.js'; +import * as customFunctionOperations from '../../components/operations.ts'; import harperLogger from '../../utility/logging/harper_logger.ts'; import readLog from '../../utility/logging/readLog.ts'; import * as export_ from '../../dataLayer/export.ts'; @@ -23,7 +23,7 @@ import { systemInformation } from '../../utility/environment/systemInformation.t import * as jobRunner from '../jobs/jobRunner.ts'; import * as tokenAuthentication from '../../security/tokenAuthentication.ts'; import * as auth from '../../security/auth.ts'; -import configUtils from '../../config/configUtils.js'; +import * as configUtils from '../../config/configUtils.ts'; import * as transactionLog from '../../utility/logging/transactionLog.ts'; import * as npmUtilities from '../../utility/npmUtilities.ts'; import { _assignPackageExport } from '../../globals.js'; @@ -58,6 +58,7 @@ const GLOBAL_SCHEMA_UPDATE_OPERATIONS_ENUM = { }; import { OperationFunctionObject } from './OperationFunctionObject.ts'; +import { onStartup } from '../../utility/lifecycle.ts'; type ValueOf = T[keyof T]; export type OperationFunctionName = ValueOf; @@ -106,7 +107,6 @@ export async function processLocalTransaction(req: OperationRequest, operationFu export const OPERATION_FUNCTION_MAP = initializeOperationFunctionMap(); -server.operation = operation; export type OperationDefinition = { name: string; execute: (operation: any) => any | Promise; @@ -119,9 +119,9 @@ export type OperationDefinition = { * Register an operation function with the server. * @param operationDefinition */ -server.registerOperation = (operationDefinition: OperationDefinition) => { +function registerOperation(operationDefinition: OperationDefinition) { OPERATION_FUNCTION_MAP.set(operationDefinition.name as any, new OperationFunctionObject(operationDefinition.execute)); -}; +} export function chooseOperation(json: OperationRequestBody) { let getOpResult: OperationFunctionObject; @@ -478,3 +478,9 @@ function initializeOperationFunctionMap(): Map { + server.operation = operation; + server.registerOperation = registerOperation; +}); diff --git a/server/static.ts b/server/static.ts index cffe396bd..bb7ffb710 100644 --- a/server/static.ts +++ b/server/static.ts @@ -1,6 +1,6 @@ import { realpathSync, existsSync } from 'node:fs'; import { join } from 'node:path'; -import { Scope } from '../components/Scope'; +import { Scope } from '../components/Scope.ts'; import send from 'send'; /** diff --git a/server/status/index.ts b/server/status/index.ts index 071484944..2f99fc6b7 100644 --- a/server/status/index.ts +++ b/server/status/index.ts @@ -14,7 +14,6 @@ export type { StatusId, StatusRecord, StatusValueMap } from './definitions.ts'; export { STATUS_IDS, DEFAULT_STATUS_ID } from './definitions.ts'; const { HTTP_STATUS_CODES } = hdbErrors; - // For direct function calls, we don't need the operation fields type StatusRequestBody = { id: StatusId; diff --git a/server/storageReclamation.ts b/server/storageReclamation.ts index e21796e3f..fb24e9de6 100644 --- a/server/storageReclamation.ts +++ b/server/storageReclamation.ts @@ -1,10 +1,14 @@ import { statfs } from 'node:fs/promises'; -import { getWorkerIndex, getWorkerCount } from '../server/threads/manageThreads.js'; +import { getWorkerIndex, getWorkerCount } from '../server/threads/manageThreads.ts'; import { logger } from '../utility/logging/logger.ts'; import { CONFIG_PARAMS } from '../utility/hdbTerms.ts'; import * as envMgr from '../utility/environment/environmentManager.ts'; import { convertToMS } from '../utility/common_utils.ts'; -envMgr.initSync(); +try { + envMgr.initSync(); +} catch { + /* tolerate ESM cycle TDZ; bin entry will re-call later */ +} const reclamationHandlers = new Map< string, { priority: number; handler: (priority: number) => Promise | void }[] diff --git a/server/threads/itc.js b/server/threads/itc.ts similarity index 56% rename from server/threads/itc.js rename to server/threads/itc.ts index 992237f4e..13da3d416 100644 --- a/server/threads/itc.js +++ b/server/threads/itc.ts @@ -1,29 +1,36 @@ -'use strict'; +import * as hdbUtils from '../../utility/common_utils.ts'; +import * as hdbTerms from '../../utility/hdbTerms.ts'; +import { ITC_ERRORS } from '../../utility/errors/commonErrors.ts'; +import { threadId } from 'worker_threads'; +import { + onMessageFromWorkers, + broadcastWithAcknowledgement as _broadcastWithAcknowledgement, +} from './manageThreads.ts'; +// Rewire-compat alias: unit tests inject `broadcastWithAcknowledgement` by name via rewire __set__ +let broadcastWithAcknowledgement = _broadcastWithAcknowledgement; -const hdbUtils = require('../../utility/common_utils.ts'); -const hdbTerms = require('../../utility/hdbTerms.ts'); -const { ITC_ERRORS } = require('../../utility/errors/commonErrors.ts'); -const { threadId } = require('worker_threads'); -const { onMessageFromWorkers, broadcastWithAcknowledgement } = require('./manageThreads.js'); - -module.exports = { - sendItcEvent, - validateEvent, - SchemaEventMsg, - UserEventMsg, -}; +export { sendItcEvent, validateEvent, SchemaEventMsg, UserEventMsg }; let serverItcHandlers; -onMessageFromWorkers(async (event, sender) => { - serverItcHandlers = serverItcHandlers || require('../itc/serverHandlers.js'); - validateEvent(event); - if (serverItcHandlers[event.type]) { - await serverItcHandlers[event.type](event); - } - if (event.requestId && sender) - sender.postMessage({ - type: 'ack', - id: event.requestId, - }); +// Defer registration so manageThreads.ts is fully evaluated when we read from +// it (ESM cycle would otherwise leave its internal state uninitialized). +setImmediate(() => { + onMessageFromWorkers(async (event, sender) => { + if (!serverItcHandlers) { + const m: any = await import('../itc/serverHandlers.ts'); + // Dynamic import returns the namespace { default: handlersObj }; in + // CJS-dist it can be double-wrapped as { default: { default: ... } }. + serverItcHandlers = m.default?.default ?? m.default ?? m; + } + validateEvent(event); + if (serverItcHandlers[event.type]) { + await serverItcHandlers[event.type](event); + } + if (event.requestId && sender) + sender.postMessage({ + type: 'ack', + id: event.requestId, + }); + }); }); /** diff --git a/server/threads/manageThreads.js b/server/threads/manageThreads.ts similarity index 83% rename from server/threads/manageThreads.js rename to server/threads/manageThreads.ts index 913dc5992..d932a6ad3 100644 --- a/server/threads/manageThreads.js +++ b/server/threads/manageThreads.ts @@ -1,43 +1,46 @@ -'use strict'; - // Must run before any component code is loaded so that process.exit() called // from component code (e.g. Next.js's `unhandledRejection` handler) is // intercepted in workers. -const { realExit } = require('./workerProcessGuard.ts'); - -const { Worker, MessageChannel, parentPort, isMainThread, threadId, workerData } = require('worker_threads'); -const { join, isAbsolute, extname } = require('path'); -const { server } = require('../Server.ts'); -const { totalmem } = require('os'); -const { setHeapSnapshotNearHeapLimit } = typeof globalThis.Bun !== 'undefined' ? {} : require('v8'); -const hdbTerms = require('../../utility/hdbTerms.ts'); -const envMgr = require('../../utility/environment/environmentManager.ts'); -const harperLogger = require('../../utility/logging/harper_logger.ts'); -const { randomBytes } = require('crypto'); -const { _assignPackageExport } = require('../../globals.js'); -const { PACKAGE_ROOT } = require('../../utility/packageUtils.js'); -const chokidar = require('chokidar'); -const isBun = typeof globalThis.Bun !== 'undefined'; -const MB = 1024 * 1024; -const workers = []; // these are our child workers that we are managing -const connectedPorts = []; // these are all known connected worker ports (siblings, children, parents) -const MAX_UNEXPECTED_RESTARTS = 50; -let threadTerminationTimeout = 10000; // threads, you got 10 seconds to die -const RESTART_TYPE = 'restart'; -const REQUEST_THREAD_INFO = 'request_thread_info'; -const RESOURCE_REPORT = 'resource_report'; -const THREAD_INFO = 'thread_info'; -const ADDED_PORT = 'added-port'; -const ACKNOWLEDGEMENT = 'ack'; -const REMOVE_PORT = 'remove-port'; -const FORCE_EXIT = 'force-exit'; -let getThreadInfo; +import { realExit } from './workerProcessGuard.ts'; +import { Worker, MessageChannel, parentPort, isMainThread, threadId, workerData } from 'worker_threads'; +import { onStartup } from '../../utility/lifecycle.ts'; +import { join, isAbsolute, extname } from 'path'; +import { server } from '../Server.ts'; +import { totalmem } from 'os'; +import { resetRestartNeeded } from '../../components/requestRestart.ts'; +import { loadRootComponents } from '../loadRootComponents.ts'; +import { setHeapSnapshotNearHeapLimit as _setHeapSnapshotNearHeapLimit } from 'node:v8'; +var setHeapSnapshotNearHeapLimit = typeof globalThis.Bun !== 'undefined' ? () => {} : _setHeapSnapshotNearHeapLimit; +import * as hdbTerms from '../../utility/hdbTerms.ts'; +import * as envMgr from '../../utility/environment/environmentManager.ts'; +import harperLogger from '../../utility/logging/harper_logger.ts'; +import { randomBytes } from 'crypto'; +import { _assignPackageExport } from '../../globals.js'; +import { RUNTIME_SRC_ROOT, RUNTIME_FILE_EXT } from '../../utility/packageUtils.js'; +import chokidar from 'chokidar'; +var isBun = typeof globalThis.Bun !== 'undefined'; +var MB = 1024 * 1024; +var workers: any[] = []; // these are our child workers that we are managing +var connectedPorts: any[] = []; // these are all known connected worker ports (siblings, children, parents) +var MAX_UNEXPECTED_RESTARTS = 50; +var threadTerminationTimeout = 10000; // threads, you got 10 seconds to die +var RESTART_TYPE = 'restart'; +var REQUEST_THREAD_INFO = 'request_thread_info'; +var RESOURCE_REPORT = 'resource_report'; +var THREAD_INFO = 'thread_info'; +var ADDED_PORT = 'added-port'; +var ACKNOWLEDGEMENT = 'ack'; +var REMOVE_PORT = 'remove-port'; +var FORCE_EXIT = 'force-exit'; +var getThreadInfo; _assignPackageExport('threads', connectedPorts); -const listenersByType = new Map(); -const messagesQueuedByType = new Map(); +var listenersByType = new Map(); +var messagesQueuedByType = new Map(); +var messageListeners = []; -module.exports = { +export let restartNumber = workerData?.restartNumber || 1; +export { startWorker, restartWorkers, shutdownWorkers, @@ -53,11 +56,10 @@ module.exports = { getTicketKeys, setMainIsWorker, setTerminateTimeout, - restartNumber: workerData?.restartNumber || 1, }; -connectedPorts.onMessageByType = onMessageByType; -connectedPorts.sendToThread = function (threadId, message) { +(connectedPorts as any).onMessageByType = onMessageByType; +(connectedPorts as any).sendToThread = function (threadId, message) { if (!message?.type) throw new Error('A message with a type must be provided'); const port = connectedPorts.find((port) => port.threadId === threadId); if (!port) return false; @@ -72,15 +74,16 @@ connectedPorts.sendToThread = function (threadId, message) { throw err; } }; -module.exports.whenThreadsStarted = new Promise((resolve) => { - module.exports.threadsHaveStarted = resolve; +export let threadsHaveStarted: (value?: unknown) => void; +export const whenThreadsStarted = new Promise((resolve) => { + threadsHaveStarted = resolve; }); // make sure this is set on all threads, including the main thread (this is no-op // if it was already with the execArgv below) if (envMgr.get(hdbTerms.CONFIG_PARAMS.THREADS_HEAPSNAPSHOTNEARLIMIT)) setHeapSnapshotNearHeapLimit(1); -let isMainWorker; +var isMainWorker; function setTerminateTimeout(newTimeout) { threadTerminationTimeout = newTimeout; } @@ -92,33 +95,35 @@ function getWorkerCount() { } function setMainIsWorker(isWorker) { isMainWorker = isWorker; - module.exports.threadsHaveStarted(); + threadsHaveStarted(); } -let workerCount = 1; // should be assigned when workers are created -let ticketKeys; +var workerCount = 1; // should be assigned when workers are created +var ticketKeys; function getTicketKeys() { if (ticketKeys) return ticketKeys; ticketKeys = isMainThread ? randomBytes(48) : workerData.ticketKeys; return ticketKeys; } -Object.defineProperty(server, 'workerIndex', { - get() { - return getWorkerIndex(); - }, -}); -Object.defineProperty(server, 'workerCount', { - get() { - return getWorkerCount(); - }, -}); -if (!parentPort) { - onMessageByType(REQUEST_THREAD_INFO, (message, worker) => { - if (worker) sendThreadInfo(worker); +onStartup(() => { + Object.defineProperty(server, 'workerIndex', { + get() { + return getWorkerIndex(); + }, }); - onMessageByType(RESOURCE_REPORT, (message, worker) => { - if (worker) recordResourceReport(worker, message); + Object.defineProperty(server, 'workerCount', { + get() { + return getWorkerCount(); + }, }); -} + if (!parentPort) { + onMessageByType(REQUEST_THREAD_INFO, (message, worker) => { + if (worker) sendThreadInfo(worker); + }); + onMessageByType(RESOURCE_REPORT, (message, worker) => { + if (worker) recordResourceReport(worker, message); + }); + } +}); // postMessage type listeners that are registered in other ways or can be registered later listenersByType.set(hdbTerms.ITC_EVENT_TYPES.CHILD_STARTED, null); listenersByType.set(hdbTerms.ITC_EVENT_TYPES.SCHEMA, null); @@ -127,7 +132,7 @@ listenersByType.set(hdbTerms.ITC_EVENT_TYPES.COMPONENT_STATUS_REQUEST, null); listenersByType.set(hdbTerms.ITC_EVENT_TYPES.RESOURCE_OPENAPI_REQUEST, null); listenersByType.set(hdbTerms.ITC_EVENT_TYPES.RESOURCE_OPENAPI_RESPONSE, null); -function startWorker(path, options = {}) { +function startWorker(path, options: any = {}) { // Take a percentage of total memory to determine the max memory for each thread. The percentage is based // on the thread count. Generally, it is unrealistic to efficiently use the majority of total memory for a single // NodeJS worker since it would lead to massive swap space usage with other processes and there is significant @@ -154,13 +159,16 @@ function startWorker(path, options = {}) { const channelsToConnect = []; const portsToSend = []; for (let existingPort of connectedPorts) { - const channel = new MessageChannel(); + const channel: any = new MessageChannel(); channel.existingPort = existingPort; channelsToConnect.push(channel); portsToSend.push(channel.port2); } - if (!extname(path)) path += '.js'; + // If the caller passed an extensionless module identifier, pick the + // extension that matches the current execution mode (.ts in typestrip + // source mode, .js in dist). + if (!extname(path)) path += RUNTIME_FILE_EXT; const isBun = typeof globalThis.Bun !== 'undefined'; const execArgv = isBun @@ -174,7 +182,7 @@ function startWorker(path, options = {}) { if (!isBun && envMgr.get(hdbTerms.CONFIG_PARAMS.THREADS_HEAPSNAPSHOTNEARLIMIT)) execArgv.push('--heapsnapshot-near-heap-limit=1'); - const worker = new Worker(isAbsolute(path) ? path : join(PACKAGE_ROOT, path), { + const worker: any = new Worker(isAbsolute(path) ? path : join(RUNTIME_SRC_ROOT, path), { resourceLimits: { maxOldGenerationSizeMb: maxOldMemory, maxYoungGenerationSizeMb: maxYoungMemory, @@ -188,7 +196,7 @@ function startWorker(path, options = {}) { workerIndex: options.workerIndex, workerCount: (workerCount = options.threadCount), name: options.name, - restartNumber: module.exports.restartNumber, + restartNumber: restartNumber, ticketKeys: getTicketKeys(), }, transferList: portsToSend, @@ -235,7 +243,7 @@ function startWorker(path, options = {}) { return worker; } -const OVERLAPPING_RESTART_TYPES = [hdbTerms.THREAD_TYPES.HTTP]; +var OVERLAPPING_RESTART_TYPES = [hdbTerms.THREAD_TYPES.HTTP]; /** * Restart all the worker threads @@ -263,15 +271,13 @@ async function restartWorkers( harperLogger.error('Unable to reestablish current working directory', e); } // problematic cyclic dependency, bind late - const { resetRestartNeeded } = require('../../components/requestRestart.ts'); resetRestartNeeded(); // This is here to prevent circular dependencies if (startReplacementThreads) { - const { loadRootComponents } = require('../loadRootComponents.js'); await loadRootComponents(); } - module.exports.restartNumber++; + restartNumber++; if (maxWorkersDown < 1) { // we accept a ratio of workers, and compute absolute maximum being down at a time from the total number of // threads @@ -285,13 +291,13 @@ async function restartWorkers( if ((name && worker.name !== name) || worker.wasShutdown) continue; // filter by type, if specified harperLogger.trace('sending shutdown request to ', worker.threadId); worker.postMessage({ - restartNumber: module.exports.restartNumber, + restartNumber: restartNumber, type: hdbTerms.ITC_EVENT_TYPES.SHUTDOWN, }); worker.wasShutdown = true; worker.emit('shutdown', {}); const overlapping = OVERLAPPING_RESTART_TYPES.indexOf(worker.name) > -1; - let whenDone = new Promise((resolve) => { + let whenDone = new Promise((resolve) => { // in case the exit inside the thread doesn't timeout, force it from the outside let timeout = setTimeout(() => { harperLogger.warn('Thread did not voluntarily terminate, terminating from the outside', worker.threadId); @@ -314,7 +320,7 @@ async function restartWorkers( waitingToFinish.push(whenDone); if (overlapping && startReplacementThreads) { let newWorker = worker.startCopy(); - let whenStarted = new Promise((resolve) => { + let whenStarted = new Promise((resolve) => { const startListener = (message) => { if (message.type === hdbTerms.ITC_EVENT_TYPES.CHILD_STARTED) { harperLogger.trace('Worker has started', newWorker.threadId); @@ -351,7 +357,7 @@ async function restartWorkers( function shutdownWorkers(name) { return restartWorkers(name, Infinity, false); } -async function shutdownWorkersNow(name) { +async function shutdownWorkersNow(name?) { shutdownWorkers(name); // set the state of all the workers to shut down. this should finish the important stuff synchronously if (isBun) { // worker.terminate() triggers a NAPI segfault in Bun; ask workers to self-exit instead @@ -365,11 +371,13 @@ async function shutdownWorkersNow(name) { } } -const messageListeners = []; function onMessageFromWorkers(listener) { + if (!messageListeners) messageListeners = []; messageListeners.push(listener); } function onMessageByType(type, listener) { + if (!listenersByType) listenersByType = new Map(); + if (!messagesQueuedByType) messagesQueuedByType = new Map(); let listeners = listenersByType.get(type); if (!listeners) listenersByType.set(type, (listeners = [])); listeners.push(listener); @@ -383,7 +391,7 @@ function onMessageByType(type, listener) { } } -const MAX_SYNC_BROADCAST = 10; +var MAX_SYNC_BROADCAST = 10; async function broadcast(message, includeSelf) { let count = 0; for (let port of connectedPorts) { @@ -403,10 +411,10 @@ async function broadcast(message, includeSelf) { } } -const awaitingResponses = new Map(); -let nextId = 1; +var awaitingResponses = new Map(); +var nextId = 1; function broadcastWithAcknowledgement(message) { - return new Promise((resolve) => { + return new Promise((resolve) => { let waitingCount = 0; for (let port of connectedPorts) { try { @@ -477,13 +485,13 @@ function recordResourceReport(worker, message) { worker.resources.updated = Date.now(); } -let monitorListener; +var monitorListener; function setMonitorListener(listener) { monitorListener = listener; } -const MONITORING_INTERVAL = 1000; -let monitoring = false; +var MONITORING_INTERVAL = 1000; +var monitoring = false; function startMonitoring() { if (monitoring) return; monitoring = true; @@ -510,15 +518,16 @@ function startMonitoring() { if (monitorListener) monitorListener(); }, MONITORING_INTERVAL).unref(); } -const REPORTING_INTERVAL = 1000; +var REPORTING_INTERVAL = 1000; if (parentPort && workerData?.addPorts) { // Main thread always has threadId 0 (worker_threads convention). Stamp it on // parentPort so sendToThread(0, ...) and similar lookups can route back to main. + // @ts-expect-error - stamping threadId on MessagePort for routing purposes parentPort.threadId = 0; addPort(parentPort); for (let i = 0, l = workerData.addPorts.length; i < l; i++) { - let port = workerData.addPorts[i]; + let port: any = workerData.addPorts[i]; port.threadId = workerData.addThreadIds[i]; addPort(port); } @@ -548,8 +557,7 @@ if (parentPort && workerData?.addPorts) { } else { getThreadInfo = getChildWorkerInfo; } -module.exports.getThreadInfo = getThreadInfo; - +export { getThreadInfo }; function removePort(port, deadThreadId) { const idx = connectedPorts.indexOf(port); if (idx === -1) return; @@ -570,7 +578,7 @@ function removePort(port, deadThreadId) { } } -function addPort(port, keepRef) { +function addPort(port, keepRef?) { connectedPorts.push(port); // Capture threadId now — Bun resets port.threadId to -1 by the time 'exit' fires. const portThreadId = port.threadId; @@ -629,7 +637,7 @@ if (isMainThread) { let beforeRestart, queuedRestart; let changedFiles = new Set(); const ignoredPaths = ['node_modules', '.git']; - const watchDir = async (dir, beforeRestartCallback) => { + const watchDir = async (dir, beforeRestartCallback?) => { if (beforeRestartCallback) beforeRestart = beforeRestartCallback; chokidar .watch(dir, { @@ -649,11 +657,10 @@ if (isMainThread) { }, 100); }); }; - module.exports.watchDir = watchDir; if (process.env.WATCH_DIR) watchDir(process.env.WATCH_DIR); } else { onMessageByType(hdbTerms.ITC_EVENT_TYPES.SHUTDOWN, async (message) => { - module.exports.restartNumber = message.restartNumber; + (globalThis as any).restartNumber = message.restartNumber; parentPort.unref(); // remove this handle setTimeout(() => { harperLogger.warn('Thread did not voluntarily terminate', threadId); diff --git a/server/threads/socketRouter.ts b/server/threads/socketRouter.ts index e5b2ccc6e..c11e2a7d8 100644 --- a/server/threads/socketRouter.ts +++ b/server/threads/socketRouter.ts @@ -1,10 +1,8 @@ -import { startWorker, setMonitorListener, setMainIsWorker, threadsHaveStarted } from './manageThreads.js'; +import { startWorker, setMonitorListener, setMainIsWorker, threadsHaveStarted } from './manageThreads.ts'; import * as hdbTerms from '../../utility/hdbTerms.ts'; import * as harperLogger from '../../utility/logging/harper_logger.ts'; import { recordHostname } from '../../resources/analytics/write.ts'; import { isMainThread } from 'worker_threads'; -import { join } from 'path'; - const workers = []; const workersReady = []; @@ -27,14 +25,14 @@ export async function startHTTPThreads(threadCount = 2, dynamicThreads?: boolean if (dynamicThreads) { startHTTPWorker(0, 1); } else { - const { loadRootComponents } = require('../loadRootComponents.js'); + const { loadRootComponents } = await import('../loadRootComponents.ts'); if (threadCount === 0) { setMainIsWorker(true); - await require('./threadServer.js').startServers(); + await (await import('./threadServer.ts')).startServers(); return Promise.resolve([]); } await loadRootComponents(); - const { listenOnPorts } = require('./threadServer.js'); + const { listenOnPorts } = await import('./threadServer.ts'); await listenOnPorts(); // Windows does not support SO_REUSEPORT, so only a single HTTP worker is supported. if (process.platform === 'win32') threadCount = 1; @@ -49,7 +47,10 @@ export async function startHTTPThreads(threadCount = 2, dynamicThreads?: boolean } function startHTTPWorker(index, threadCount = 1) { - startWorker(join(__dirname, './threadServer.js'), { + // Worker entry path is resolved by startWorker() against PACKAGE_ROOT, + // which differs between source (`core/server/threads/...`) and dist + // (`core/dist/server/threads/...`). startWorker handles the rewrite. + startWorker('server/threads/threadServer', { name: hdbTerms.THREAD_TYPES.HTTP, workerIndex: index, threadCount, diff --git a/server/threads/threadServer.js b/server/threads/threadServer.ts similarity index 77% rename from server/threads/threadServer.js rename to server/threads/threadServer.ts index b74ecc417..d5fa76d00 100644 --- a/server/threads/threadServer.js +++ b/server/threads/threadServer.ts @@ -1,31 +1,31 @@ -'use strict'; - -const { isMainThread, parentPort, threadId, workerData } = require('node:worker_threads'); -const { createServer: createSocketServer } = require('node:net'); -const { unlinkSync, existsSync, mkdirSync } = require('fs'); -const { join } = require('path'); +import * as inspector from 'node:inspector'; +import { isMainThread, parentPort, threadId, workerData } from 'node:worker_threads'; +import { createServer as createSocketServer } from 'node:net'; +import { unlinkSync, existsSync, mkdirSync } from 'fs'; +import { join } from 'path'; +import * as loaded from '../loadRootComponents.ts'; let componentsLoadedResolve; -exports.whenComponentsLoaded = new Promise((resolve) => { +export const whenComponentsLoaded = new Promise((resolve) => { componentsLoadedResolve = resolve; }); -const harperLogger = require('../../utility/logging/harper_logger.ts'); -const env = require('../../utility/environment/environmentManager.ts'); -const terms = require('../../utility/hdbTerms.ts'); -const { server } = require('../Server.ts'); -let { createServer: createSecureSocketServer } = require('node:tls'); -const { restartNumber, getWorkerIndex } = require('./manageThreads.js'); -const { realExit } = require('./workerProcessGuard.ts'); -const { isBun } = require('../serverHelpers/Request.ts'); -const { createTLSSelector } = require('../../security/keys.ts'); -const { startupLog } = require('../../bin/run.ts'); -const { SERVERS, setPortServerMap, portServer } = require('../serverRegistry.ts'); -const httpComponent = require('../http.ts'); -const globals = require('../../globals.js'); - +import harperLogger from '../../utility/logging/harper_logger.ts'; +import * as env from '../../utility/environment/environmentManager.ts'; +import * as terms from '../../utility/hdbTerms.ts'; +import { server } from '../Server.ts'; +import { createServer as createSecureSocketServer } from 'node:tls'; +import { restartNumber, getWorkerIndex } from './manageThreads.ts'; +import { realExit } from './workerProcessGuard.ts'; +import { isBun } from '../serverHelpers/Request.ts'; +import { createTLSSelector } from '../../security/keys.ts'; +import { startupLog } from '../../bin/run.ts'; +import { SERVERS, setPortServerMap, portServer } from '../serverRegistry.ts'; +import * as httpComponent from '../http.ts'; +import * as globals from '../../globals.js'; +import { onStartup } from '../../utility/lifecycle.ts'; +import { getComponentName } from '../../components/componentLoader.ts'; const debugThreads = env.get(terms.CONFIG_PARAMS.THREADS_DEBUG); const isWindows = process.platform === 'win32'; -server.socket = onSocket; if (!isBun) { if (debugThreads) { @@ -34,7 +34,7 @@ if (!isBun) { port = env.get(terms.CONFIG_PARAMS.THREADS_DEBUG_PORT) ?? 9229; const closeInspector = () => { try { - require('inspector').close(); + inspector.close(); } catch (error) { harperLogger.info('Could not close debugger', error); } @@ -52,14 +52,14 @@ if (!isBun) { const host = env.get(terms.CONFIG_PARAMS.THREADS_DEBUG_HOST); const waitForDebugger = env.get(terms.CONFIG_PARAMS.THREADS_DEBUG_WAITFORDEBUGGER); try { - require('inspector').open(port, host, waitForDebugger); + inspector.open(port, host, waitForDebugger); } catch (error) { harperLogger.trace(`Could not start debugging on port ${port}, you may already be debugging:`, error.message); } } } else if (process.env.DEV_MODE && isMainThread) { try { - require('inspector').open(9229); + inspector.open(9229); } catch (error) { if (restartNumber <= 1) harperLogger.trace('Could not start debugging on port 9229, you may already be debugging:', error.message); @@ -67,7 +67,7 @@ if (!isBun) { } } -process.on('uncaughtException', (error) => { +process.on('uncaughtException', (error: any) => { if (error.isHandled) return; if (error.code === 'ECONNRESET' || error.code === 'ECONNREFUSED') return; // that's what network connections do if (error.message === 'write EIO') return; // that means the terminal is closed @@ -77,16 +77,14 @@ process.on('uncaughtException', (error) => { // handler is registered. Without this, any async path that rejects without being caught // (e.g. a cache-update commit error when the caller has already resolved) will kill the // worker thread. Mirror the uncaughtException behavior: log and continue. -process.on('unhandledRejection', (reason) => { +process.on('unhandledRejection', (reason: any) => { if (reason?.isHandled) return; harperLogger.error('unhandledRejection', reason); }); -env.initSync(); -exports.globals = globals; -exports.listenOnPorts = listenOnPorts; -exports.startServers = startServers; -exports.closeServers = closeServers; - +export { globals }; +export { listenOnPorts }; +export { startServers }; +export { closeServers }; function closeServers() { if (isBun) { // Bun servers use .stop() for graceful shutdown @@ -138,7 +136,7 @@ function closeServers() { } // And we tell the server not to accept any more incoming connections promises.push( - new Promise((resolve) => { + new Promise((resolve) => { server.close?.(() => { resolve(); }); @@ -163,45 +161,45 @@ function startServers() { // ignore any errors with this; just a best effort for now } } - let loaded = require('../loadRootComponents.js') - .loadRootComponents(true) - .then(() => { - parentPort - ?.on('message', (message) => { - if (message.type === terms.ITC_EVENT_TYPES.SHUTDOWN) { - harperLogger.trace('received shutdown request', threadId); - // shutdown (for these threads) means stop listening for incoming requests (finish what we are working) and - // close connections as possible, then let the event loop complete - closeServers().then(() => { - realExit(0); - }); - // Clean up per-thread UDS socket and metadata files - httpComponent.cleanupUdsFiles(); - if (!isBun && (debugThreads || process.env.DEV_MODE)) { - try { - require('inspector').close(); - } catch (error) { - harperLogger.info('Could not close debugger', error); - } + const loadedPromise = loaded.loadRootComponents(true).then(() => { + parentPort + ?.on('message', (message) => { + if (message.type === terms.ITC_EVENT_TYPES.SHUTDOWN) { + harperLogger.trace('received shutdown request', threadId); + // shutdown (for these threads) means stop listening for incoming requests (finish what we are working) and + // close connections as possible, then let the event loop complete + closeServers().then(() => { + realExit(0); + }); + // Clean up per-thread UDS socket and metadata files + httpComponent.cleanupUdsFiles(); + if (!isBun && (debugThreads || process.env.DEV_MODE)) { + try { + inspector.close(); + } catch (error) { + harperLogger.info('Could not close debugger', error); } } - }) - .ref(); // use this to keep the thread running until we are ready to shutdown and clean up handles - const listening = listenOnPorts(); + } + }) + .ref(); // use this to keep the thread running until we are ready to shutdown and clean up handles + const listening = listenOnPorts(); - // notify that we are now ready to start receiving requests - Promise.resolve(listening).then(() => { - if (getWorkerIndex() === 0) { - try { - startupLog(portServer); - } catch (err) { - console.error('Error displaying start-up log', err); - } + // notify that we are now ready to start receiving requests. Chained with + // `return` so the outer loadedPromise / whenComponentsLoaded only fulfils + // once HTTP/HTTPS sockets are actually bound — apitests await that. + return Promise.resolve(listening).then(() => { + if (getWorkerIndex() === 0) { + try { + startupLog(portServer); + } catch (err) { + console.error('Error displaying start-up log', err); } - parentPort?.postMessage({ type: terms.ITC_EVENT_TYPES.CHILD_STARTED }); - }); + } + parentPort?.postMessage({ type: terms.ITC_EVENT_TYPES.CHILD_STARTED }); }); - componentsLoadedResolve(loaded); + }); + componentsLoadedResolve(loadedPromise); // Clean up UDS files and force-close Bun server connections on unexpected exit. // Without the stop(true) call, clients holding keep-alive connections to a dead Bun // worker never receive a FIN/RST and hang indefinitely waiting for a response. @@ -218,7 +216,6 @@ function startServers() { } httpComponent.cleanupUdsFiles(); }); - return loaded; } let listening; function listenOnPorts() { @@ -314,7 +311,7 @@ async function listenOnPortsBun() { } else { portNumber = +port; } - const serveOptions = { + const serveOptions: any = { port: portNumber, reusePort: !isWindows && !isMac, fetch: config.fetch, @@ -325,7 +322,7 @@ async function listenOnPortsBun() { // Wait for TLS certs to be loaded const defaultContext = await config.tlsSelector.ready; if (defaultContext) { - serveOptions.tls = { + const tlsOpts: any = { cert: defaultContext.options.cert, key: defaultContext.options.key, }; @@ -334,9 +331,10 @@ async function listenOnPortsBun() { if (ca) { if (Array.isArray(ca)) ca = ca.filter((entry) => typeof entry === 'string'); if (typeof ca === 'string' || (Array.isArray(ca) && ca.length > 0)) { - serveOptions.tls.ca = ca; + tlsOpts.ca = ca; } } + serveOptions.tls = tlsOpts; } // Set up listener for cert updates to reload TLS const pseudoServer = config.pseudoServer; @@ -344,7 +342,7 @@ async function listenOnPortsBun() { pseudoServer.secureContextsListeners.push(() => { const updatedCtx = config.tlsSelector.defaultContext; if (updatedCtx && SERVERS[port]?.reload) { - const tlsUpdate = { + const tlsUpdate: any = { cert: updatedCtx.options.cert, key: updatedCtx.options.key, }; @@ -371,6 +369,7 @@ async function listenOnPortsBun() { delete serveOptions.port; } if (isNaN(serveOptions.port)) continue; + // @ts-expect-error - Bun is a runtime global only available in Bun environment const bunServer = Bun.serve(serveOptions); SERVERS[port] = bunServer; harperLogger.trace('Bun listening on port ' + port, threadId); @@ -385,6 +384,7 @@ async function listenOnPortsBun() { if (existsSync(udsPath)) unlinkSync(udsPath); // Create a plain HTTP Bun server on the UDS (no TLS) + // @ts-expect-error - Bun is a runtime global only available in Bun environment const udsServer = Bun.serve({ unix: udsPath, fetch: config.fetch, @@ -445,7 +445,21 @@ async function listenOnPortsBun() { return Promise.all(listening); } if (!isMainThread && !workerData?.noServerStart) { - startServers(); + // Workers start with an empty environment manager. Run the same init+startup + // sequence as the main entry (bin/harper.ts) before bringing up servers. + // startServers schedules its loadRootComponents().then(...) chain internally + // and notifies the parent via parentPort.postMessage(CHILD_STARTED) once the + // HTTP port is bound — don't await it here, otherwise the worker IIFE blocks + // on the same chain that main is waiting on, deadlocking the startup. + (async () => { + env.initSync(); + const { runStartup } = await import('../../utility/lifecycle.ts'); + await runStartup(); + startServers(); + })().catch((err) => { + harperLogger.fatal('Worker failed to start', err); + realExit(1); + }); } /** @@ -454,7 +468,6 @@ if (!isMainThread && !workerData?.noServerStart) { * @param options */ function onSocket(listener, options) { - let getComponentName = require('../../components/componentLoader.ts').getComponentName; let socketServer; if (options.securePort) { setPortServerMap(options.securePort, { protocol_name: 'TLS', name: getComponentName() }); @@ -486,11 +499,10 @@ function onSocket(listener, options) { const udsPath = join(socketsDir, `${socketName}.sock`); const yamlPath = join(socketsDir, `${socketName}.yaml`); - const udsServer = createSocketServer(listener, { - noDelay: true, - keepAlive: true, - keepAliveInitialDelay: 600, - }); + const udsServer: any = createSocketServer( + { allowHalfOpen: false, noDelay: true, keepAlive: true, keepAliveInitialDelay: 600 } as any, + listener + ); udsServer.isPerThreadSocket = true; SERVERS[udsPath] = udsServer; @@ -503,13 +515,17 @@ function onSocket(listener, options) { } if (options.port) { setPortServerMap(options.port, { protocol_name: 'TCP', name: getComponentName() }); - socketServer = createSocketServer(listener, { - noDelay: true, - keepAlive: true, - keepAliveInitialDelay: 600, - }); + socketServer = createSocketServer( + { allowHalfOpen: false, noDelay: true, keepAlive: true, keepAliveInitialDelay: 600 } as any, + listener + ); socketServer.noReusePort = true; SERVERS[options.port] = socketServer; } return socketServer; } + +// Wire server singletons during the startup phase +onStartup(() => { + server.socket = onSocket; +}); diff --git a/sqlTranslator/SelectValidator.ts b/sqlTranslator/SelectValidator.ts index ba9cb439d..16abe28bc 100644 --- a/sqlTranslator/SelectValidator.ts +++ b/sqlTranslator/SelectValidator.ts @@ -100,7 +100,7 @@ class SelectValidator { } //let theTable = clone(table); - let schemaTable = databases[table.databaseid][table.tableid]; + let schemaTable: any = databases[table.databaseid][table.tableid]; /*TODO rather than putting every attribute in an array we will create a Map there will be a map element for every table and every table alias (this will create duplicate map elements) this will have downstream effects in comparison functions like findColumn*/ schemaTable.attributes.forEach((attribute) => { diff --git a/sqlTranslator/alasqlFunctionImporter.ts b/sqlTranslator/alasqlFunctionImporter.ts index 14c9d7d4a..9660d22a1 100644 --- a/sqlTranslator/alasqlFunctionImporter.ts +++ b/sqlTranslator/alasqlFunctionImporter.ts @@ -4,9 +4,9 @@ * PUrpose of this is to set up a central module to define and import custom functions into alasql */ -import * as alasqlExtension from '../utility/functions/sql/alaSQLExtension.js'; +import * as alasqlExtension from '../utility/functions/sql/alaSQLExtension.ts'; import * as dateFunctions from '../utility/functions/date/dateFunctions.js'; -import * as geo from '../utility/functions/geo.js'; +import * as geo from '../utility/functions/geo.ts'; //import the custom function, need to define an upper and lower case version of the function so it is parsed properly in alasql export default function (alasql: any) { diff --git a/sqlTranslator/sql_statement_bucket.ts b/sqlTranslator/sql_statement_bucket.ts index 0bd69ef46..3b5b95058 100644 --- a/sqlTranslator/sql_statement_bucket.ts +++ b/sqlTranslator/sql_statement_bucket.ts @@ -6,7 +6,8 @@ import * as alasql from 'alasql'; import RecursiveIterator from 'recursive-iterator'; -const harperLogger = require('../utility/logging/harper_logger').default || require('../utility/logging/harper_logger'); +import _harperLogger from '../utility/logging/harper_logger.ts'; +const harperLogger = _harperLogger; import * as hdbUtils from '../utility/common_utils.ts'; import * as terms from '../utility/hdbTerms.ts'; diff --git a/unitTests/apiTests/setupTestApp.mjs b/unitTests/apiTests/setupTestApp.mjs index 6f38f542e..17398c309 100644 --- a/unitTests/apiTests/setupTestApp.mjs +++ b/unitTests/apiTests/setupTestApp.mjs @@ -35,6 +35,16 @@ function makeString() { let createdRecords; let serverStarted; export async function setupTestApp() { + // Drain onStartup hooks first so server-singleton wiring (server.http, server.getUser, + // server.operation, etc.) is installed before we override server.getUser below. + // Production runs this from bin/harper.ts; the API-test setup needs the same effect. + // Without this, the override below is later clobbered when user.ts's onStartup hook + // fires inside startHTTPThreads, and auth checks fall through to the real getUser. + if (typeof process !== 'undefined' && !serverStarted) { + const { runStartup } = await import('#src/utility/lifecycle'); + await runStartup(); + } + analytics.setAnalyticsEnabled(false); bypassAuth(); bypassAuthMQTT(); @@ -89,6 +99,11 @@ export async function setupTestApp() { } else { const { startHTTPThreads } = await import('#src/server/threads/socketRouter'); serverStarted = await startHTTPThreads(config.threads || 0); + // startServers() schedules its loadRootComponents().then(listenOnPorts) chain + // internally without returning the resolved promise. Block on the shared + // `whenComponentsLoaded` so the test's first axios call sees a bound socket. + const { whenComponentsLoaded } = await import('#src/server/threads/threadServer'); + await whenComponentsLoaded; } try { seed = 0; // reset the seed to make sure we are deterministic here diff --git a/unitTests/config/configUtils-runtimeEnvVars.test.js b/unitTests/config/configUtils-runtimeEnvVars.test.js index 5f30b6f4c..45a4c839c 100644 --- a/unitTests/config/configUtils-runtimeEnvVars.test.js +++ b/unitTests/config/configUtils-runtimeEnvVars.test.js @@ -37,13 +37,8 @@ describe('configUtils - applyRuntimeEnvVarConfig', function () { configUtils.__set__('fs', { writeFileSync: fsWriteFileSyncStub, renameSync: fsRenameSyncStub }); configUtils.__set__('YAML', YAMLStub); - // Mock harperConfigEnvVars module - configUtils.__set__('require', function (modulePath) { - if (modulePath.startsWith('./harperConfigEnvVars')) { - return { applyRuntimeEnvConfig: applyRuntimeEnvConfigStub }; - } - return require(modulePath); - }); + // Stub the top-level-imported applyRuntimeEnvConfig + configUtils.__set__('applyRuntimeEnvConfig', applyRuntimeEnvConfigStub); }); beforeEach(function () { diff --git a/unitTests/resources/create.test.js b/unitTests/resources/create.test.js index 2426ff0d6..3f030aafe 100644 --- a/unitTests/resources/create.test.js +++ b/unitTests/resources/create.test.js @@ -65,7 +65,17 @@ describe('Create records', () => { let id_after = CreateTest.getNewId(); assert(Math.abs(id_before - id_after) > 1000000); }); - after(() => { - test_thread.terminate(); + after(async () => { + // Graceful shutdown — let the worker close its rocksdb handles on its own + // event loop rather than relying on Worker.terminate(), which under + // rocksdb-js triggers a native finalizer crash during the next test's + // process-wide handle setup. + await new Promise((resolve) => { + test_thread.once('exit', resolve); + test_thread.postMessage({ type: 'shutdown' }); + setTimeout(() => { + test_thread.terminate().then(resolve); + }, 1000).unref(); + }); }); }); diff --git a/unitTests/testUtils.js b/unitTests/testUtils.js index 64d956515..30fe155f2 100644 --- a/unitTests/testUtils.js +++ b/unitTests/testUtils.js @@ -12,6 +12,7 @@ const harperBridge = require('#src/dataLayer/harperBridge/harperBridge').default const { isMainThread } = require('node:worker_threads'); const { getDatabases, databases } = require('#src/resources/databases'); const { handleHDBError } = require('#src/utility/errors/hdbError'); +const lifecycle = require('#src/utility/lifecycle'); let envMgrInitSyncStub; @@ -83,6 +84,12 @@ function preTestPrep(testConfigObj) { // Try to change to bin changeProcessToBinDir(); env.initTestEnvironment(testConfigObj); + + // Drain startup hooks so modules that defer side effects via `onStartup(...)` + // (server-singleton wiring, listener registration, config-derived constants) + // see a wired-up state. Production calls this from `bin/harper.ts`; tests + // that go through this helper get the same effect. Idempotent. + void lifecycle.runStartup(); } /** diff --git a/unitTests/utility/logging/readLog.test.js b/unitTests/utility/logging/readLog.test.js index 589ecd5f6..50d16db45 100644 --- a/unitTests/utility/logging/readLog.test.js +++ b/unitTests/utility/logging/readLog.test.js @@ -74,7 +74,7 @@ describe('Test readLog module', () => { }); beforeEach(() => { - getConfigPath_rw = read_log.__set__('configUtils_js_1', { + getConfigPath_rw = read_log.__set__('configUtils_ts_1', { getConfigPath: (key) => { if (key === hdb_terms.HDB_SETTINGS_NAMES.LOG_PATH_KEY) { return TEST_LOG_DIR; diff --git a/unitTests/utility/mount_hdb.test.js b/unitTests/utility/mount_hdb.test.js index 9dd0a4a35..0a03e5577 100644 --- a/unitTests/utility/mount_hdb.test.js +++ b/unitTests/utility/mount_hdb.test.js @@ -55,7 +55,7 @@ describe('test mount_hdb module', () => { create_table_stub = sandbox.stub(bridge, 'createTable'); mount_hdb.__set__('mkdirpSync', mk_dirp_sync_stub); mount_hdb.__set__('copySync', sandbox.stub()); - mount_hdb.__set__('systemSchema_json_1', { default: test_system_schema }); + mount_hdb.__set__('systemSchema', test_system_schema); }); after(() => { diff --git a/upgrade/directives/5-2-0.ts b/upgrade/directives/5-2-0.ts index 03c9cf45b..5e4ceb4f1 100644 --- a/upgrade/directives/5-2-0.ts +++ b/upgrade/directives/5-2-0.ts @@ -6,8 +6,13 @@ // json/systemSchema.json on first boot). This directive handles the upgrade path: existing // installs that already have a system schema need the new table added explicitly. +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { PACKAGE_ROOT } from '../../utility/packageUtils.js'; +const systemSchema: Record = JSON.parse( + readFileSync(join(PACKAGE_ROOT, 'json/systemSchema.json'), 'utf-8') +); import { databases } from '../../resources/databases.ts'; -import systemSchema from '../../json/systemSchema.json'; import * as terms from '../../utility/hdbTerms.ts'; import * as initPaths from '../../dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js'; import bridge from '../../dataLayer/harperBridge/harperBridge.ts'; diff --git a/upgrade/upgradeUtilities.ts b/upgrade/upgradeUtilities.ts index 9d2151c91..abb686608 100644 --- a/upgrade/upgradeUtilities.ts +++ b/upgrade/upgradeUtilities.ts @@ -1,7 +1,7 @@ 'use strict'; import * as hdbUtil from '../utility/common_utils.ts'; -import * as configUtils from '../config/configUtils.js'; +import * as configUtils from '../config/configUtils.ts'; /** * We need to make sure we are setting empty string for values that are null/undefined/empty string - PropertiesReader diff --git a/utility/common_utils.ts b/utility/common_utils.ts index 26b5e8dd1..0f2402cf0 100644 --- a/utility/common_utils.ts +++ b/utility/common_utils.ts @@ -1,8 +1,8 @@ 'use strict'; import * as path from 'path'; -import * as fs from 'fs-extra'; +import fs from 'fs-extra'; import log from './logging/harper_logger.ts'; -import * as fsExtra from 'fs-extra'; +import fsExtra from 'fs-extra'; import * as os from 'os'; import * as net from 'net'; import RecursiveIterator from 'recursive-iterator'; diff --git a/utility/environment/environmentManager.ts b/utility/environment/environmentManager.ts index 98b04608a..26b725c2b 100644 --- a/utility/environment/environmentManager.ts +++ b/utility/environment/environmentManager.ts @@ -1,27 +1,27 @@ 'use strict'; -import * as fs from 'fs-extra'; +import fs from 'fs-extra'; import * as path from 'path'; import * as os from 'os'; import PropertiesReader from 'properties-reader'; import log from '../logging/harper_logger.ts'; import * as commonUtils from '../common_utils.ts'; import * as hdbTerms from '../hdbTerms.ts'; -import * as configUtils from '../../config/configUtils.js'; +import * as configUtils from '../../config/configUtils.ts'; import { mkdirSync } from 'node:fs'; -const INIT_ERR = 'Error initializing environment manager'; -const BOOT_PROPS_FILE_PATH = 'BOOT_PROPS_FILE_PATH'; +var INIT_ERR = 'Error initializing environment manager'; +var BOOT_PROPS_FILE_PATH = 'BOOT_PROPS_FILE_PATH'; -let propFileExists = false; +var propFileExists = false; -const installPropsToSave = { +var installPropsToSave = { [hdbTerms.HDB_SETTINGS_NAMES.INSTALL_USER]: true, [hdbTerms.HDB_SETTINGS_NAMES.SETTINGS_PATH_KEY]: true, [hdbTerms.HDB_SETTINGS_NAMES.HDB_ROOT_KEY]: true, BOOT_PROPS_FILE_PATH: true, }; -let installProps: any = {}; +var installProps: any = {}; export { BOOT_PROPS_FILE_PATH }; /** @@ -30,7 +30,7 @@ export { BOOT_PROPS_FILE_PATH }; * currently known base path here to help with this case. */ export function getHdbBasePath() { - return installProps[hdbTerms.HDB_SETTINGS_NAMES.HDB_ROOT_KEY]; + return installProps?.[hdbTerms.HDB_SETTINGS_NAMES.HDB_ROOT_KEY]; } /** @@ -48,11 +48,18 @@ export function setHdbBasePath(hdbPath: string) { * @returns {*} */ export function get(propName: string): any { - const value = configUtils.getConfigValue(propName); + // Tolerate calls before this module's body has executed (ESM cycle): + // configUtils.ts / installProps may not be initialized yet, in which case + // no config has been read so the value is genuinely unknown. + let value; + try { + value = configUtils.getConfigValue(propName); + } catch (err: any) { + if (err?.name !== 'ReferenceError') throw err; + } if (value === undefined) { - return installProps[propName]; + return installProps?.[propName]; } - return value; } @@ -114,7 +121,13 @@ export function initSync(force: boolean = false) { installProps[hdbTerms.HDB_SETTINGS_NAMES.HDB_ROOT_KEY] = configHdbRoot; } } - } catch (err) { + } catch (err: any) { + // During typestrip ESM evaluation, module-load callers of initSync may + // reach this before configUtils has finished its own top-level evaluation, + // producing a ReferenceError (TDZ) on a module-scope binding. Don't exit + // the process for that — the same module-load chain will retry once + // evaluation completes, or bin/harper.ts will re-call initSync(). + if (err?.name === 'ReferenceError') return; log.error(INIT_ERR); log.error(err); console.error(err); diff --git a/utility/environment/systemInformation.ts b/utility/environment/systemInformation.ts index 1070a0997..48799eebd 100644 --- a/utility/environment/systemInformation.ts +++ b/utility/environment/systemInformation.ts @@ -4,14 +4,16 @@ import si from 'systeminformation'; import logger from '../logging/harper_logger.ts'; import * as hdbTerms from '../hdbTerms.ts'; import { lmdbGetTableSize } from '../../dataLayer/harperBridge/lmdbBridge/lmdbUtility/lmdbGetTableSize.ts'; -import { getThreadInfo } from '../../server/threads/manageThreads.js'; +import { getThreadInfo } from '../../server/threads/manageThreads.ts'; import * as env from './environmentManager.ts'; import { getDatabases, type Table } from '../../resources/databases.ts'; import { TableSizeObject } from '../../dataLayer/harperBridge/TableSizeObject.ts'; -import { RocksDatabase, StatsHistogramData } from '@harperfast/rocksdb-js'; - -env.initSync(); - +import { RocksDatabase, type StatsHistogramData } from '@harperfast/rocksdb-js'; +try { + env.initSync(); +} catch { + /* tolerate ESM cycle TDZ; bin entry will re-call later */ +} //this will hold the system_information which is static to improve performance let systemInformationCache = undefined; diff --git a/utility/functions/geo.js b/utility/functions/geo.ts similarity index 85% rename from utility/functions/geo.js rename to utility/functions/geo.ts index 11df524c2..4e1c27c46 100644 --- a/utility/functions/geo.js +++ b/utility/functions/geo.ts @@ -1,5 +1,3 @@ -'use strict'; - /*** * geo.js * @@ -7,20 +5,20 @@ * turf.js has very robust internal validation as such we offload the validation to turf.js */ -const turfArea = require('@turf/area'); -const turfLength = require('@turf/length'); -const turfCircle = require('@turf/circle'); -const turfDifference = require('@turf/difference'); -const turfDistance = require('@turf/distance'); -const turfBooleanContains = require('@turf/boolean-contains'); -const turfBooleanEqual = require('@turf/boolean-equal'); -const turfBooleanDisjoint = require('@turf/boolean-disjoint'); -const turfHelpers = require('@turf/helpers'); -const hdbTerms = require('../hdbTerms.ts'); -const commonUtils = require('../common_utils.ts'); -const hdbLog = require('../logging/harper_logger.ts'); - -module.exports = { +import turfArea from '@turf/area'; +import turfLength from '@turf/length'; +import turfCircle from '@turf/circle'; +import turfDifference from '@turf/difference'; +import turfDistance from '@turf/distance'; +import turfBooleanContains from '@turf/boolean-contains'; +import turfBooleanEqual from '@turf/boolean-equal'; +import turfBooleanDisjoint from '@turf/boolean-disjoint'; +import * as turfHelpers from '@turf/helpers'; +import * as hdbTerms from '../hdbTerms.ts'; +import * as commonUtils from '../common_utils.ts'; +import hdbLog from '../logging/harper_logger.ts'; + +export { geoArea, geoLength, geoCircle, @@ -32,7 +30,6 @@ module.exports = { geoCrosses, geoConvert, }; - /*** * Takes one or more features and returns the area in square meters * @param geoJSON @@ -47,7 +44,7 @@ function geoArea(geoJSON) { geoJSON = commonUtils.autoCastJSON(geoJSON); } try { - return turfArea.default(geoJSON); + return turfArea(geoJSON); } catch (err) { hdbLog.trace(err, geoJSON); return NaN; @@ -70,7 +67,7 @@ function geoLength(geoJSON, units) { } try { - return turfLength.default(geoJSON, { units: units ? units : 'kilometers' }); + return turfLength(geoJSON, { units: units ? units : 'kilometers' }); } catch (err) { hdbLog.trace(err, geoJSON); return NaN; @@ -98,7 +95,7 @@ function geoCircle(point, radius, units) { } try { - return turfCircle.default(point, radius, { units: units ? units : 'kilometers' }); + return turfCircle(point, radius, { units: units ? units : 'kilometers' }); } catch (err) { hdbLog.trace(err, point, radius); return NaN; @@ -160,7 +157,7 @@ function geoDistance(point1, point2, units) { } try { - return turfDistance.default(point1, point2, { units: units ? units : 'kilometers' }); + return turfDistance(point1, point2, { units: units ? units : 'kilometers' }); } catch (err) { hdbLog.trace(err, point1, point2); return NaN; @@ -239,7 +236,7 @@ function geoContains(geo1, geo2) { } try { - return turfBooleanContains.default(geo1, geo2); + return turfBooleanContains(geo1, geo2); } catch (err) { hdbLog.trace(err, geo1, geo2); return false; @@ -277,7 +274,7 @@ function geoEqual(geo1, geo2) { } try { - return turfBooleanEqual.default(geo1, geo2); + return turfBooleanEqual(geo1, geo2); } catch (err) { hdbLog.trace(err, geo1, geo2); return false; @@ -316,7 +313,7 @@ function geoCrosses(geo1, geo2) { try { //need to do ! as this checks for non-intersections of geometries - return !turfBooleanDisjoint.default(geo1, geo2); + return !turfBooleanDisjoint(geo1, geo2); } catch (err) { hdbLog.trace(err, geo1, geo2); return false; diff --git a/utility/functions/sql/alaSQLExtension.js b/utility/functions/sql/alaSQLExtension.ts similarity index 60% rename from utility/functions/sql/alaSQLExtension.js rename to utility/functions/sql/alaSQLExtension.ts index db3bdaef5..701db0eab 100644 --- a/utility/functions/sql/alaSQLExtension.js +++ b/utility/functions/sql/alaSQLExtension.ts @@ -1,50 +1,26 @@ -'use strict'; - /*** * alaSQLExtension.js * purpose of this module is to hold custom functions for alasql */ -const _ = require('lodash'); -const mathjs = require('mathjs'); -const jsonata = require('jsonata'); -const hdbUtils = require('../../common_utils.ts'); - -module.exports = { - /*** - * distinctArray takes in an array an dedupes its values using lodash. this works on complex as well as simple datatypes - * @param array - * @returns array - */ - distinct_array: (array) => { - if (Array.isArray(array) && array.length > 1) { - return _.uniqWith(array, _.isEqual); - } +import _ from 'lodash'; +import * as mathjs from 'mathjs'; +import jsonata from 'jsonata'; +import * as hdbUtils from '../../common_utils.ts'; +export const distinct_array = (array) => { + if (Array.isArray(array) && array.length > 1) { + return _.uniqWith(array, _.isEqual); + } - return array; - }, - searchJSON, - /*** - * median absolute deviation aggregate function based on http://mathjs.org/docs/reference/functions/mad.html - */ - mad: aggregateFunction.bind(null, mathjs.mad), - /*** - * mean aggregate function based on http://mathjs.org/docs/reference/functions/mean.html - */ - mean: aggregateFunction.bind(null, mathjs.mean), - /*** - * computes the mode of values on http://mathjs.org/docs/reference/functions/mode.html - */ - mode: aggregateFunction.bind(null, mathjs.mode), - /*** - * compute the product based on http://mathjs.org/docs/reference/functions/prod.html - */ - prod: aggregateFunction.bind(null, mathjs.prod), - /*** - * compute the median based on http://mathjs.org/docs/reference/functions/median.html - */ - median: aggregateFunction.bind(null, mathjs.median), + return array; }; +export const mad = aggregateFunction.bind(null, mathjs.mad); +export const mean = aggregateFunction.bind(null, mathjs.mean); +export const mode = aggregateFunction.bind(null, mathjs.mode); +export const prod = aggregateFunction.bind(null, mathjs.prod); +export const median = aggregateFunction.bind(null, mathjs.median); +export { searchJSON }; +export default { distinct_array, searchJSON, mad, mean, mode, prod, median }; /*** * handles the 3 pass loop for aggregates and executes the final calc with the passed in aggregator function diff --git a/utility/globalSchema.ts b/utility/globalSchema.ts index 3a21f225b..44637e135 100644 --- a/utility/globalSchema.ts +++ b/utility/globalSchema.ts @@ -1,5 +1,11 @@ -import systemSchema from '../json/systemSchema.json'; +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { PACKAGE_ROOT } from './packageUtils.js'; import { promisify } from 'util'; +// Import JSON without static import attributes (unsupported in CJS output; `with {type:'json'}` unsupported in tsc NodeNext+CJS) +const systemSchema: Record = JSON.parse( + readFileSync(join(PACKAGE_ROOT, 'json/systemSchema.json'), 'utf-8') +); import { getDatabases } from '../resources/databases.ts'; export const setSchemaDataToGlobalAsync = promisify(setSchemaDataToGlobal); diff --git a/utility/hdbTerms.ts b/utility/hdbTerms.ts index d1599190c..4bebbd96c 100644 --- a/utility/hdbTerms.ts +++ b/utility/hdbTerms.ts @@ -19,7 +19,7 @@ export const HDB_COMPONENT_CONFIG_FILE = 'config.yaml'; /** Name of the Harper Process Script */ export const HDB_PROC_NAME = 'harper.js'; /** Name of the Harper Restart Script */ -export const HDB_RESTART_SCRIPT = 'restartHdb.js'; +export const HDB_RESTART_SCRIPT = 'restartHdb.ts'; /** Harper Process Descriptor */ const HDB_PROC_DESCRIPTOR = 'Harper'; diff --git a/utility/install/checkJWTTokensExist.js b/utility/install/checkJWTTokensExist.ts similarity index 69% rename from utility/install/checkJWTTokensExist.js rename to utility/install/checkJWTTokensExist.ts index ecc437876..95a58265d 100644 --- a/utility/install/checkJWTTokensExist.js +++ b/utility/install/checkJWTTokensExist.ts @@ -1,14 +1,20 @@ -'use strict'; +import * as env from '../../utility/environment/environmentManager.ts'; +import fs from 'fs-extra'; +import path from 'path'; +import * as terms from '../../utility/hdbTerms.ts'; +import crypto from 'crypto'; +import { v4 as uuid } from 'uuid'; -const env = require('../../utility/environment/environmentManager.ts'); -env.initSync(); -const fs = require('fs-extra'); -const path = require('path'); -const terms = require('../../utility/hdbTerms.ts'); -const crypto = require('crypto'); -const uuid = require('uuid').v4; +export default checkJWTTokenExist; -module.exports = checkJWTTokenExist; +// Preserve the CJS-callable shape consumers/tests expect — under tsc CJS emit +// this replaces the `exports` object with the function itself, so +// `require('./checkJWTTokenExist')(...)` works. No-op under typestrip ESM. +declare const module: any; +if (typeof module !== 'undefined') { + module.exports = checkJWTTokenExist; + module.exports.default = checkJWTTokenExist; +} /** * checks that the RSA keys exist for JWT generation, if not we create them */ diff --git a/utility/install/installer.ts b/utility/install/installer.ts index b54a86514..573e775e3 100644 --- a/utility/install/installer.ts +++ b/utility/install/installer.ts @@ -2,8 +2,9 @@ import * as os from 'os'; import inquirer from 'inquirer'; -import * as fs from 'fs-extra'; -import PropertiesReader from 'properties-reader'; +import fs from 'fs-extra'; +import _PropertiesReader from 'properties-reader'; +const PropertiesReader = _PropertiesReader; import chalk from 'chalk'; import * as path from 'path'; let ora; // Will be loaded dynamically as it's an ES module @@ -17,12 +18,15 @@ import * as hdbInfoController from '../../dataLayer/hdbInfoController.ts'; import { packageJson } from '../packageUtils.js'; import * as hdbTerms from '../hdbTerms.ts'; const { CONFIG_PARAMS } = hdbTerms; -import installValidator from '../../validation/installValidator.ts'; -import mountHdb from '../mount_hdb.ts'; -import * as configUtils from '../../config/configUtils.js'; +import _installValidator from '../../validation/installValidator.ts'; +const installValidator = _installValidator; +import _mountHdb from '../mount_hdb.ts'; +const mountHdb = _mountHdb; +import * as configUtils from '../../config/configUtils.ts'; import * as userOps from '../../security/user.ts'; import * as roleOps from '../../security/role.ts'; -import checkJwtTokens from './checkJWTTokensExist.js'; +import _checkJwtTokens from './checkJWTTokensExist.ts'; +const checkJwtTokens = _checkJwtTokens; import * as globalSchema from '../globalSchema.ts'; import { promisify } from 'util'; const pSchemaToGlobal = promisify(globalSchema.setSchemaDataToGlobal); diff --git a/utility/lifecycle.ts b/utility/lifecycle.ts new file mode 100644 index 000000000..d8640a364 --- /dev/null +++ b/utility/lifecycle.ts @@ -0,0 +1,68 @@ +// Startup-phase lifecycle: lets modules declare side-effectful initialization +// (server-singleton wiring, config-derived constants, listener registration) +// without running it at module-load time. The entry point invokes +// `runStartup()` after `env.initSync()` and before the server starts handling +// requests, so all hooks see a fully-linked module graph and an initialized +// environment. +// +// Usage: +// import { onStartup } from '.../utility/lifecycle.ts'; +// onStartup(() => { +// server.recordAnalytics = recordAction; +// }); +// +// Unit tests: any test that exercises code paths depending on these hooks must +// either call `runStartup()` itself in a `before`/`beforeEach`, or import the +// real CLI entry point. `runStartup()` is idempotent — calling it a second +// time is a no-op until `resetStartupForTests()` is called. + +type StartupCallback = () => void | Promise; + +const callbacks: StartupCallback[] = []; +let started = false; +let runningPromise: Promise | null = null; + +/** + * Register a callback to be run during the startup phase. If startup has + * already run, the callback is invoked on the next microtask. + */ +export function onStartup(cb: StartupCallback): void { + if (started) { + Promise.resolve().then(cb); + return; + } + callbacks.push(cb); +} + +/** + * Run all registered startup callbacks in registration order. Idempotent: + * subsequent calls return the same promise as the first invocation. + */ +export function runStartup(): Promise { + if (runningPromise) return runningPromise; + runningPromise = (async () => { + started = true; + // Snapshot in case callbacks register more callbacks (they'll run + // immediately via the microtask path above). + const pending = callbacks.splice(0, callbacks.length); + for (const cb of pending) { + await cb(); + } + })(); + return runningPromise; +} + +/** + * Reset startup state. Intended for unit tests that want to re-run startup + * (e.g. between describe blocks). Production code should never call this. + */ +export function resetStartupForTests(): void { + callbacks.length = 0; + started = false; + runningPromise = null; +} + +/** True once `runStartup()` has begun. */ +export function hasStarted(): boolean { + return started; +} diff --git a/utility/lmdb/OpenDBIObject.ts b/utility/lmdb/OpenDBIObject.ts index ab2f44046..3a8dc785f 100644 --- a/utility/lmdb/OpenDBIObject.ts +++ b/utility/lmdb/OpenDBIObject.ts @@ -2,8 +2,11 @@ import * as envMngr from '../environment/environmentManager.ts'; import * as terms from '../../utility/hdbTerms.ts'; import { RecordEncoder } from '../../resources/RecordEncoder.ts'; -envMngr.initSync(); - +try { + envMngr.initSync(); +} catch { + /* tolerate ESM cycle TDZ; bin entry will re-call later */ +} const LMDB_CACHING = envMngr.get(terms.CONFIG_PARAMS.STORAGE_CACHING) !== false; /** diff --git a/utility/lmdb/OpenEnvironmentObject.ts b/utility/lmdb/OpenEnvironmentObject.ts index c14583108..5c800983b 100644 --- a/utility/lmdb/OpenEnvironmentObject.ts +++ b/utility/lmdb/OpenEnvironmentObject.ts @@ -7,8 +7,11 @@ const MAX_DBS = 10000; const MAX_READERS = 2048; import * as envMngr from '../environment/environmentManager.ts'; import * as terms from '../../utility/hdbTerms.ts'; -envMngr.initSync(); - +try { + envMngr.initSync(); +} catch { + /* tolerate ESM cycle TDZ; bin entry will re-call later */ +} export default class OpenEnvironmentObject { [key: string]: any; static MAX_DBS = MAX_DBS; diff --git a/utility/lmdb/environmentUtility.ts b/utility/lmdb/environmentUtility.ts index c602e91fd..caf3a2901 100644 --- a/utility/lmdb/environmentUtility.ts +++ b/utility/lmdb/environmentUtility.ts @@ -1,7 +1,7 @@ 'use strict'; import * as lmdb from 'lmdb'; -import * as fs from 'fs-extra'; +import fs from 'fs-extra'; import * as path from 'path'; import * as common from './commonUtility.ts'; import log from '../logging/harper_logger.ts'; diff --git a/utility/lmdb/writeUtility.ts b/utility/lmdb/writeUtility.ts index bd29bc6ca..e18e08e33 100644 --- a/utility/lmdb/writeUtility.ts +++ b/utility/lmdb/writeUtility.ts @@ -13,8 +13,11 @@ import { v4 as uuidv4 } from 'uuid'; import * as lmdb from 'lmdb'; import { handleHDBError, hdbErrors } from '../errors/hdbError.ts'; import * as envMngr from '../environment/environmentManager.ts'; -envMngr.initSync(); - +try { + envMngr.initSync(); +} catch { + /* tolerate ESM cycle TDZ; bin entry will re-call later */ +} const LMDB_PREFETCH_WRITES = envMngr.get(hdbTerms.CONFIG_PARAMS.STORAGE_PREFETCHWRITES); const CREATED_TIME_ATTRIBUTE_NAME = hdbTerms.TIME_STAMP_NAMES_ENUM.CREATED_TIME; diff --git a/utility/logging/harper_logger.ts b/utility/logging/harper_logger.ts index eb2f8e3b2..05acbe43f 100644 --- a/utility/logging/harper_logger.ts +++ b/utility/logging/harper_logger.ts @@ -1,11 +1,12 @@ 'use strict'; // Note - do not import/use commonUtils.js in this module, it will cause circular dependencies. -import * as fs from 'fs-extra'; +import fs from 'fs-extra'; import { workerData, threadId, isMainThread } from 'worker_threads'; import * as pathModule from 'path'; import * as YAML from 'yaml'; -const PropertiesReader = require('properties-reader'); +import _PropertiesReader from 'properties-reader'; +const PropertiesReader = _PropertiesReader; import * as hdbTerms from '../hdbTerms.ts'; import assignCMDENVVariables from '../assignCmdEnvVariables.ts'; import * as os from 'os'; @@ -22,7 +23,6 @@ let nativeStdWrite = process.env.IS_SCRIPTED_SERVICE : (process.stdout as any).nativeWrite || ((process.stdout as any).nativeWrite = process.stdout.write); let fileLoggers = new Map(); const { join } = pathModule; - const MAX_LOG_BUFFER = 10000; const LOG_LEVEL_HIERARCHY = { notify: 7, @@ -141,6 +141,9 @@ function resolveLogPath(configPath: string, rootPath: string) { } async function updateLogSettings() { if (!rootConfig) { + // Lazy-load to avoid a circular dependency at module evaluation time + // (RootConfigWatcher imports configUtils which imports this logger). + const { RootConfigWatcher } = await import('../../config/RootConfigWatcher.ts'); // set up the initial watcher rootConfig = new RootConfigWatcher(); // wait for it to be ready @@ -264,36 +267,41 @@ class HarperLogger extends Console { if (hdbProperties === undefined) initLogSettings(); -module.exports = { - notify, - fatal, - error, - warn, - info, - debug, - trace, - logLevel, - loggerWithTag, - suppressLogging, - initLogSettings, - logCustomLevel, - closeLogFile, - createLogger, - logsAtLevel, - getLogFilePath: () => logFilePath, - forComponent: (name, isExternal) => mainLogger.forComponent(name, isExternal), - setMainLogger, - setLogLevel, - OUTPUTS, - AuthAuditLog, - // for now these functions at least notify us of when the component system is ready so - // we can start using the RootConfigWatcher - start: updateLogSettings, - startOnMainThread: updateLogSettings, - errorToString, - disableStdio, - externalLogger, -}; +// Under tsc CJS emit, expose the same CommonJS shape historical consumers +// (`const logger = require('./harper_logger')`) depend on. Skipped when this +// file is loaded as ESM (Node type-strip), where `module` is undefined. +if (typeof module !== 'undefined') { + module.exports = { + notify, + fatal, + error, + warn, + info, + debug, + trace, + logLevel, + loggerWithTag, + suppressLogging, + initLogSettings, + logCustomLevel, + closeLogFile, + createLogger, + logsAtLevel, + getLogFilePath: () => logFilePath, + forComponent: (name, isExternal) => mainLogger.forComponent(name, isExternal), + setMainLogger, + setLogLevel, + OUTPUTS, + AuthAuditLog, + // for now these functions at least notify us of when the component system is ready so + // we can start using the RootConfigWatcher + start: updateLogSettings, + startOnMainThread: updateLogSettings, + errorToString, + disableStdio, + externalLogger, + }; +} /** * We call this if stdio is not functional @@ -630,15 +638,21 @@ function getFileLogger(path, rotation, isExternalInstance) { setTimeout(() => { logger.rotator?.end(); if (!rotation) return; - const logRotator = require('./logRotator'); - try { - logger.rotator = logRotator({ - logger, - ...rotation, + import('./logRotator.ts') + .then((mod) => { + try { + const logRotator: any = mod.default ?? mod; + logger.rotator = logRotator({ + logger, + ...rotation, + }); + } catch (error) { + logger('Error initializing log rotator', error); + } + }) + .catch((error) => { + logger('Error initializing log rotator', error); }); - } catch (error) { - logger('Error initializing log rotator', error); - } }, 100); } return logger; @@ -921,9 +935,6 @@ export function AuthAuditLog( this.request_method = requestMethod; this.path = path; } -// we have to load this at the end to avoid circular dependencies problems -import { RootConfigWatcher } from '../../config/RootConfigWatcher.ts'; - export const getLogFilePath = () => logFilePath; export const forComponent = (name: string, isExternal?: boolean) => mainLogger.forComponent(name, isExternal); export default { diff --git a/utility/logging/logRotator.ts b/utility/logging/logRotator.ts index a0e24c7df..e3ca19da7 100644 --- a/utility/logging/logRotator.ts +++ b/utility/logging/logRotator.ts @@ -7,7 +7,11 @@ import { pipeline } from 'stream'; const pipe = promisify(pipeline); import * as path from 'path'; import * as envMgr from '../environment/environmentManager.ts'; -envMgr.initSync(); +try { + envMgr.initSync(); +} catch { + /* tolerate ESM cycle TDZ; bin entry will re-call later */ +} import hdbLogger from './harper_logger.ts'; import { CONFIG_PARAMS } from '../hdbTerms.ts'; import { convertToMS } from '../common_utils.ts'; diff --git a/utility/logging/readLog.ts b/utility/logging/readLog.ts index 5d5fd4b17..63e77ad50 100644 --- a/utility/logging/readLog.ts +++ b/utility/logging/readLog.ts @@ -4,9 +4,9 @@ import * as hdbTerms from '../hdbTerms.ts'; import hdbLogger from './harper_logger.ts'; import validator from '../../validation/readLogValidator.ts'; import * as path from 'path'; -import * as fs from 'fs-extra'; +import fs from 'fs-extra'; import { once } from 'events'; -import { getConfigPath } from '../../config/configUtils.js'; +import { getConfigPath } from '../../config/configUtils.ts'; import { handleHDBError, hdbErrors } from '../errors/hdbError.ts'; import { server } from '../../server/Server.ts'; diff --git a/utility/logging/transactionLog.ts b/utility/logging/transactionLog.ts index 2d9bb66f4..46bbeafec 100644 --- a/utility/logging/transactionLog.ts +++ b/utility/logging/transactionLog.ts @@ -9,7 +9,7 @@ import { readTransactionLogValidator, deleteTransactionLogsBeforeValidator, } from '../../validation/transactionLogValidator.ts'; -const harperBridge = require('../../dataLayer/harperBridge/harperBridge').default; +import harperBridge from '../../dataLayer/harperBridge/harperBridge.ts'; export async function readTransactionLog(req: any) { const validation = readTransactionLogValidator(req); diff --git a/utility/mount_hdb.ts b/utility/mount_hdb.ts index f9355ff04..fda954f2b 100644 --- a/utility/mount_hdb.ts +++ b/utility/mount_hdb.ts @@ -1,13 +1,19 @@ 'use strict'; -const { mkdirpSync, copySync } = require('fs-extra'); +import fsExtra from 'fs-extra'; +const { mkdirpSync, copySync } = fsExtra; import * as path from 'path'; import * as terms from '../utility/hdbTerms.ts'; import hdbLogger from '../utility/logging/harper_logger.ts'; import bridge from '../dataLayer/harperBridge/harperBridge.ts'; -import systemSchema from '../json/systemSchema.json'; -import * as initPaths from '../dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.js'; +import CreateTableObject from '../dataLayer/CreateTableObject.ts'; +import * as initPaths from '../dataLayer/harperBridge/lmdbBridge/lmdbUtility/initializePaths.ts'; import { PACKAGE_ROOT } from '../utility/packageUtils.js'; +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +const systemSchema: Record = JSON.parse( + readFileSync(join(PACKAGE_ROOT, 'json/systemSchema.json'), 'utf-8') +); export default async function mountHdb(hdbPath: string) { hdbLogger.trace('Mounting Harper'); @@ -28,9 +34,6 @@ export default async function mountHdb(hdbPath: string) { * @returns {Promise} */ async function createTables() { - const CreateTableObject = - require('../dataLayer/CreateTableObject').default || require('../dataLayer/CreateTableObject'); - let tables = Object.keys(systemSchema); for (const tableName of tables) { diff --git a/utility/npmUtilities.ts b/utility/npmUtilities.ts index 876519465..c1310eedc 100644 --- a/utility/npmUtilities.ts +++ b/utility/npmUtilities.ts @@ -6,12 +6,11 @@ import * as path from 'path'; import { handleHDBError, hdbErrors } from './errors/hdbError.ts'; const { HTTP_STATUS_CODES } = hdbErrors; - import * as validator from '../validation/validationWrapper.ts'; import harperLogger from './logging/harper_logger.ts'; import { CONFIG_PARAMS } from './hdbTerms.ts'; -import { getConfigPath } from '../config/configUtils.js'; +import { getConfigPath } from '../config/configUtils.ts'; import { nonInteractiveSpawn } from '../components/Application.ts'; /** diff --git a/utility/operation_authorization.ts b/utility/operation_authorization.ts index ee768a049..a4206cb18 100644 --- a/utility/operation_authorization.ts +++ b/utility/operation_authorization.ts @@ -26,12 +26,12 @@ import * as commonUtils from './common_utils.ts'; import * as restart from '../bin/restart.ts'; import * as terms from './hdbTerms.ts'; import { expandOperationsPerms } from './operationPermissions.ts'; -import * as permsTranslator from '../security/permissionsTranslator.js'; +import * as permsTranslator from '../security/permissionsTranslator.ts'; import { systemInformation } from '../utility/environment/systemInformation.ts'; import * as tokenAuthentication from '../security/tokenAuthentication.ts'; import * as auth from '../security/auth.ts'; -import * as configUtils from '../config/configUtils.js'; -import * as functionsOperations from '../components/operations.js'; +import * as configUtils from '../config/configUtils.ts'; +import * as functionsOperations from '../components/operations.ts'; import * as transactionLog from '../utility/logging/transactionLog.ts'; import * as npmUtilities from './npmUtilities.ts'; import * as analytics from '../resources/analytics/read.ts'; @@ -295,12 +295,6 @@ requiredPermissions.set(terms.VALID_SQL_OPS_ENUM.SELECT, new (permission as any) requiredPermissions.set(terms.VALID_SQL_OPS_ENUM.INSERT, new (permission as any)(false, [INSERT_PERM])); requiredPermissions.set(terms.VALID_SQL_OPS_ENUM.UPDATE, new (permission as any)(false, [UPDATE_PERM])); -module.exports = { - verifyPerms, - verifyPermsAST, - verifyBulkLoadAttributePerms, -}; - /** * Verifies permissions and restrictions for a SQL operation based on the user's assigned role. * @param ast - The SQL statement in Syntax Tree form. diff --git a/utility/packageUtils.js b/utility/packageUtils.js index 2279d4397..d5b9b8071 100644 --- a/utility/packageUtils.js +++ b/utility/packageUtils.js @@ -43,4 +43,22 @@ const packageJson = JSON.parse(readFileSync(packageJsonPath, 'utf8')); */ const PACKAGE_ROOT = dirname(packageJsonPath); -module.exports = { packageJson, PACKAGE_ROOT }; +/** + * The directory that holds source files at runtime: `PACKAGE_ROOT` in + * type-strip mode (where `node bin/harper.ts` runs the .ts sources directly) + * and `PACKAGE_ROOT/dist` in dist mode (where transpiled .js files live). + * + * `__dirname` of this CJS file resolves to either `/utility` + * (source) or `/dist/utility` (dist), so we can detect the mode + * just by looking at this file's own location. + */ +const RUNTIME_SRC_ROOT = __dirname.startsWith(join(PACKAGE_ROOT, 'dist')) ? join(PACKAGE_ROOT, 'dist') : PACKAGE_ROOT; + +/** + * File extension of the running modules: `.ts` in type-strip mode, `.js` in + * dist mode. Use this when constructing file paths for `new Worker(...)` or + * similar APIs that need the on-disk filename. + */ +const RUNTIME_FILE_EXT = RUNTIME_SRC_ROOT === PACKAGE_ROOT ? '.ts' : '.js'; + +module.exports = { packageJson, PACKAGE_ROOT, RUNTIME_SRC_ROOT, RUNTIME_FILE_EXT }; diff --git a/utility/processManagement/processManagement.js b/utility/processManagement/processManagement.ts similarity index 87% rename from utility/processManagement/processManagement.js rename to utility/processManagement/processManagement.ts index 828255c50..261cf5f19 100644 --- a/utility/processManagement/processManagement.js +++ b/utility/processManagement/processManagement.ts @@ -1,16 +1,14 @@ -'use strict'; +import * as hdbTerms from '../hdbTerms.ts'; +import * as servicesConfig from './servicesConfig.ts'; +import * as envMangr from '../environment/environmentManager.ts'; +import hdbLogger from '../../utility/logging/harper_logger.ts'; +import { onMessageFromWorkers } from '../../server/threads/manageThreads.ts'; +import fs from 'fs'; +import path from 'node:path'; +import { setTimeout as delay } from 'node:timers/promises'; +import { execFile, fork } from 'node:child_process'; -const hdbTerms = require('../hdbTerms.ts'); -const servicesConfig = require('./servicesConfig.js'); -const envMangr = require('../environment/environmentManager.ts'); -const hdbLogger = require('../../utility/logging/harper_logger.ts'); -const { onMessageFromWorkers } = require('../../server/threads/manageThreads.js'); -const fs = require('fs'); -const path = require('node:path'); -const { setTimeout: delay } = require('node:timers/promises'); -const { execFile, fork } = require('node:child_process'); - -module.exports = { +export { start, restart, kill, @@ -20,9 +18,11 @@ module.exports = { cleanupChildrenProcesses, expectedRestartOfChildren, }; - -onMessageFromWorkers((message) => { - if (message.type === 'restart') envMangr.initSync(true); +// Defer registration to setImmediate so manageThreads internal state is initialized +setImmediate(() => { + onMessageFromWorkers((message) => { + if (message.type === 'restart') envMangr.initSync(true); + }); }); let childProcesses = []; @@ -41,7 +41,7 @@ function start(procConfig, noKill = false) { ...procConfig.env, HARPER_PARENT_PROCESS_PID: process.pid.toString(), }; - const subprocess = procConfig.script + const subprocess: any = procConfig.script ? fork(procConfig.script, args, procConfig) : execFile(procConfig.binFile, args, procConfig); subprocess.name = procConfig.name; diff --git a/utility/processManagement/servicesConfig.js b/utility/processManagement/servicesConfig.ts similarity index 80% rename from utility/processManagement/servicesConfig.js rename to utility/processManagement/servicesConfig.ts index 0e269b038..4e98ea652 100644 --- a/utility/processManagement/servicesConfig.js +++ b/utility/processManagement/servicesConfig.ts @@ -1,9 +1,7 @@ -'use strict'; - -const hdbTerms = require('../hdbTerms.ts'); -const path = require('path'); -const { PACKAGE_ROOT } = require('../../utility/packageUtils.js'); -const hdbUtils = require('../common_utils.ts'); +import * as hdbTerms from '../hdbTerms.ts'; +import path from 'path'; +import { PACKAGE_ROOT } from '../../utility/packageUtils.js'; +import * as hdbUtils from '../common_utils.ts'; const SCRIPTS_DIR = path.join(PACKAGE_ROOT, 'utility/scripts'); const RESTART_SCRIPT = path.join(SCRIPTS_DIR, hdbTerms.HDB_RESTART_SCRIPT); @@ -49,8 +47,4 @@ function generateAllServiceConfigs() { }; } -module.exports = { - generateAllServiceConfigs, - generateMainServerConfig, - generateRestart, -}; +export { generateAllServiceConfigs, generateMainServerConfig, generateRestart }; diff --git a/utility/scripts/restartHdb.js b/utility/scripts/restartHdb.ts similarity index 83% rename from utility/scripts/restartHdb.js rename to utility/scripts/restartHdb.ts index 141702fa6..b95743a81 100644 --- a/utility/scripts/restartHdb.js +++ b/utility/scripts/restartHdb.ts @@ -1,7 +1,6 @@ -'use strict'; - -const pm2Utils = require('../processManagement/processManagement.js'); -const hdbTerms = require('../hdbTerms.ts'); +import * as _pm2Utils from '../processManagement/processManagement.ts'; +const pm2Utils: any = _pm2Utils; +import * as hdbTerms from '../hdbTerms.ts'; /** * Gets a list of all the running Harper processes and calls reload on each one. diff --git a/utility/signalling.ts b/utility/signalling.ts index 01a5930e4..5a53235d3 100644 --- a/utility/signalling.ts +++ b/utility/signalling.ts @@ -3,15 +3,33 @@ import * as hdbTerms from './hdbTerms.ts'; import hdbLogger from '../utility/logging/harper_logger.ts'; import ITCEventObject from '../server/itc/utility/ITCEventObject.js'; +import { sendItcEvent } from '../server/threads/itc.ts'; +import { onStartup } from './lifecycle.ts'; + let serverItcHandlers; -import { sendItcEvent } from '../server/threads/itc.js'; +// Preload server-itc-handlers during the startup phase so signalSchemaChange +// and signalUserChange can stay synchronous (their callers and tests assume so). +onStartup(async () => { + const mod: any = await import('../server/itc/serverHandlers.ts'); + // CJS dist double-wraps default: namespace.default is `exports`, exports.default is the real value. + serverItcHandlers = mod.default?.default ?? mod.default ?? mod; +}); + +function ensureServerItcHandlers() { + // In production the `onStartup` hook above preloads this. In unit tests + // where startup never runs, downstream consumers may call us synchronously; + // if `serverItcHandlers` is still undefined the optional-chained access in + // the caller no-ops, matching the original lazy `require` semantics where + // a failed load was caught and logged. + return serverItcHandlers; +} export function signalSchemaChange(message: any) { try { hdbLogger.debug('signalSchemaChange called with message:', message); - serverItcHandlers = serverItcHandlers || require('../server/itc/serverHandlers.js'); + ensureServerItcHandlers(); const itcEventSchema = new ITCEventObject(hdbTerms.ITC_EVENT_TYPES.SCHEMA, message); - serverItcHandlers.schema(itcEventSchema); + serverItcHandlers?.schema(itcEventSchema); return sendItcEvent(itcEventSchema); } catch (err) { hdbLogger.error(err); @@ -21,9 +39,9 @@ export function signalSchemaChange(message: any) { export function signalUserChange(message: any) { try { hdbLogger.trace('signalUserChange called with message:', message); - serverItcHandlers = serverItcHandlers || require('../server/itc/serverHandlers.js'); + ensureServerItcHandlers(); const itcEventUser = new ITCEventObject(hdbTerms.ITC_EVENT_TYPES.USER, message); - serverItcHandlers.user(itcEventUser); + serverItcHandlers?.user(itcEventUser); return sendItcEvent(itcEventUser); } catch (err) { hdbLogger.error(err); diff --git a/validation/configValidator.ts b/validation/configValidator.ts index ffc6d6bbd..0e7657204 100644 --- a/validation/configValidator.ts +++ b/validation/configValidator.ts @@ -1,6 +1,6 @@ 'use strict'; -import * as fs from 'fs-extra'; +import fs from 'fs-extra'; import Joi from 'joi'; import * as os from 'os'; const { boolean, string, number, array } = Joi.types(); diff --git a/validation/fileLoadValidator.ts b/validation/fileLoadValidator.ts index b4d11e6c3..4ad056f22 100644 --- a/validation/fileLoadValidator.ts +++ b/validation/fileLoadValidator.ts @@ -7,7 +7,6 @@ import joi from 'joi'; const { string } = joi.types(); import { hdbErrors, handleHDBError } from '../utility/errors/hdbError.ts'; const { HTTP_STATUS_CODES } = hdbErrors; - import { commonValidators } from './common_validators.ts'; const isRequiredString = ' is required'; diff --git a/validation/installValidator.ts b/validation/installValidator.ts index 63b6521d6..3251c35ae 100644 --- a/validation/installValidator.ts +++ b/validation/installValidator.ts @@ -2,7 +2,7 @@ import Joi from 'joi'; const { string, number } = Joi.types(); -import * as fs from 'fs-extra'; +import fs from 'fs-extra'; import * as hdbTerms from '../utility/hdbTerms.ts'; import * as path from 'path'; import * as validator from './validationWrapper.ts'; diff --git a/validation/readLogValidator.ts b/validation/readLogValidator.ts index e31d6d1df..5ebd81f2a 100644 --- a/validation/readLogValidator.ts +++ b/validation/readLogValidator.ts @@ -3,9 +3,9 @@ import Joi from 'joi'; import * as validator from './validationWrapper.ts'; import moment from 'moment'; -import * as fs from 'fs-extra'; +import fs from 'fs-extra'; import * as path from 'path'; -import { getConfigPath } from '../config/configUtils.js'; +import { getConfigPath } from '../config/configUtils.ts'; import * as hdbTerms from '../utility/hdbTerms.ts'; import { LOG_LEVELS } from '../utility/hdbTerms.ts'; diff --git a/validation/role_validation.ts b/validation/role_validation.ts index a2c65b4b6..3b7b2d1d4 100644 --- a/validation/role_validation.ts +++ b/validation/role_validation.ts @@ -1,11 +1,11 @@ -const validate = require('validate.js'); -const validator = require('./validationWrapper'); +import validate from 'validate.js'; +import * as _validator from './validationWrapper.ts'; +const validator = _validator; import * as terms from '../utility/hdbTerms.ts'; import { validateOperations } from '../utility/operationPermissions.ts'; import { handleHDBError, hdbErrors } from '../utility/errors/hdbError.ts'; const { HDB_ERROR_MSGS, HTTP_STATUS_CODES } = hdbErrors; - const constraintsTemplate = () => ({ role: { presence: true, diff --git a/validation/schemaMetadataValidator.ts b/validation/schemaMetadataValidator.ts index 28d6486e0..266125237 100644 --- a/validation/schemaMetadataValidator.ts +++ b/validation/schemaMetadataValidator.ts @@ -1,6 +1,8 @@ 'use strict'; -export const schemaDescribe = require('../dataLayer/schemaDescribe'); +import * as _schemaDescribe from '../dataLayer/schemaDescribe.ts'; +const schemaDescribe = _schemaDescribe; +export { schemaDescribe }; import { hdbErrors } from '../utility/errors/hdbError.ts'; import { getDatabases } from '../resources/databases.ts'; diff --git a/validation/searchValidator.ts b/validation/searchValidator.ts index 5c1b1f4f9..58f75e9bc 100644 --- a/validation/searchValidator.ts +++ b/validation/searchValidator.ts @@ -1,4 +1,4 @@ -import * as _ from 'lodash'; +import _ from 'lodash'; import * as validator from './validationWrapper.ts'; import Joi from 'joi'; import * as hdbUtils from '../utility/common_utils.ts'; @@ -7,7 +7,6 @@ import { handleHDBError, hdbErrors } from '../utility/errors/hdbError.ts'; import { getDatabases } from '../resources/databases.ts'; const { HTTP_STATUS_CODES } = hdbErrors; - const searchByValueSchema = Joi.object({ database: hdbDatabase, schema: hdbDatabase, @@ -112,7 +111,7 @@ export default function (searchObject: any, type: any) { return handleHDBError(new Error(), checkSchemaTable, HTTP_STATUS_CODES.NOT_FOUND); } - let tableSchema = getDatabases()[searchObject.schema][searchObject.table]; + let tableSchema: any = getDatabases()[searchObject.schema][searchObject.table]; let allTableAttributes = tableSchema.attributes; //this clones the get_attributes array diff --git a/validation/validationWrapper.ts b/validation/validationWrapper.ts index c538435d4..d378a5b4d 100644 --- a/validation/validationWrapper.ts +++ b/validation/validationWrapper.ts @@ -11,7 +11,8 @@ * These are rare enough for it not to be worth creating wrapper functions for those as well. */ -const validate = require('validate.js'); +import _validate from 'validate.js'; +const validate = _validate; //This validator is added here b/c we are still on version 0.11.1 that does not include this build in functionality. When // we do update, we can remove. The reason we have not is related to a breaking change on the "presence" validator rule @@ -74,7 +75,7 @@ export async function validateObjectAsync(object, fileConstraints) { } try { - await validate.async(object, fileConstraints, { format: 'flat' }); + await validate.async(object, fileConstraints, { format: 'flat' } as any); } catch (err) { // unroll the array and make a full error message. let msg = err.join(`,`);