feat(common): TEMP COMMIT -- precompile ajv schemas

mcdurdin · mcdurdin · commit b4f03453218a · 2023-10-05T12:36:27.000+07:00
ajv seems to have some bugs around precompiling and formats, particularly when using ESM: * ajv-validator/ajv#1837 * ajv-validator/ajv-cli#200 Workaround for non-esm works for cloudevents/sdk-javascript#471 So if we want to work with this, we should look at how that is done, but it is taking too many cycles to pursue for now.
diff --git a/common/web/types/.eslintrc.cjs b/common/web/types/.eslintrc.cjs
@@ -8,6 +8,8 @@ module.exports = {
     "coverage/*",
     "node_modules/*",
     "test/fixtures/*",
+    "tools/*",
+    "src/schemas/*"
   ],
   overrides: [
     {
diff --git a/common/web/types/build.sh b/common/web/types/build.sh
@@ -46,9 +46,16 @@ function compile_schemas() {
 
   # TODO: use https://github.com/tc39/proposal-json-modules instead of this once it stablises
   for schema in "${schemas[@]}"; do
-    local fn="$THIS_SCRIPT_PATH/src/schemas/$(basename "$schema" .json)"
+    local schema_base="$(basename "$schema" .json)"
+    local fn="$THIS_SCRIPT_PATH/src/schemas/$schema_base"
+
+    builder_echo "Compiling schema $schema_base.json"
     echo 'export default ' > "$fn.ts"
     cat "$fn.json" >> "$fn.ts"
+    # echo "declare module '$schema_base';" > "$fn.validator.d.ts"
+    # node tools/compile-schema.js "$fn.json" "$fn.validator.js"
+    ajv compile -c "./tools/formats.cjs" -s "$fn.json" -o "$fn.validator.cjs"
+    #--code-esm true
   done
 }
 
@@ -69,7 +76,7 @@ function copy_cldr_imports() {
 function do_configure() {
   compile_schemas
   copy_cldr_imports
-  verify_npm_setup
+  # verify_npm_setup
 }
 
 function do_test() {
diff --git a/common/web/types/package.json b/common/web/types/package.json
@@ -27,7 +27,6 @@
   },
   "dependencies": {
     "@keymanapp/keyman-version": "*",
-    "ajv": "^8.11.0",
     "restructure": "git+https://github.com/keymanapp/dependency-restructure.git#7a188a1e26f8f36a175d95b67ffece8702363dfc",
     "semver": "^7.5.2",
     "xml2js": "git+https://github.com/keymanapp/dependency-node-xml2js#535fe732dc408d697e0f847c944cc45f0baf0829"
@@ -39,6 +38,9 @@
     "@types/node": "^20.4.1",
     "@types/semver": "^7.3.12",
     "@types/xml2js": "^0.4.5",
+    "ajv": "^8.12.0",
+    "ajv-cli": "^5.0.0",
+    "ajv-formats": "^2.1.1",
     "c8": "^7.12.0",
     "chai": "^4.3.4",
     "chalk": "^2.4.2",
@@ -74,6 +76,7 @@
       "src/ldml-keyboard/unicodeset-parser-api.ts",
       "src/keyman-touch-layout/keyman-touch-layout-file-writer.ts",
       "src/osk/osk.ts",
+      "src/schemas/*",
       "test/"
     ]
   }
diff --git a/common/web/types/src/keyman-touch-layout/keyman-touch-layout-file-reader.ts b/common/web/types/src/keyman-touch-layout/keyman-touch-layout-file-reader.ts
@@ -1,7 +1,5 @@
-import { default as AjvModule } from 'ajv';
-const Ajv = AjvModule.default; // The actual expected Ajv type.
 import { TouchLayoutFile } from "./keyman-touch-layout-file.js";
-import Schemas from '../../src/schemas.js';
+import SchemaValidators from '../schema-validators.js';
 
 export class TouchLayoutFileReader {
   public read(source: Uint8Array): TouchLayoutFile {
@@ -69,11 +67,10 @@ export class TouchLayoutFileReader {
   }
 
   public validate(source: TouchLayoutFile): void {
-    const ajv = new Ajv();
-    if(!ajv.validate(Schemas.touchLayoutClean, source))
+    if(!SchemaValidators.touchLayoutClean(source))
     /* c8 ignore next 3 */
     {
-      throw new Error(ajv.errorsText());
+      throw new Error((<any>SchemaValidators.touchLayoutClean).errors);
     }
   }
 
diff --git a/common/web/types/src/kpj/kpj-file-reader.ts b/common/web/types/src/kpj/kpj-file-reader.ts
@@ -1,11 +1,9 @@
 import * as xml2js from 'xml2js';
 import { KPJFile, KPJFileProject } from './kpj-file.js';
-import { default as AjvModule } from 'ajv';
-const Ajv = AjvModule.default; // The actual expected Ajv type.
 import { boxXmlArray } from '../util/util.js';
 import { KeymanDeveloperProject, KeymanDeveloperProjectFile10, KeymanDeveloperProjectType } from './keyman-developer-project.js';
 import { CompilerCallbacks } from '../util/compiler-interfaces.js';
-import Schemas from '../schemas.js';
+import SchemaValidators from '../schema-validators.js';
 
 export class KPJFileReader {
   constructor(private callbacks: CompilerCallbacks) {
@@ -35,13 +33,11 @@ export class KPJFileReader {
   }
 
   public validate(source: KPJFile): void {
-    const ajv = new Ajv();
-    if(!ajv.validate(Schemas.kpj, source)) {
-      const ajvLegacy = new Ajv();
-      if(!ajvLegacy.validate(Schemas.kpj90, source)) {
+    if(!SchemaValidators.kpj(source)) {
+      if(!SchemaValidators.kpj90(source)) {
         // If the legacy schema also does not validate, then we will only report
         // the errors against the modern schema
-        throw new Error(ajv.errorsText());
+        throw new Error((<any>SchemaValidators.kpj).errors);
       }
     }
   }
diff --git a/common/web/types/src/kvk/kvks-file-reader.ts b/common/web/types/src/kvk/kvks-file-reader.ts
@@ -1,12 +1,10 @@
 import * as xml2js from 'xml2js';
 import KVKSourceFile from './kvks-file.js';
-import { default as AjvModule } from 'ajv';
-const Ajv = AjvModule.default; // The actual expected Ajv type.
 import { boxXmlArray } from '../util/util.js';
 import { DEFAULT_KVK_FONT, VisualKeyboard, VisualKeyboardHeaderFlags, VisualKeyboardKey, VisualKeyboardKeyFlags, VisualKeyboardLegalShiftStates, VisualKeyboardShiftState } from './visual-keyboard.js';
 import { USVirtualKeyCodes } from '../consts/virtual-key-constants.js';
 import { BUILDER_KVK_HEADER_VERSION, KVK_HEADER_IDENTIFIER_BYTES } from './kvk-file.js';
-import Schemas from '../schemas.js';
+import SchemaValidators from '../schema-validators.js';
 
 
 export default class KVKSFileReader {
@@ -85,9 +83,8 @@ export default class KVKSFileReader {
   }
 
   public validate(source: KVKSourceFile): void {
-    const ajv = new Ajv();
-    if(!ajv.validate(Schemas.kvks, source)) {
-      throw new Error(ajv.errorsText());
+    if(!SchemaValidators.kvks(source)) {
+      throw new Error((<any>SchemaValidators.kvks).errorsText());
     }
   }
 
diff --git a/common/web/types/src/ldml-keyboard/ldml-keyboard-xml-reader.ts b/common/web/types/src/ldml-keyboard/ldml-keyboard-xml-reader.ts
@@ -1,13 +1,11 @@
 import * as xml2js from 'xml2js';
 import { LDMLKeyboardXMLSourceFile, LKImport } from './ldml-keyboard-xml.js';
-import { default as AjvModule } from 'ajv';
-const Ajv = AjvModule.default; // The actual expected Ajv type.
 import { boxXmlArray } from '../util/util.js';
 import { CompilerCallbacks } from '../util/compiler-interfaces.js';
 import { constants } from '@keymanapp/ldml-keyboard-constants';
 import { CommonTypesMessages } from '../util/common-events.js';
 import { LDMLKeyboardTestDataXMLSourceFile, LKTTest, LKTTests } from './ldml-keyboard-testdata-xml.js';
-import Schemas from '../schemas.js';
+import SchemaValidators from '../schema-validators.js';
 
 interface NameAndProps  {
   '$'?: any; // content
@@ -207,9 +205,8 @@ export class LDMLKeyboardXMLSourceFileReader {
    * @returns true if valid, false if invalid
    */
   public validate(source: LDMLKeyboardXMLSourceFile | LDMLKeyboardTestDataXMLSourceFile): boolean {
-    const ajv = new Ajv();
-    if(!ajv.validate(Schemas.ldmlKeyboard3, source)) {
-      for (let err of ajv.errors) {
+    if(!SchemaValidators.ldmlKeyboard3(source)) {
+      for (let err of (<any>SchemaValidators.ldmlKeyboard3).errors) {
         this.callbacks.reportMessage(CommonTypesMessages.Error_SchemaValidationError({
           instancePath: err.instancePath,
           keyword: err.keyword,
diff --git a/common/web/types/src/main.ts b/common/web/types/src/main.ts
@@ -47,4 +47,5 @@ export * as KeymanFileTypes from './util/file-types.js';
 
 export * as Osk from './osk/osk.js';
 
-export * as Schemas from './schemas.js';
+export * as Schemas from './schemas.js';
+export * as SchemaValidators from './schema-validators.js';
diff --git a/common/web/types/src/osk/osk.ts b/common/web/types/src/osk/osk.ts
@@ -1,8 +1,6 @@
 import { TouchLayoutFile, TouchLayoutFlick, TouchLayoutKey, TouchLayoutPlatform, TouchLayoutSubKey } from "src/keyman-touch-layout/keyman-touch-layout-file.js";
 import { VisualKeyboard } from "../kvk/visual-keyboard.js";
-import { default as AjvModule } from 'ajv';
-import Schemas from "../schemas.js";
-const Ajv = AjvModule.default; // The actual expected Ajv type.
+import SchemaValidators from "../schema-validators.js";
 
 export interface StringRefUsage {
   filename: string;
@@ -24,11 +22,10 @@ export interface StringResult {
 export type PuaMap = {[index:string]: string};
 
 export function parseMapping(mapping: any) {
-  const ajv = new Ajv();
-  if(!ajv.validate(Schemas.displayMap, <any>mapping))
+  if(!SchemaValidators.displayMap(<any>mapping))
   /* c8 ignore next 3 */
   {
-    throw new Error(ajv.errorsText());
+    throw new Error((<any>SchemaValidators.displayMap).errorsText());
   }
 
   let map: PuaMap = {};
diff --git a/common/web/types/src/schema-validators.ts b/common/web/types/src/schema-validators.ts
@@ -0,0 +1,27 @@
+import kpj from './schemas/kpj.schema.validator.cjs';
+import kpj90 from './schemas/kpj-9.0.schema.validator.cjs';
+import kvks from './schemas/kvks.schema.validator.cjs';
+import ldmlKeyboard3 from './schemas/ldml-keyboard3.schema.validator.cjs';
+import ldmlKeyboardTest3 from './schemas/ldml-keyboardtest3.schema.validator.cjs';
+import displayMap from './schemas/displaymap.schema.validator.cjs';
+import touchLayoutClean from './schemas/keyman-touch-layout.clean.spec.validator.cjs';
+import touchLayout from './schemas/keyman-touch-layout.spec.validator.cjs';
+import keyboard_info from './schemas/keyboard_info.schema.validator.cjs';
+
+// import AjvFormatsModule from 'ajv-formats';
+// const ajvFormats = AjvFormatsModule.default;
+// ajvFormats.default(keyboard_info);
+
+const SchemaValidators = {
+  kpj,
+  kpj90,
+  kvks,
+  ldmlKeyboard3,
+  ldmlKeyboardTest3,
+  displayMap,
+  touchLayoutClean,
+  touchLayout,
+  keyboard_info,
+};
+
+export default SchemaValidators;
diff --git a/common/web/types/test/helpers/index.ts b/common/web/types/test/helpers/index.ts
@@ -1,5 +1,5 @@
-import path from "path";
-import fs from "fs";
+import * as path from "path";
+import * as fs from "fs";
 import { fileURLToPath } from "url";
 
 /**
diff --git a/common/web/types/test/tsconfig.json b/common/web/types/test/tsconfig.json
@@ -7,7 +7,8 @@
     "outDir": "../build/test",
     "baseUrl": ".",
     "strictNullChecks": false, // TODO: get rid of this as some point
-    "allowSyntheticDefaultImports": true // for ajv
+    "allowJs": true,
+    "allowSyntheticDefaultImports": true
   },
   "include": [
     "**/test-*.ts",
diff --git a/common/web/types/tools/compile-schema.cjs b/common/web/types/tools/compile-schema.cjs
@@ -0,0 +1,41 @@
+const fs = require('fs');
+// import * as fs from 'fs';
+// import * as path from 'path';
+const Ajv = require('ajv');
+// import AjvModule from 'ajv';
+// const AjvFormats = require('ajv-formats');
+// import AjvFormatsModule from 'ajv-formats';
+const standaloneCode = require('ajv/dist/standalone');
+// import standaloneCode from 'ajv/dist/standalone/index.js';
+// import formats from 'ajv-formats/dist/formats.js';
+// const Ajv = AjvModule.default; // The actual expected Ajv type.
+// const ajvFormats = AjvFormatsModule.default;
+const ajvFormats = require('ajv-formats');
+// import formats from './formats.cjs';
+
+// import { createRequire } from 'module';
+// const require = createRequire;
+
+const schema = JSON.parse(fs.readFileSync(process.argv[2], 'utf-8'));
+
+// ajvFormats['date-time'] = {validate, compile}
+
+// The generated code will have a default export:
+// `module.exports = <validateFunctionCode>;module.exports.default = <validateFunctionCode>;`
+const ajv = new Ajv({code: {source: true}})
+// let formats = require('ajv-formats/dist/formats');
+ajvFormats(ajv);//.default(ajv);
+//ajv.opts.code.formats = formats.fullFormats;
+// formats(ajv);
+// ajv.opts.code.formats = require('ajv-formats').fullFormats;
+// ajv.opts.code.formats = formats(); //{validate:ajv.validate, compile:ajv.compile};
+// ajv.opts.code.formats = { "date-time": { validate: ajv.formats['date-time'].validate.toString(), compare: ajv.formats['date-time'].compare.toString() } };
+
+const validate = ajv.compile(schema)
+// ajvFormats.default(validate)
+const moduleCode = standaloneCode(ajv, validate)
+
+fs.writeFileSync(process.argv[3], moduleCode, 'utf-8');
+
+// Now you can write the module code to file
+// fs.writeFileSync(path.join(__dirname, "../consume/validate-cjs.js"), moduleCode)
diff --git a/common/web/types/tools/formats.cjs b/common/web/types/tools/formats.cjs
@@ -0,0 +1,11 @@
+// import * as ajvFormats from 'ajv-formats';
+
+// export default function formats(ajv) {
+//   ajvFormats.default(ajv);
+// }
+
+function formats(ajv) {
+  require("ajv-formats")(ajv);
+}
+
+module.exports = formats;
diff --git a/common/web/types/tsconfig.json b/common/web/types/tsconfig.json
@@ -5,11 +5,12 @@
     "outDir": "build/src/",
     "rootDir": "src/",
     "baseUrl": ".",
-    "allowSyntheticDefaultImports": true, // for ajv
-    "resolveJsonModule": true,
+    "allowJs": true,
+    "allowSyntheticDefaultImports": true,
   },
   "include": [
-    "src/**/*.ts"
+    "src/**/*.ts",
+    "src/schemas/*.cjs" // Import the validators
   ],
   "references": [
     { "path": "../keyman-version" },
diff --git a/common/web/utils/package.json b/common/web/utils/package.json
@@ -23,17 +23,20 @@
   },
   "homepage": "https://github.com/keymanapp/keyman#readme",
   "devDependencies": {
-    "@keymanapp/resources-gosh": "*",
     "@keymanapp/keyman-version": "*",
+    "@keymanapp/resources-gosh": "*",
+    "@types/node": "^14.0.5",
     "c8": "^7.12.0",
     "chai": "^4.3.4",
     "mocha": "^10.0.0",
     "mocha-teamcity-reporter": "^4.0.0",
-    "@types/node": "^14.0.5",
     "typescript": "^4.9.5"
   },
   "type": "module",
   "paths": {
     "@keymanapp/keyman-version": "*"
+  },
+  "dependencies": {
+    "ajv-formats": "^2.1.1"
   }
 }
diff --git a/developer/src/kmc-keyboard-info/build.sh b/developer/src/kmc-keyboard-info/build.sh
@@ -11,6 +11,7 @@ cd "$THIS_SCRIPT_PATH"
 
 builder_describe "Build Keyman kmc keyboard-info Compiler module" \
   "@/common/web/types" \
+  "@/developer/src/common/web/utils" \
   "clean" \
   "configure" \
   "build" \
diff --git a/developer/src/kmc-keyboard-info/package.json b/developer/src/kmc-keyboard-info/package.json
@@ -25,9 +25,8 @@
   },
   "dependencies": {
     "@keymanapp/common-types": "*",
-    "@keymanapp/kmc-package": "*",
     "@keymanapp/developer-utils": "*",
-    "ajv": "^8.11.0",
+    "@keymanapp/kmc-package": "*",
     "ajv-formats": "^2.1.1"
   },
   "bundleDependencies": [
diff --git a/developer/src/kmc-keyboard-info/src/index.ts b/developer/src/kmc-keyboard-info/src/index.ts
@@ -11,12 +11,7 @@ import langtags from "./imports/langtags.js";
 import { validateMITLicense } from "@keymanapp/developer-utils";
 import { KmpCompiler } from "@keymanapp/kmc-package";
 
-import AjvModule from 'ajv';
-import AjvFormatsModule from 'ajv-formats';
-const Ajv = AjvModule.default; // The actual expected Ajv type.
-const ajvFormats = AjvFormatsModule.default;
-
-import { Schemas } from "@keymanapp/common-types";
+import { SchemaValidators } from "@keymanapp/common-types";
 import { packageKeysExamplesToKeyboardInfo } from "./example-keys.js";
 
 const regionNames = new Intl.DisplayNames(['en'], { type: "region" });
@@ -290,17 +285,10 @@ export class KeyboardInfoCompiler {
 
     const jsonOutput = JSON.stringify(keyboard_info, null, 2);
 
-    // TODO: look at performance improvements by precompiling Ajv schemas on first use
-    const ajv = new Ajv({ logger: {
-      log: (message) => this.callbacks.reportMessage(KeyboardInfoCompilerMessages.Hint_OutputValidation({message})),
-      warn: (message) => this.callbacks.reportMessage(KeyboardInfoCompilerMessages.Warn_OutputValidation({message})),
-      error: (message) => this.callbacks.reportMessage(KeyboardInfoCompilerMessages.Error_OutputValidation({message})),
-    }});
-    ajvFormats.default(ajv);
-    if(!ajv.validate(Schemas.default.keyboard_info, keyboard_info)) {
+    if(!SchemaValidators.default.keyboard_info(keyboard_info)) {
       // This is an internal fatal error; we should not be capable of producing
       // invalid output, so it is best to throw and die
-      throw new Error(ajv.errorsText());
+      throw new Error((<any>SchemaValidators.default.keyboard_info).errorsText());
     }
 
     return new TextEncoder().encode(jsonOutput);
diff --git a/developer/src/kmc-ldml/package.json b/developer/src/kmc-ldml/package.json
diff --git a/developer/src/kmc-ldml/tsconfig.json b/developer/src/kmc-ldml/tsconfig.json
diff --git a/package-lock.json b/package-lock.json
diff --git a/tsconfig.esm-base.json b/tsconfig.esm-base.json

Original file line number	Diff line number	Diff line change
`@@ -8,6 +8,8 @@ module.exports = {`
`8`	`8`	`"coverage/*",`
`9`	`9`	`"node_modules/*",`
`10`	`10`	`"test/fixtures/*",`
	`11`	`+ "tools/*",`
	`12`	`+ "src/schemas/*"`
`11`	`13`	`],`
`12`	`14`	`overrides: [`
`13`	`15`	`{`
Original file line number	Diff line number	Diff line change
`@@ -1,7 +1,5 @@`
`1`		`-import { default as AjvModule } from 'ajv';`
`2`		`-const Ajv = AjvModule.default; // The actual expected Ajv type.`
`3`	`1`	`import { TouchLayoutFile } from "./keyman-touch-layout-file.js";`
`4`		`-import Schemas from '../../src/schemas.js';`
	`2`	`+import SchemaValidators from '../schema-validators.js';`
`5`	`3`
`6`	`4`	`export class TouchLayoutFileReader {`
`7`	`5`	`public read(source: Uint8Array): TouchLayoutFile {`
`@@ -69,11 +67,10 @@ export class TouchLayoutFileReader {`
`69`	`67`	`}`
`70`	`68`
`71`	`69`	`public validate(source: TouchLayoutFile): void {`
`72`		`- const ajv = new Ajv();`
`73`		`- if(!ajv.validate(Schemas.touchLayoutClean, source))`
	`70`	`+ if(!SchemaValidators.touchLayoutClean(source))`
`74`	`71`	`/* c8 ignore next 3 */`
`75`	`72`	`{`
`76`		`- throw new Error(ajv.errorsText());`
	`73`	`+ throw new Error((<any>SchemaValidators.touchLayoutClean).errors);`
`77`	`74`	`}`
`78`	`75`	`}`
`79`	`76`