From 153683cfb007c6066c9dcf71d25afa4c66efa17f Mon Sep 17 00:00:00 2001
From: Eberhard Beilharz
Date: Tue, 20 Aug 2024 15:10:14 +0200
Subject: [PATCH 001/360] fix(core): look for `emcc` instead of `emcc.py` on
Linux and Mac
`emcc.py` is not marked as executable in emcripten's git repo so the
build failed when trying to locate emscripten. However, it turns out
that `emcc` is marked as executable, so we use that instead.
On Windows however, we still need to use `emcc.py` because otherwise
Meson won't detect it as valid compiler.
---
core/wasm.build.linux.in | 6 ++---
core/wasm.build.mac.in | 6 ++---
core/wasm.defs.build | 6 ++---
developer/src/kmcmplib/wasm.build.linux.in | 6 ++---
developer/src/kmcmplib/wasm.build.mac.in | 6 ++---
resources/locate_emscripten.inc.sh | 30 +++++++++++++---------
6 files changed, 33 insertions(+), 27 deletions(-)
diff --git a/core/wasm.build.linux.in b/core/wasm.build.linux.in
index d449dcf94a0..27cb7ba68fe 100644
--- a/core/wasm.build.linux.in
+++ b/core/wasm.build.linux.in
@@ -1,4 +1,4 @@
[binaries]
-c = ['$EMSCRIPTEN_BASE/emcc.py']
-cpp = ['$EMSCRIPTEN_BASE/em++.py']
-ar = ['$EMSCRIPTEN_BASE/emar.py']
\ No newline at end of file
+c = ['$EMSCRIPTEN_BASE/emcc']
+cpp = ['$EMSCRIPTEN_BASE/em++']
+ar = ['$EMSCRIPTEN_BASE/emar']
diff --git a/core/wasm.build.mac.in b/core/wasm.build.mac.in
index d449dcf94a0..27cb7ba68fe 100644
--- a/core/wasm.build.mac.in
+++ b/core/wasm.build.mac.in
@@ -1,4 +1,4 @@
[binaries]
-c = ['$EMSCRIPTEN_BASE/emcc.py']
-cpp = ['$EMSCRIPTEN_BASE/em++.py']
-ar = ['$EMSCRIPTEN_BASE/emar.py']
\ No newline at end of file
+c = ['$EMSCRIPTEN_BASE/emcc']
+cpp = ['$EMSCRIPTEN_BASE/em++']
+ar = ['$EMSCRIPTEN_BASE/emar']
diff --git a/core/wasm.defs.build b/core/wasm.defs.build
index 4a598a1e1c1..f41af337a38 100644
--- a/core/wasm.defs.build
+++ b/core/wasm.defs.build
@@ -1,7 +1,7 @@
[binaries]
-c = ['emcc.py']
-cpp = ['em++.py']
-ar = ['emar.py']
+c = ['emcc']
+cpp = ['em++']
+ar = ['emar']
exe_wrapper = 'node'
[properties]
diff --git a/developer/src/kmcmplib/wasm.build.linux.in b/developer/src/kmcmplib/wasm.build.linux.in
index c5e61a9bda9..27cb7ba68fe 100644
--- a/developer/src/kmcmplib/wasm.build.linux.in
+++ b/developer/src/kmcmplib/wasm.build.linux.in
@@ -1,4 +1,4 @@
[binaries]
-c = ['$EMSCRIPTEN_BASE/emcc.py']
-cpp = ['$EMSCRIPTEN_BASE/em++.py']
-ar = ['$EMSCRIPTEN_BASE/emar.py']
+c = ['$EMSCRIPTEN_BASE/emcc']
+cpp = ['$EMSCRIPTEN_BASE/em++']
+ar = ['$EMSCRIPTEN_BASE/emar']
diff --git a/developer/src/kmcmplib/wasm.build.mac.in b/developer/src/kmcmplib/wasm.build.mac.in
index c5e61a9bda9..27cb7ba68fe 100644
--- a/developer/src/kmcmplib/wasm.build.mac.in
+++ b/developer/src/kmcmplib/wasm.build.mac.in
@@ -1,4 +1,4 @@
[binaries]
-c = ['$EMSCRIPTEN_BASE/emcc.py']
-cpp = ['$EMSCRIPTEN_BASE/em++.py']
-ar = ['$EMSCRIPTEN_BASE/emar.py']
+c = ['$EMSCRIPTEN_BASE/emcc']
+cpp = ['$EMSCRIPTEN_BASE/em++']
+ar = ['$EMSCRIPTEN_BASE/emar']
diff --git a/resources/locate_emscripten.inc.sh b/resources/locate_emscripten.inc.sh
index be66adfa32f..20923770e2a 100644
--- a/resources/locate_emscripten.inc.sh
+++ b/resources/locate_emscripten.inc.sh
@@ -2,30 +2,36 @@
# no hashbang for .inc.sh
#
-# We don't want to rely on emcc.py being on the path, because Emscripten puts far
+# We don't want to rely on emcc being on the path, because Emscripten puts far
# too many things onto the path (in particular for us, node).
#
-# The following comment suggests that we don't need emcc.py on the path.
+# The following comment suggests that we don't need emcc on the path.
# https://github.com/emscripten-core/emscripten/issues/4848#issuecomment-1097357775
#
-# So we try and locate emcc.py in common locations ourselves. The search pattern
+# So we try and locate emcc in common locations ourselves. The search pattern
# is:
#
# 1. Look for $EMSCRIPTEN_BASE (our primary emscripten variable), which should
-# point to the folder that emcc.py is located in
-# 2. Look for $EMCC which should point to the emcc.py executable
-# 3. Look for emcc.py on the path
+# point to the folder that emcc is located in
+# 2. Look for $EMCC which should point to the emcc executable
+# 3. Look for emcc on the path
#
locate_emscripten() {
+ local EMCC_EXECUTABLE
+ if [[ "${BUILDER_OS}" == "win" ]]; then
+ EMCC_EXECUTABLE="emcc.py"
+ else
+ EMCC_EXECUTABLE="emcc"
+ fi
if [[ -z ${EMSCRIPTEN_BASE+x} ]]; then
if [[ -z ${EMCC+x} ]]; then
- local EMCC=`which emcc.py`
- [[ -z $EMCC ]] && builder_die "locate_emscripten: Could not locate emscripten (emcc.py) on the path or with \$EMCC or \$EMSCRIPTEN_BASE"
+ local EMCC=$(which ${EMCC_EXECUTABLE})
+ [[ -z $EMCC ]] && builder_die "locate_emscripten: Could not locate emscripten (${EMCC_EXECUTABLE}) on the path or with \$EMCC or \$EMSCRIPTEN_BASE"
fi
- [[ -f $EMCC && ! -x $EMCC ]] && builder_die "locate_emscripten: Variable EMCC ($EMCC) points to emcc.py but it is not executable"
- [[ -x $EMCC ]] || builder_die "locate_emscripten: Variable EMCC ($EMCC) does not point to a valid executable emcc.py"
+ [[ -f $EMCC && ! -x $EMCC ]] && builder_die "locate_emscripten: Variable EMCC ($EMCC) points to ${EMCC_EXECUTABLE} but it is not executable"
+ [[ -x $EMCC ]] || builder_die "locate_emscripten: Variable EMCC ($EMCC) does not point to a valid executable ${EMCC_EXECUTABLE}"
EMSCRIPTEN_BASE="$(dirname "$EMCC")"
fi
- [[ -f ${EMSCRIPTEN_BASE}/emcc.py && ! -x ${EMSCRIPTEN_BASE}/emcc.py ]] && builder_die "locate_emscripten: Variable EMSCRIPTEN_BASE ($EMSCRIPTEN_BASE) contains emcc.py but it is not executable"
- [[ -x ${EMSCRIPTEN_BASE}/emcc.py ]] || builder_die "locate_emscripten: Variable EMSCRIPTEN_BASE ($EMSCRIPTEN_BASE) does not point to emcc.py's folder"
+ [[ -f ${EMSCRIPTEN_BASE}/${EMCC_EXECUTABLE} && ! -x ${EMSCRIPTEN_BASE}/${EMCC_EXECUTABLE} ]] && builder_die "locate_emscripten: Variable EMSCRIPTEN_BASE ($EMSCRIPTEN_BASE) contains ${EMCC_EXECUTABLE} but it is not executable"
+ [[ -x ${EMSCRIPTEN_BASE}/${EMCC_EXECUTABLE} ]] || builder_die "locate_emscripten: Variable EMSCRIPTEN_BASE ($EMSCRIPTEN_BASE) does not point to ${EMCC_EXECUTABLE}'s folder"
}
From 0abc250af38a64b2a3aaa7113a2ca99067fe0bd3 Mon Sep 17 00:00:00 2001
From: Eberhard Beilharz
Date: Tue, 20 Aug 2024 12:45:16 +0200
Subject: [PATCH 002/360] feat(web): POC of Core WASM integration into Keyman
Web
- add temporary function to Core for this POC
- add new CoreProcessor to access Keyman Core WASM
- add unit tests for new core processor
- add code to KeymanEngine and InputProcessor to load the new CoreProcessor
- add web server to manual tests and new action `start` to build script
This change requires the manual tests to be loaded from a web server
instead of loaded as file, because otherwise the wasm code won't be
loaded.
Currently we always load CoreProcessor. This should be improved in a future
change to only load when it is actually needed.
Part-of: #11293
---
core/src/meson.build | 38 ++++++++
core/src/wasm.cpp | 92 +++++++++++++++++++
package-lock.json | 2 +
resources/build/minimum-versions.inc.sh | 2 +-
web/README.md | 5 +-
web/build.sh | 8 ++
web/package.json | 14 ++-
web/src/app/browser/build.sh | 4 +
web/src/app/webview/build.sh | 11 +++
web/src/engine/core-processor/.gitignore | 1 +
web/src/engine/core-processor/build.sh | 64 +++++++++++++
.../core-processor/src/core-processor.ts | 30 ++++++
web/src/engine/core-processor/src/index.ts | 1 +
web/src/engine/core-processor/tsconfig.json | 13 +++
.../interfaces/src/pathConfiguration.ts | 4 +
web/src/engine/main/build.sh | 1 +
.../main/src/headless/inputProcessor.ts | 9 +-
web/src/engine/main/src/keymanEngine.ts | 2 +
.../dom/cases/core-processor/basic.spec.ts | 21 +++++
.../test/auto/dom/web-test-runner.config.mjs | 13 ++-
web/src/test/manual/build.sh | 2 +-
web/src/tools/testing/test-server/index.cjs | 12 +++
22 files changed, 339 insertions(+), 10 deletions(-)
create mode 100644 core/src/wasm.cpp
create mode 100644 web/src/engine/core-processor/.gitignore
create mode 100755 web/src/engine/core-processor/build.sh
create mode 100644 web/src/engine/core-processor/src/core-processor.ts
create mode 100644 web/src/engine/core-processor/src/index.ts
create mode 100644 web/src/engine/core-processor/tsconfig.json
create mode 100644 web/src/test/auto/dom/cases/core-processor/basic.spec.ts
create mode 100644 web/src/tools/testing/test-server/index.cjs
diff --git a/core/src/meson.build b/core/src/meson.build
index 41c198543cb..564472b7980 100644
--- a/core/src/meson.build
+++ b/core/src/meson.build
@@ -118,6 +118,7 @@ api_files = files(
'km_core_state_api.cpp',
'km_core_debug_api.cpp',
'km_core_processevent_api.cpp',
+ 'wasm.cpp',
)
core_files = files(
@@ -133,6 +134,23 @@ mock_files = files(
'mock/mock_processor.cpp',
)
+if cpp_compiler.get_id() == 'emscripten'
+ host_links = ['--whole-archive', '-sALLOW_MEMORY_GROWTH=1', '-sMODULARIZE=1', '-sEXPORT_ES6', '-sENVIRONMENT=webview', '--embind-emit-tsd', 'core-interface.d.ts', '-sERROR_ON_UNDEFINED_SYMBOLS=0']
+
+ if cpp_compiler.version().version_compare('>=3.1.44')
+ # emscripten 3.1.44 removes .asm object and so we need to export `wasmExports`
+ # #9375; https://github.com/emscripten-core/emscripten/blob/main/ChangeLog.md#3144---072523
+ links += ['-sEXPORTED_RUNTIME_METHODS=[\'UTF8ToString\',\'stringToNewUTF8\',\'wasmExports\']']
+ else
+ # emscripten < 3.1.44 does not include `wasmExports`
+ links += ['-sEXPORTED_RUNTIME_METHODS=[\'UTF8ToString\',\'stringToNewUTF8\']']
+ endif
+
+ links += [
+ # Forcing inclusion of debug symbols
+ '-g', '-Wlimited-postlink-optimizations', '--bind']
+endif
+
lib = library('keymancore',
api_files,
core_files,
@@ -160,3 +178,23 @@ pkg.generate(
description: 'Keyman processor for KMN keyboards.',
subdirs: headerdirs,
libraries: lib)
+
+if cpp_compiler.get_id() == 'emscripten'
+ # Build an executable
+ host = executable('core',
+ cpp_args: defns,
+ include_directories: inc,
+ link_args: links + host_links,
+ objects: lib.extract_all_objects(recursive: false))
+
+ if get_option('buildtype') == 'release'
+ # Split debug symbols into separate wasm file for release builds only
+ # as the release symbols will be uploaded to sentry
+ # custom_target('kmcmplib.wasm',
+ # depends: host,
+ # input: host,
+ # output: 'kmcmplib.wasm',
+ # command: ['wasm-split', '@OUTDIR@/wasm-host.wasm', '-o', '@OUTPUT@', '--strip', '--debug-out=@OUTDIR@/kmcmplib.debug.wasm'],
+ # build_by_default: true)
+ endif
+endif
diff --git a/core/src/wasm.cpp b/core/src/wasm.cpp
new file mode 100644
index 00000000000..d9c4277a96b
--- /dev/null
+++ b/core/src/wasm.cpp
@@ -0,0 +1,92 @@
+#ifdef __EMSCRIPTEN__
+#ifdef __EMSCRIPTEN__
+#include
+#include
+
+#else
+#define EMSCRIPTEN_KEEPALIVE
+#endif
+
+#ifdef __cplusplus
+#define EXTERN extern "C" EMSCRIPTEN_KEEPALIVE
+#else
+#define EXTERN EMSCRIPTEN_KEEPALIVE
+#endif
+
+#include
+
+constexpr km_core_attr const engine_attrs = {
+ 256,
+ KM_CORE_LIB_CURRENT,
+ KM_CORE_LIB_AGE,
+ KM_CORE_LIB_REVISION,
+ KM_CORE_TECH_KMX,
+ "SIL International"
+};
+
+EMSCRIPTEN_KEEPALIVE km_core_attr const & tmp_wasm_attributes() {
+ return engine_attrs;
+}
+
+EMSCRIPTEN_BINDINGS(compiler_interface) {
+
+ // emscripten::class_("WasmCallbackInterface")
+ // .function("message", &WasmCallbackInterface::message, emscripten::pure_virtual())
+ // .function("loadFile", &WasmCallbackInterface::loadFile, emscripten::pure_virtual())
+ // .allow_subclass("WasmCallbackInterfaceWrapper")
+ // ;
+
+ // emscripten::class_("CompilerOptions")
+ // .constructor<>()
+ // .property("saveDebug", &KMCMP_COMPILER_OPTIONS::saveDebug)
+ // .property("compilerWarningsAsErrors", &KMCMP_COMPILER_OPTIONS::compilerWarningsAsErrors)
+ // .property("warnDeprecatedCode", &KMCMP_COMPILER_OPTIONS::warnDeprecatedCode)
+ // .property("shouldAddCompilerVersion", &KMCMP_COMPILER_OPTIONS::shouldAddCompilerVersion)
+ // .property("target", &KMCMP_COMPILER_OPTIONS::target)
+ // ;
+
+ // emscripten::class_("CompilerResult")
+ // .constructor<>()
+ // .property("result", &WASM_COMPILER_RESULT::result)
+ // .property("kmx", &WASM_COMPILER_RESULT::kmx)
+ // .property("kmxSize", &WASM_COMPILER_RESULT::kmxSize)
+ // .property("extra", &WASM_COMPILER_RESULT::extra)
+ // ;
+
+ // emscripten::class_("CompilerResultMessage")
+ // .constructor<>()
+ // .property("errorCode", &KMCMP_COMPILER_RESULT_MESSAGE::errorCode)
+ // .property("lineNumber", &KMCMP_COMPILER_RESULT_MESSAGE::lineNumber)
+ // .property("columnNumber", &KMCMP_COMPILER_RESULT_MESSAGE::columnNumber)
+ // .property("filename", &KMCMP_COMPILER_RESULT_MESSAGE::filename)
+ // .property("parameters", &KMCMP_COMPILER_RESULT_MESSAGE::parameters)
+ // ;
+
+ // emscripten::class_("CompilerResultExtra")
+ // .constructor<>()
+ // .property("targets", &KMCMP_COMPILER_RESULT_EXTRA::targets)
+ // .property("kmnFilename", &KMCMP_COMPILER_RESULT_EXTRA::kmnFilename)
+ // .property("kvksFilename", &KMCMP_COMPILER_RESULT_EXTRA::kvksFilename)
+ // .property("displayMapFilename", &KMCMP_COMPILER_RESULT_EXTRA::displayMapFilename)
+ // .property("stores", &KMCMP_COMPILER_RESULT_EXTRA::stores)
+ // .property("groups", &KMCMP_COMPILER_RESULT_EXTRA::groups)
+ // ;
+
+ // emscripten::value_object("CompilerResultExtraStore")
+ // .field("storeType", &KMCMP_COMPILER_RESULT_EXTRA_STORE::storeType)
+ // .field("name", &KMCMP_COMPILER_RESULT_EXTRA_STORE::name)
+ // .field("line", &KMCMP_COMPILER_RESULT_EXTRA_STORE::line)
+ // ;
+
+ emscripten::value_object("km_core_attr")
+ .field("max_context", &km_core_attr::max_context)
+ .field("current", &km_core_attr::current)
+ .field("revision", &km_core_attr::revision)
+ .field("age", &km_core_attr::age)
+ .field("technology", &km_core_attr::technology)
+ //.field("vendor", &km_core_attr::vendor, emscripten::allow_raw_pointers())
+ ;
+
+ emscripten::function("tmp_wasm_attributes", &tmp_wasm_attributes);
+}
+#endif
diff --git a/package-lock.json b/package-lock.json
index cee7d8528b7..c8451a75191 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -8514,6 +8514,7 @@
"version": "4.19.2",
"resolved": "https://registry.npmjs.org/express/-/express-4.19.2.tgz",
"integrity": "sha512-5T6nhjsT+EOMzuck8JjBHARTHfMht0POzlA60WV2pMD3gyXw2LZnZ+ueGdNxG+0calOJcWKbpFcuzLZ91YWq9Q==",
+ "license": "MIT",
"dependencies": {
"accepts": "~1.3.8",
"array-flatten": "1.1.1",
@@ -14760,6 +14761,7 @@
"@sentry/cli": "^2.31.0",
"@zip.js/zip.js": "^2.7.32",
"c8": "^7.12.0",
+ "express": "^4.19.2",
"jsdom": "^23.0.1",
"mocha": "^10.0.0"
}
diff --git a/resources/build/minimum-versions.inc.sh b/resources/build/minimum-versions.inc.sh
index c060d31c58f..385f1647544 100644
--- a/resources/build/minimum-versions.inc.sh
+++ b/resources/build/minimum-versions.inc.sh
@@ -20,7 +20,7 @@ KEYMAN_MIN_TARGET_VERSION_CHROME=95.0 # Final version that runs on Andro
# Dependency versions
KEYMAN_MIN_VERSION_NODE_MAJOR=20 # node version source of truth is /package.json:/engines/node
KEYMAN_MIN_VERSION_NPM=10.5.1 # 10.5.0 has bug, discussed in #10350
-KEYMAN_MIN_VERSION_EMSCRIPTEN=3.1.44 # Warning: 3.1.45 is bad (#9529); newer versions work
+KEYMAN_MIN_VERSION_EMSCRIPTEN=3.1.58
KEYMAN_MAX_VERSION_EMSCRIPTEN=3.1.58 # See #9529
KEYMAN_MIN_VERSION_VISUAL_STUDIO=2019
KEYMAN_MIN_VERSION_MESON=1.0.0
diff --git a/web/README.md b/web/README.md
index f09fe519bbb..ef7344adffb 100644
--- a/web/README.md
+++ b/web/README.md
@@ -29,8 +29,9 @@ src/test/auto A Node-driven test suite for automated testing of Key
## Usage
-Open **index.html** or **samples/index.html** in your browser. Be sure to
-compile Keyman Engine for Web before viewing the pages.
+Start the test server by running `./build.sh start`, then open
+your browser to http://localhost:3000. Be sure to compile Keyman Engine
+for Web before viewing the pages.
Refer to the samples for usage details.
diff --git a/web/build.sh b/web/build.sh
index 16d3d112356..27243535940 100755
--- a/web/build.sh
+++ b/web/build.sh
@@ -17,12 +17,14 @@ builder_describe "Builds engine modules for Keyman Engine for Web (KMW)." \
"clean" \
"configure" \
"build" \
+ "start Starts the test server" \
"test" \
"coverage Create an HTML page with code coverage" \
":app/browser The form of Keyman Engine for Web for use on websites" \
":app/webview A puppetable version of KMW designed for use in a host app's WebView" \
":app/ui Builds KMW's desktop form-factor keyboard-selection UI modules" \
":engine/attachment Subset used for detecting valid page contexts for use in text editing " \
+ ":engine/core-processor Keyman Core WASM integration" \
":engine/device-detect Subset used for device-detection " \
":engine/dom-utils A common subset of function used for DOM calculations, layout, etc" \
":engine/events Specialized classes utilized to support KMW API events" \
@@ -56,6 +58,7 @@ builder_describe_outputs \
build:app/webview "/web/build/app/webview/${config}/keymanweb-webview.js" \
build:app/ui "/web/build/app/ui/${config}/kmwuitoggle.js" \
build:engine/attachment "/web/build/engine/attachment/lib/index.mjs" \
+ build:engine/core-processor "/web/build/engine/core-processor/lib/index.mjs" \
build:engine/device-detect "/web/build/engine/device-detect/lib/index.mjs" \
build:engine/dom-utils "/web/build/engine/dom-utils/obj/index.js" \
build:engine/events "/web/build/engine/events/lib/index.mjs" \
@@ -159,6 +162,8 @@ builder_run_child_actions build:engine/attachment
# Uses engine/interfaces (due to resource-path config interface)
builder_run_child_actions build:engine/keyboard-storage
+builder_run_child_actions build:engine/core-processor
+
# Uses engine/interfaces, engine/device-detect, engine/keyboard-storage, & engine/osk
builder_run_child_actions build:engine/main
@@ -188,3 +193,6 @@ builder_run_action test test_action
# Create coverage report
builder_run_action coverage coverage_action
+
+# Start the test server
+builder_run_action start node src/tools/testing/test-server/index.cjs
diff --git a/web/package.json b/web/package.json
index 96a11c0cd3e..8a170dedaf4 100644
--- a/web/package.json
+++ b/web/package.json
@@ -12,10 +12,10 @@
"types": "./build/engine/attachment/obj/index.d.ts",
"import": "./build/engine/attachment/obj/index.js"
},
- "./engine/interfaces": {
- "es6-bundling": "./src/engine/interfaces/src/index.ts",
- "types": "./build/engine/interfaces/obj/index.d.ts",
- "import": "./build/engine/interfaces/obj/index.js"
+ "./engine/core-processor": {
+ "es6-bundling": "./src/engine/core-processor/src/index.ts",
+ "types": "./build/engine/core-processor/obj/index.d.ts",
+ "import": "./build/engine/core-processor/obj/index.js"
},
"./engine/device-detect": {
"es6-bundling": "./src/engine/device-detect/src/index.ts",
@@ -37,6 +37,11 @@
"types": "./build/engine/events/obj/index.d.ts",
"import": "./build/engine/events/obj/index.js"
},
+ "./engine/interfaces": {
+ "es6-bundling": "./src/engine/interfaces/src/index.ts",
+ "types": "./build/engine/interfaces/obj/index.d.ts",
+ "import": "./build/engine/interfaces/obj/index.js"
+ },
"./engine/js-processor": {
"es6-bundling": "./src/engine/js-processor/src/index.ts",
"types": "./build/engine/js-processor/obj/index.d.ts",
@@ -112,6 +117,7 @@
"@sentry/cli": "^2.31.0",
"@zip.js/zip.js": "^2.7.32",
"c8": "^7.12.0",
+ "express": "^4.19.2",
"jsdom": "^23.0.1",
"mocha": "^10.0.0"
},
diff --git a/web/src/app/browser/build.sh b/web/src/app/browser/build.sh
index 1d757e264a2..bf135637d1c 100755
--- a/web/src/app/browser/build.sh
+++ b/web/src/app/browser/build.sh
@@ -73,6 +73,10 @@ compile_and_copy() {
mkdir -p "$KEYMAN_ROOT/web/build/app/resources/osk"
cp -R "$KEYMAN_ROOT/web/src/resources/osk/." "$KEYMAN_ROOT/web/build/app/resources/osk/"
+ # Copy the WASM host
+ cp "${KEYMAN_ROOT}/web/build/engine/core-processor/obj/import/core/"core.{js,wasm} "${KEYMAN_ROOT}/web/build/app/browser/debug/"
+ cp "${KEYMAN_ROOT}/web/build/engine/core-processor/obj/import/core/"core.{js,wasm} "${KEYMAN_ROOT}/web/build/app/browser/release/"
+
# Update the build/publish copy of our build artifacts
prepare
diff --git a/web/src/app/webview/build.sh b/web/src/app/webview/build.sh
index e22e1b470ef..13e6fe2ddf1 100755
--- a/web/src/app/webview/build.sh
+++ b/web/src/app/webview/build.sh
@@ -58,8 +58,16 @@ compile_and_copy() {
mkdir -p "$KEYMAN_ROOT/web/build/app/resources/osk"
cp -R "$KEYMAN_ROOT/web/src/resources/osk/." "$KEYMAN_ROOT/web/build/app/resources/osk/"
+ # Copy the WASM host
+ cp "${KEYMAN_ROOT}/web/build/engine/core-processor/obj/import/core/"core.{js,wasm} "${KEYMAN_ROOT}/web/build/app/webview/debug/"
+ cp "${KEYMAN_ROOT}/web/build/engine/core-processor/obj/import/core/"core.{js,wasm} "${KEYMAN_ROOT}/web/build/app/webview/release/"
+
# Clean the sourcemaps of .. and . components
for script in "$KEYMAN_ROOT/web/build/$SUBPROJECT_NAME/debug/"*.js; do
+ if [[ "${script}" == *"/core.js" ]]; then
+ continue
+ fi
+
sourcemap="$script.map"
node "$KEYMAN_ROOT/web/build/tools/building/sourcemap-root/index.js" \
"$script" "$sourcemap" --clean --inline
@@ -68,6 +76,9 @@ compile_and_copy() {
# Do NOT inline sourcemaps for release builds - we don't want them to affect
# load time.
for script in "$KEYMAN_ROOT/web/build/$SUBPROJECT_NAME/release/"*.js; do
+ if [[ "${script}" == *"/core.js" ]]; then
+ continue
+ fi
sourcemap="$script.map"
node "$KEYMAN_ROOT/web/build/tools/building/sourcemap-root/index.js" \
"$script" "$sourcemap" --clean
diff --git a/web/src/engine/core-processor/.gitignore b/web/src/engine/core-processor/.gitignore
new file mode 100644
index 00000000000..282f91762fb
--- /dev/null
+++ b/web/src/engine/core-processor/.gitignore
@@ -0,0 +1 @@
+src/import/
\ No newline at end of file
diff --git a/web/src/engine/core-processor/build.sh b/web/src/engine/core-processor/build.sh
new file mode 100755
index 00000000000..14379c87654
--- /dev/null
+++ b/web/src/engine/core-processor/build.sh
@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+
+## START STANDARD BUILD SCRIPT INCLUDE
+# adjust relative paths as necessary
+THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")"
+. "${THIS_SCRIPT%/*}/../../../../resources/build/builder.inc.sh"
+## END STANDARD BUILD SCRIPT INCLUDE
+
+SUBPROJECT_NAME=engine/core-processor
+
+. "${KEYMAN_ROOT}/web/common.inc.sh"
+. "${KEYMAN_ROOT}/resources/shellHelperFunctions.sh"
+
+# ################################ Main script ################################
+
+builder_describe "Keyman Core WASM integration" \
+ "@/core:wasm" \
+ "clean" \
+ "configure" \
+ "build" \
+ "test" \
+ "--ci+ Set to utilize CI-based test configurations & reporting."
+
+builder_describe_outputs \
+ configure "/web/src/engine/core-processor/src/import/core/core-interface.d.ts" \
+ build "/web/build/${SUBPROJECT_NAME}/lib/index.mjs"
+
+builder_parse "$@"
+
+#### Build action definitions ####
+
+do_clean() {
+ rm -rf "${KEYMAN_ROOT}/web/build/${SUBPROJECT_NAME}"
+ rm -rf "src/import/"
+}
+
+do_configure() {
+ verify_npm_setup
+
+ mkdir -p "src/import/core/"
+ # we don't need this file here, but it's nice to have for reference and auto-completion
+ cp "${KEYMAN_ROOT}/core/build/wasm/${BUILDER_CONFIGURATION}/src/core-interface.d.ts" "src/import/core/"
+}
+
+copy_deps() {
+ mkdir -p "${KEYMAN_ROOT}/web/build/${SUBPROJECT_NAME}/obj/import/core/"
+ cp "${KEYMAN_ROOT}/core/build/wasm/${BUILDER_CONFIGURATION}/src/"core{.js,.wasm,-interface.d.ts} "${KEYMAN_ROOT}/web/build/${SUBPROJECT_NAME}/obj/import/core/"
+}
+
+do_build () {
+ copy_deps
+ compile "${SUBPROJECT_NAME}"
+
+ ${BUNDLE_CMD} "${KEYMAN_ROOT}/web/build/${SUBPROJECT_NAME}/obj/index.js" \
+ --out "${KEYMAN_ROOT}/web/build/${SUBPROJECT_NAME}/lib/index.mjs" \
+ --format esm
+}
+
+builder_run_action clean do_clean
+builder_run_action configure do_configure
+builder_run_action build do_build
+
+# No headless tests for this child project. Currently, DOM-based unit &
+# integrated tests are run solely by the top-level $KEYMAN_ROOT/web project.
diff --git a/web/src/engine/core-processor/src/core-processor.ts b/web/src/engine/core-processor/src/core-processor.ts
new file mode 100644
index 00000000000..3b53a0ec838
--- /dev/null
+++ b/web/src/engine/core-processor/src/core-processor.ts
@@ -0,0 +1,30 @@
+type km_core_attr = import('./import/core/core-interface.js').km_core_attr;
+
+export class CoreProcessor {
+ private instance: any;
+
+ /**
+ * Initialize Core Processor
+ * @param baseurl - The url where core.js is located
+ */
+ public async init(baseurl: string): Promise {
+
+ if (!this.instance) {
+ try {
+ const module = await import(baseurl + '/core.js');
+ this.instance = await module.default({
+ locateFile: function (path: string, scriptDirectory: string) {
+ return baseurl + '/' + path;
+ }
+ });
+ } catch (e: any) {
+ return false;
+ }
+ }
+ return !!this.instance;
+ };
+
+ public tmp_wasm_attributes(): km_core_attr {
+ return this.instance.tmp_wasm_attributes();
+ }
+}
diff --git a/web/src/engine/core-processor/src/index.ts b/web/src/engine/core-processor/src/index.ts
new file mode 100644
index 00000000000..06f040ce538
--- /dev/null
+++ b/web/src/engine/core-processor/src/index.ts
@@ -0,0 +1 @@
+export * from './core-processor.js';
\ No newline at end of file
diff --git a/web/src/engine/core-processor/tsconfig.json b/web/src/engine/core-processor/tsconfig.json
new file mode 100644
index 00000000000..84996aaca16
--- /dev/null
+++ b/web/src/engine/core-processor/tsconfig.json
@@ -0,0 +1,13 @@
+{
+ // While the actual references themselves are headless, it compiles against the DOM-reliant OSK module.
+ "extends": "../../tsconfig.dom.json",
+
+ "compilerOptions": {
+ "baseUrl": "./",
+ "outDir": "../../../build/engine/core-processor/obj/",
+ "tsBuildInfoFile": "../../../build/engine/core-processor/obj/tsconfig.tsbuildinfo",
+ "rootDir": "./src"
+ },
+
+ "include": [ "**/*.ts", "src/import/core/core.js" ],
+}
diff --git a/web/src/engine/interfaces/src/pathConfiguration.ts b/web/src/engine/interfaces/src/pathConfiguration.ts
index 0eff8f65f78..18080374e48 100644
--- a/web/src/engine/interfaces/src/pathConfiguration.ts
+++ b/web/src/engine/interfaces/src/pathConfiguration.ts
@@ -106,6 +106,10 @@ export default class PathConfiguration implements OSKResourcePathConfiguration {
return this._root;
}
+ get basePath(): string {
+ return this.sourcePath;
+ }
+
get resources(): string {
return this._resources;
}
diff --git a/web/src/engine/main/build.sh b/web/src/engine/main/build.sh
index 77673626e02..5abcb59008c 100755
--- a/web/src/engine/main/build.sh
+++ b/web/src/engine/main/build.sh
@@ -14,6 +14,7 @@ SUBPROJECT_NAME=engine/main
builder_describe "Builds the Keyman Engine for Web's common top-level base classes." \
"@/common/web/keyman-version" \
+ "@/web/src/engine/core-processor" \
"@/web/src/engine/keyboard" \
"@/web/src/engine/interfaces build" \
"@/web/src/engine/device-detect build" \
diff --git a/web/src/engine/main/src/headless/inputProcessor.ts b/web/src/engine/main/src/headless/inputProcessor.ts
index 6d8904c563d..94c95f56012 100644
--- a/web/src/engine/main/src/headless/inputProcessor.ts
+++ b/web/src/engine/main/src/headless/inputProcessor.ts
@@ -4,9 +4,10 @@
import ContextWindow from "./contextWindow.js";
import { LanguageProcessor } from "./languageProcessor.js";
-import type { ModelSpec } from "keyman/engine/interfaces";
+import type { ModelSpec, PathConfiguration } from "keyman/engine/interfaces";
import { globalObject, DeviceSpec } from "@keymanapp/web-utils";
+import { CoreProcessor } from "keyman/engine/core-processor";
import { Codes, type Keyboard, type KeyEvent } from "keyman/engine/keyboard";
import {
type Alternate,
@@ -35,6 +36,7 @@ export class InputProcessor {
private contextDevice: DeviceSpec;
private kbdProcessor: KeyboardProcessor;
private lngProcessor: LanguageProcessor;
+ private coreProcessor: CoreProcessor;
private readonly contextCache = new TranscriptionCache();
@@ -50,6 +52,11 @@ export class InputProcessor {
this.contextDevice = device;
this.kbdProcessor = new KeyboardProcessor(device, options);
this.lngProcessor = new LanguageProcessor(predictiveTextWorker, this.contextCache);
+ this.coreProcessor = new CoreProcessor();
+ }
+
+ public async init(paths: PathConfiguration) {
+ this.coreProcessor.init(paths.basePath);
}
public get languageProcessor(): LanguageProcessor {
diff --git a/web/src/engine/main/src/keymanEngine.ts b/web/src/engine/main/src/keymanEngine.ts
index 851134f258f..5901bbded3e 100644
--- a/web/src/engine/main/src/keymanEngine.ts
+++ b/web/src/engine/main/src/keymanEngine.ts
@@ -234,6 +234,8 @@ export default class KeymanEngine<
// Initialize supplementary plane string extensions
String.kmwEnableSupplementaryPlane(true);
+ await this.core.init(config.paths);
+
// Since we're not sandboxing keyboard loads yet, we just use `window` as the jsGlobal object.
// All components initialized below require a properly-configured `config.paths` or similar.
const keyboardLoader = new KeyboardLoader(this.interface, config.applyCacheBusting);
diff --git a/web/src/test/auto/dom/cases/core-processor/basic.spec.ts b/web/src/test/auto/dom/cases/core-processor/basic.spec.ts
new file mode 100644
index 00000000000..c24748e9c58
--- /dev/null
+++ b/web/src/test/auto/dom/cases/core-processor/basic.spec.ts
@@ -0,0 +1,21 @@
+import { assert } from 'chai';
+import { CoreProcessor } from 'keyman/engine/core-processor';
+
+const coreurl = '/web/build/engine/core-processor/obj/import/core/';
+
+// Test the CoreProcessor interface.
+describe('CoreProcessor', function () {
+ it('can initialize without errors', async function () {
+ const kp = new CoreProcessor();
+ assert.isTrue(await kp.init(coreurl));
+ });
+
+ it('can call temp function', async function () {
+ const kp = new CoreProcessor();
+ await kp.init(coreurl);
+ const a = kp.tmp_wasm_attributes();
+ assert.isNotNull(a);
+ assert.isNumber(a.max_context);
+ console.dir(a);
+ });
+});
diff --git a/web/src/test/auto/dom/web-test-runner.config.mjs b/web/src/test/auto/dom/web-test-runner.config.mjs
index 62edfc7dbf9..1d9534a77ba 100644
--- a/web/src/test/auto/dom/web-test-runner.config.mjs
+++ b/web/src/test/auto/dom/web-test-runner.config.mjs
@@ -40,6 +40,11 @@ export default {
// Relative, from the containing package.json
files: ['build/test/dom/cases/browser/**/*.spec.mjs']
},
+ {
+ name: 'engine/core-processor',
+ // Relative, from the containing package.json
+ files: ['build/test/dom/cases/core-processor/**/*.spec.mjs']
+ },
{
name: 'engine/dom-utils',
// Relative, from the containing package.json
@@ -59,7 +64,7 @@ export default {
name: 'engine/keyboard-storage',
// Relative, from the containing package.json
files: ['build/test/dom/cases/keyboard-storage/**/*.spec.mjs']
- }
+ },
],
middleware: [
// Rewrites short-hand paths for test resources, making them fully relative to the repo root.
@@ -68,6 +73,12 @@ export default {
context.url = '/web/src/test/auto' + context.url;
}
+ return next();
+ },
+ function rewriteWasmContentType(context, next) {
+ if (context.url.endsWith('.wasm')) {
+ context.headers['content-type'] = 'application/wasm';
+ }
return next();
}
],
diff --git a/web/src/test/manual/build.sh b/web/src/test/manual/build.sh
index 046cefe1520..6edae31b08f 100755
--- a/web/src/test/manual/build.sh
+++ b/web/src/test/manual/build.sh
@@ -57,4 +57,4 @@ function do_copy() {
}
builder_run_action clean rm -rf "$KEYMAN_ROOT/$DEST"
-builder_run_action build do_copy
\ No newline at end of file
+builder_run_action build do_copy
diff --git a/web/src/tools/testing/test-server/index.cjs b/web/src/tools/testing/test-server/index.cjs
new file mode 100644
index 00000000000..80237e9f756
--- /dev/null
+++ b/web/src/tools/testing/test-server/index.cjs
@@ -0,0 +1,12 @@
+const express = require('express')
+const path = require('path')
+const app = express()
+const port = 3000
+
+app.use(express.static(path.join(__dirname, '../../../../')))
+
+app.listen(port, () => {
+ console.log(`Keyman test app listening on port ${port}`)
+})
+
+
From f6c822906461db162da31840b8f5fba3f34708f1 Mon Sep 17 00:00:00 2001
From: Eberhard Beilharz
Date: Wed, 21 Aug 2024 09:20:10 +0200
Subject: [PATCH 003/360] feat(web): don't use deprecated emcc parameter
---
core/src/meson.build | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/core/src/meson.build b/core/src/meson.build
index 564472b7980..25499ab600b 100644
--- a/core/src/meson.build
+++ b/core/src/meson.build
@@ -135,7 +135,7 @@ mock_files = files(
)
if cpp_compiler.get_id() == 'emscripten'
- host_links = ['--whole-archive', '-sALLOW_MEMORY_GROWTH=1', '-sMODULARIZE=1', '-sEXPORT_ES6', '-sENVIRONMENT=webview', '--embind-emit-tsd', 'core-interface.d.ts', '-sERROR_ON_UNDEFINED_SYMBOLS=0']
+ host_links = ['--whole-archive', '-sALLOW_MEMORY_GROWTH=1', '-sMODULARIZE=1', '-sEXPORT_ES6', '-sENVIRONMENT=webview', '--emit-tsd', 'core-interface.d.ts', '-sERROR_ON_UNDEFINED_SYMBOLS=0']
if cpp_compiler.version().version_compare('>=3.1.44')
# emscripten 3.1.44 removes .asm object and so we need to export `wasmExports`
From 3912bf4c1390cc1dbee31c1a5edf0fd61c227362 Mon Sep 17 00:00:00 2001
From: Marc Durdin
Date: Wed, 21 Aug 2024 12:25:51 +0200
Subject: [PATCH 004/360] chore(developer): support for emscripten 3.1.64
---
developer/src/kmcmplib/src/CompilerInterfacesWasm.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/developer/src/kmcmplib/src/CompilerInterfacesWasm.cpp b/developer/src/kmcmplib/src/CompilerInterfacesWasm.cpp
index a857b41dce4..3848d89c4d2 100644
--- a/developer/src/kmcmplib/src/CompilerInterfacesWasm.cpp
+++ b/developer/src/kmcmplib/src/CompilerInterfacesWasm.cpp
@@ -75,8 +75,8 @@ struct BindingType> {
using ValBinding = BindingType;
using WireType = ValBinding::WireType;
- static WireType toWireType(const std::vector &vec) {
- return ValBinding::toWireType(val::array(vec));
+ static WireType toWireType(const std::vector &vec, rvp::default_tag) {
+ return ValBinding::toWireType(val::array(vec), rvp::default_tag{});
}
static std::vector fromWireType(WireType value) {
From e015072213a01c58482a15bfd19ee2c8563f0425 Mon Sep 17 00:00:00 2001
From: Marc Durdin
Date: Thu, 22 Aug 2024 15:58:53 +0200
Subject: [PATCH 005/360] chore(web): add utils dependency to core-processor
(es-bundling)
---
web/src/engine/core-processor/build.sh | 1 +
1 file changed, 1 insertion(+)
diff --git a/web/src/engine/core-processor/build.sh b/web/src/engine/core-processor/build.sh
index 14379c87654..4d0818f578b 100755
--- a/web/src/engine/core-processor/build.sh
+++ b/web/src/engine/core-processor/build.sh
@@ -15,6 +15,7 @@ SUBPROJECT_NAME=engine/core-processor
builder_describe "Keyman Core WASM integration" \
"@/core:wasm" \
+ "@/common/web/utils" \
"clean" \
"configure" \
"build" \
From c021999f9e7a9a11aa07bbd5247f1eba4f7c879d Mon Sep 17 00:00:00 2001
From: Marc Durdin
Date: Fri, 23 Aug 2024 07:11:41 +0200
Subject: [PATCH 006/360] chore(common): use `npm install` and set min ver for
emsdk
---
docs/minimum-versions.md | 3 +--
resources/build/minimum-versions.inc.sh | 2 +-
resources/locate_emscripten.inc.sh | 2 ++
3 files changed, 4 insertions(+), 3 deletions(-)
diff --git a/docs/minimum-versions.md b/docs/minimum-versions.md
index a9a2ead26b9..4b308450690 100644
--- a/docs/minimum-versions.md
+++ b/docs/minimum-versions.md
@@ -49,7 +49,6 @@ https://help.keyman.com/developer/engine/android/latest-version/
| KEYMAN Variable | Value |
|-----------------------------------|--------------|
| KEYMAN_DEFAULT_VERSION_UBUNTU_CONTAINER | noble |
-| KEYMAN_MAX_VERSION_EMSCRIPTEN | 3.1.58 |
| KEYMAN_MIN_TARGET_VERSION_ANDROID | 5 |
| KEYMAN_MIN_TARGET_VERSION_CHROME | 95.0 |
| KEYMAN_MIN_TARGET_VERSION_IOS | 12.2 |
@@ -58,7 +57,7 @@ https://help.keyman.com/developer/engine/android/latest-version/
| KEYMAN_MIN_TARGET_VERSION_WINDOWS | 10 |
| KEYMAN_MIN_VERSION_ANDROID_SDK | 21 |
| KEYMAN_MIN_VERSION_CPP | 17 |
-| KEYMAN_MIN_VERSION_EMSCRIPTEN | 3.1.44 |
+| KEYMAN_MIN_VERSION_EMSCRIPTEN | 3.1.64 |
| KEYMAN_MIN_VERSION_MESON | 1.0.0 |
| KEYMAN_MIN_VERSION_NODE_MAJOR | 20 |
| KEYMAN_MIN_VERSION_NPM | 10.5.1 |
diff --git a/resources/build/minimum-versions.inc.sh b/resources/build/minimum-versions.inc.sh
index 97f25094067..4dcf5e64f1e 100644
--- a/resources/build/minimum-versions.inc.sh
+++ b/resources/build/minimum-versions.inc.sh
@@ -20,7 +20,7 @@ KEYMAN_MIN_TARGET_VERSION_CHROME=95.0 # Final version that runs on Andro
# Dependency versions
KEYMAN_MIN_VERSION_NODE_MAJOR=20 # node version source of truth is /package.json:/engines/node
KEYMAN_MIN_VERSION_NPM=10.5.1 # 10.5.0 has bug, discussed in #10350
-KEYMAN_MIN_VERSION_EMSCRIPTEN=3.1.58 # Use KEYMAN_USE_EMSDK to automatically update to this version
+KEYMAN_MIN_VERSION_EMSCRIPTEN=3.1.64 # Use KEYMAN_USE_EMSDK to automatically update to this version
KEYMAN_MIN_VERSION_VISUAL_STUDIO=2019
KEYMAN_MIN_VERSION_MESON=1.0.0
diff --git a/resources/locate_emscripten.inc.sh b/resources/locate_emscripten.inc.sh
index f678e8449e6..3d470f75772 100644
--- a/resources/locate_emscripten.inc.sh
+++ b/resources/locate_emscripten.inc.sh
@@ -69,5 +69,7 @@ _select_emscripten_version_with_emsdk() {
git pull
./emsdk install "$KEYMAN_MIN_VERSION_EMSCRIPTEN"
./emsdk activate "$KEYMAN_MIN_VERSION_EMSCRIPTEN"
+ cd upstream/emscripten
+ npm install
popd > /dev/null
}
From 13f052f766931db3c3f3225a76c41d5c22b7fcb1 Mon Sep 17 00:00:00 2001
From: Eberhard Beilharz
Date: Mon, 26 Aug 2024 14:49:54 +0700
Subject: [PATCH 007/360] Apply suggestions from code review
Co-authored-by: Marc Durdin
---
web/src/app/browser/build.sh | 2 +-
web/src/app/webview/build.sh | 2 +-
web/src/engine/core-processor/build.sh | 6 ++----
3 files changed, 4 insertions(+), 6 deletions(-)
diff --git a/web/src/app/browser/build.sh b/web/src/app/browser/build.sh
index bf135637d1c..b5402d29218 100755
--- a/web/src/app/browser/build.sh
+++ b/web/src/app/browser/build.sh
@@ -73,7 +73,7 @@ compile_and_copy() {
mkdir -p "$KEYMAN_ROOT/web/build/app/resources/osk"
cp -R "$KEYMAN_ROOT/web/src/resources/osk/." "$KEYMAN_ROOT/web/build/app/resources/osk/"
- # Copy the WASM host
+ # Copy Keyman Core build artifacts for local reference
cp "${KEYMAN_ROOT}/web/build/engine/core-processor/obj/import/core/"core.{js,wasm} "${KEYMAN_ROOT}/web/build/app/browser/debug/"
cp "${KEYMAN_ROOT}/web/build/engine/core-processor/obj/import/core/"core.{js,wasm} "${KEYMAN_ROOT}/web/build/app/browser/release/"
diff --git a/web/src/app/webview/build.sh b/web/src/app/webview/build.sh
index 13e6fe2ddf1..4a2bc2fb544 100755
--- a/web/src/app/webview/build.sh
+++ b/web/src/app/webview/build.sh
@@ -58,7 +58,7 @@ compile_and_copy() {
mkdir -p "$KEYMAN_ROOT/web/build/app/resources/osk"
cp -R "$KEYMAN_ROOT/web/src/resources/osk/." "$KEYMAN_ROOT/web/build/app/resources/osk/"
- # Copy the WASM host
+ # Copy Keyman Core build artifacts for local reference
cp "${KEYMAN_ROOT}/web/build/engine/core-processor/obj/import/core/"core.{js,wasm} "${KEYMAN_ROOT}/web/build/app/webview/debug/"
cp "${KEYMAN_ROOT}/web/build/engine/core-processor/obj/import/core/"core.{js,wasm} "${KEYMAN_ROOT}/web/build/app/webview/release/"
diff --git a/web/src/engine/core-processor/build.sh b/web/src/engine/core-processor/build.sh
index 4d0818f578b..cee72fea011 100755
--- a/web/src/engine/core-processor/build.sh
+++ b/web/src/engine/core-processor/build.sh
@@ -39,7 +39,8 @@ do_configure() {
verify_npm_setup
mkdir -p "src/import/core/"
- # we don't need this file here, but it's nice to have for reference and auto-completion
+ # we don't need this file for release builds, but it's nice to have
+ # for reference and auto-completion
cp "${KEYMAN_ROOT}/core/build/wasm/${BUILDER_CONFIGURATION}/src/core-interface.d.ts" "src/import/core/"
}
@@ -60,6 +61,3 @@ do_build () {
builder_run_action clean do_clean
builder_run_action configure do_configure
builder_run_action build do_build
-
-# No headless tests for this child project. Currently, DOM-based unit &
-# integrated tests are run solely by the top-level $KEYMAN_ROOT/web project.
From eb33c53b9cb5dc3b7ea1828e75004a0f1bb55ec7 Mon Sep 17 00:00:00 2001
From: Eberhard Beilharz
Date: Mon, 26 Aug 2024 15:10:43 +0700
Subject: [PATCH 008/360] chore(core): address code review comments
---
core/src/meson.build | 22 +++++++------------
core/src/wasm.cpp | 50 +-------------------------------------------
package-lock.json | 8 +++----
3 files changed, 13 insertions(+), 67 deletions(-)
diff --git a/core/src/meson.build b/core/src/meson.build
index 25499ab600b..74acf6e54b9 100644
--- a/core/src/meson.build
+++ b/core/src/meson.build
@@ -135,18 +135,11 @@ mock_files = files(
)
if cpp_compiler.get_id() == 'emscripten'
- host_links = ['--whole-archive', '-sALLOW_MEMORY_GROWTH=1', '-sMODULARIZE=1', '-sEXPORT_ES6', '-sENVIRONMENT=webview', '--emit-tsd', 'core-interface.d.ts', '-sERROR_ON_UNDEFINED_SYMBOLS=0']
-
- if cpp_compiler.version().version_compare('>=3.1.44')
- # emscripten 3.1.44 removes .asm object and so we need to export `wasmExports`
- # #9375; https://github.com/emscripten-core/emscripten/blob/main/ChangeLog.md#3144---072523
- links += ['-sEXPORTED_RUNTIME_METHODS=[\'UTF8ToString\',\'stringToNewUTF8\',\'wasmExports\']']
- else
- # emscripten < 3.1.44 does not include `wasmExports`
- links += ['-sEXPORTED_RUNTIME_METHODS=[\'UTF8ToString\',\'stringToNewUTF8\']']
- endif
+ host_links = ['--whole-archive', '-sALLOW_MEMORY_GROWTH=1', '-sMODULARIZE=1',
+ '-sEXPORT_ES6', '-sENVIRONMENT=webview',
+ '--emit-tsd', 'core-interface.d.ts', '-sERROR_ON_UNDEFINED_SYMBOLS=0']
- links += [
+ links += ['-sEXPORTED_RUNTIME_METHODS=[\'UTF8ToString\',\'stringToNewUTF8\',\'wasmExports\']',
# Forcing inclusion of debug symbols
'-g', '-Wlimited-postlink-optimizations', '--bind']
endif
@@ -188,13 +181,14 @@ if cpp_compiler.get_id() == 'emscripten'
objects: lib.extract_all_objects(recursive: false))
if get_option('buildtype') == 'release'
+ # TODO: #12888
# Split debug symbols into separate wasm file for release builds only
# as the release symbols will be uploaded to sentry
- # custom_target('kmcmplib.wasm',
+ # custom_target('core.wasm',
# depends: host,
# input: host,
- # output: 'kmcmplib.wasm',
- # command: ['wasm-split', '@OUTDIR@/wasm-host.wasm', '-o', '@OUTPUT@', '--strip', '--debug-out=@OUTDIR@/kmcmplib.debug.wasm'],
+ # output: 'core.wasm',
+ # command: ['wasm-split', '@OUTDIR@/core.wasm', '-o', '@OUTPUT@', '--strip', '--debug-out=@OUTDIR@/core.debug.wasm'],
# build_by_default: true)
endif
endif
diff --git a/core/src/wasm.cpp b/core/src/wasm.cpp
index d9c4277a96b..90fc85cda88 100644
--- a/core/src/wasm.cpp
+++ b/core/src/wasm.cpp
@@ -28,55 +28,7 @@ EMSCRIPTEN_KEEPALIVE km_core_attr const & tmp_wasm_attributes() {
return engine_attrs;
}
-EMSCRIPTEN_BINDINGS(compiler_interface) {
-
- // emscripten::class_("WasmCallbackInterface")
- // .function("message", &WasmCallbackInterface::message, emscripten::pure_virtual())
- // .function("loadFile", &WasmCallbackInterface::loadFile, emscripten::pure_virtual())
- // .allow_subclass("WasmCallbackInterfaceWrapper")
- // ;
-
- // emscripten::class_("CompilerOptions")
- // .constructor<>()
- // .property("saveDebug", &KMCMP_COMPILER_OPTIONS::saveDebug)
- // .property("compilerWarningsAsErrors", &KMCMP_COMPILER_OPTIONS::compilerWarningsAsErrors)
- // .property("warnDeprecatedCode", &KMCMP_COMPILER_OPTIONS::warnDeprecatedCode)
- // .property("shouldAddCompilerVersion", &KMCMP_COMPILER_OPTIONS::shouldAddCompilerVersion)
- // .property("target", &KMCMP_COMPILER_OPTIONS::target)
- // ;
-
- // emscripten::class_("CompilerResult")
- // .constructor<>()
- // .property("result", &WASM_COMPILER_RESULT::result)
- // .property("kmx", &WASM_COMPILER_RESULT::kmx)
- // .property("kmxSize", &WASM_COMPILER_RESULT::kmxSize)
- // .property("extra", &WASM_COMPILER_RESULT::extra)
- // ;
-
- // emscripten::class_("CompilerResultMessage")
- // .constructor<>()
- // .property("errorCode", &KMCMP_COMPILER_RESULT_MESSAGE::errorCode)
- // .property("lineNumber", &KMCMP_COMPILER_RESULT_MESSAGE::lineNumber)
- // .property("columnNumber", &KMCMP_COMPILER_RESULT_MESSAGE::columnNumber)
- // .property("filename", &KMCMP_COMPILER_RESULT_MESSAGE::filename)
- // .property("parameters", &KMCMP_COMPILER_RESULT_MESSAGE::parameters)
- // ;
-
- // emscripten::class_("CompilerResultExtra")
- // .constructor<>()
- // .property("targets", &KMCMP_COMPILER_RESULT_EXTRA::targets)
- // .property("kmnFilename", &KMCMP_COMPILER_RESULT_EXTRA::kmnFilename)
- // .property("kvksFilename", &KMCMP_COMPILER_RESULT_EXTRA::kvksFilename)
- // .property("displayMapFilename", &KMCMP_COMPILER_RESULT_EXTRA::displayMapFilename)
- // .property("stores", &KMCMP_COMPILER_RESULT_EXTRA::stores)
- // .property("groups", &KMCMP_COMPILER_RESULT_EXTRA::groups)
- // ;
-
- // emscripten::value_object("CompilerResultExtraStore")
- // .field("storeType", &KMCMP_COMPILER_RESULT_EXTRA_STORE::storeType)
- // .field("name", &KMCMP_COMPILER_RESULT_EXTRA_STORE::name)
- // .field("line", &KMCMP_COMPILER_RESULT_EXTRA_STORE::line)
- // ;
+EMSCRIPTEN_BINDINGS(core_interface) {
emscripten::value_object("km_core_attr")
.field("max_context", &km_core_attr::max_context)
diff --git a/package-lock.json b/package-lock.json
index c8451a75191..eaaffeea7e4 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -404,7 +404,7 @@
"eventemitter3": "^5.0.0",
"restructure": "^3.0.1",
"sax": ">=0.6.0",
- "semver": "^7.5.2",
+ "semver": "^7.5.4",
"xmlbuilder": "~11.0.0"
},
"devDependencies": {
@@ -1153,7 +1153,7 @@
"@keymanapp/keyman-version": "*",
"@keymanapp/kmc-kmn": "*",
"@keymanapp/ldml-keyboard-constants": "*",
- "semver": "^7.5.2"
+ "semver": "^7.5.4"
},
"devDependencies": {
"@keymanapp/developer-test-helpers": "*",
@@ -1942,7 +1942,7 @@
"open": "^8.4.0",
"restructure": "^3.0.1",
"sax": ">=0.6.0",
- "semver": "^7.5.2",
+ "semver": "^7.5.4",
"ws": "^8.17.1",
"xmlbuilder": "~11.0.0"
},
@@ -14631,7 +14631,7 @@
"devDependencies": {
"@types/semver": "^7.1.0",
"@types/yargs": "^17.0.26",
- "semver": "^7.5.2"
+ "semver": "^7.5.4"
}
},
"resources/build/version/node_modules/ansi-regex": {
From 1deaa323ada28a6212da06d76ef886abc9c1335c Mon Sep 17 00:00:00 2001
From: Eberhard Beilharz
Date: Fri, 30 Aug 2024 14:44:40 +0700
Subject: [PATCH 009/360] feat(core): spec out API extensions
- loading a keyboard from a BLOB
- getting the on-screen keyboard layout from Core. This is an internal-
only API because of it's use of C++.
Part-of: #11293
Part-of: #8093
---
core/include/keyman/keyman_core_api.h | 45 +++++++
core/src/layout.hpp | 185 ++++++++++++++++++++++++++
core/src/mock/mock_processor.cpp | 2 +-
3 files changed, 231 insertions(+), 1 deletion(-)
create mode 100644 core/src/layout.hpp
diff --git a/core/include/keyman/keyman_core_api.h b/core/include/keyman/keyman_core_api.h
index 328965e8761..341b5fa1fad 100644
--- a/core/include/keyman/keyman_core_api.h
+++ b/core/include/keyman/keyman_core_api.h
@@ -1140,6 +1140,51 @@ km_core_keyboard_load(km_core_path_name kb_path,
-------------------------------------------------------------------------------
+# km_core_keyboard_load_from_blob()
+
+## Description
+
+Parse and load keyboard from the supplied blob and a pointer to the loaded keyboard
+into the out paramter.
+
+## Specification
+
+```c */
+KMN_API
+km_core_status km_core_keyboard_load_from_blob(void* blob, km_core_keyboard** keyboard);
+
+/*
+```
+
+## Parameters
+
+`blob`
+: a byte array containing the content of a KMX/KMX+ file
+
+`keyboard`
+: A pointer to result variable: A pointer to the opaque keyboard
+ object returned by the Processor. This memory must be freed with a
+ call to [km_core_keyboard_dispose].
+
+## Returns
+
+`KM_CORE_STATUS_OK`
+: On success.
+
+`KM_CORE_STATUS_NO_MEM`
+: In the event an internal memory allocation fails.
+
+`KM_CORE_STATUS_IO_ERROR`
+: In the event the keyboard file is unparseable for any reason
+
+`KM_CORE_STATUS_INVALID_ARGUMENT`
+: In the event `keyboard` is null.
+
+`KM_CORE_STATUS_OS_ERROR`
+: Bit 31 (high bit) set, bits 0-30 are an OS-specific error code.
+
+-------------------------------------------------------------------------------
+
# km_core_keyboard_dispose()
## Description
diff --git a/core/src/layout.hpp b/core/src/layout.hpp
new file mode 100644
index 00000000000..54577919be3
--- /dev/null
+++ b/core/src/layout.hpp
@@ -0,0 +1,185 @@
+/*
+ * Keyman is copyright (C) SIL International. MIT License.
+ *
+ * Keyman Keyboard Processor API - On-Screen Keyboard Layout Interfaces
+ */
+
+#pragma once
+
+#include
+#include
+ KeymanWeb Sample Page - Core Testing
+ This page is designed to stress-test the loading a KMX keyboard from blob.
+ Be sure to reference the developer console for additional feedback.
+
+
+
+