diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 080954455c..804ae71d17 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -426,6 +426,36 @@ jobs: - name: Run C++ IDL Tests run: ./integration_tests/idl_tests/run_cpp_tests.sh + javascript_xlang: + name: JavaScript Xlang Test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - name: Use Node.js 20.x + uses: actions/setup-node@v4 + with: + node-version: 20.x + - name: Set up JDK 21 + uses: actions/setup-java@v4 + with: + java-version: 21 + distribution: "temurin" + - name: Cache Maven local repository + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + - name: Run JavaScript Xlang Test + env: + FORY_JavaScript_JAVA_CI: "1" + run: | + cd java + mvn -T16 --no-transfer-progress clean install -DskipTests + cd fory-core + mvn -T16 --no-transfer-progress test -Dtest=org.apache.fory.xlang.JavaScriptXlangTest + cpp_examples: name: C++ Examples runs-on: ubuntu-latest diff --git a/java/fory-core/src/test/java/org/apache/fory/xlang/JavaScriptXlangTest.java b/java/fory-core/src/test/java/org/apache/fory/xlang/JavaScriptXlangTest.java new file mode 100644 index 0000000000..bddfaecf2a --- /dev/null +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/JavaScriptXlangTest.java @@ -0,0 +1,283 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.xlang; + +import com.google.common.collect.ImmutableMap; +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.testng.SkipException; +import org.testng.annotations.Test; + +/** Executes cross-language tests against the Rust implementation. */ +@Test +public class JavaScriptXlangTest extends XlangTestBase { + private static final String NODE_EXECUTABLE = "npx"; + private static final String NODE_MODULE = "crossLanguage.test.ts"; + + private static final List RUST_BASE_COMMAND = + Arrays.asList(NODE_EXECUTABLE, "jest", NODE_MODULE, "-t", "caseName"); + + private static final int NODE_TESTCASE_INDEX = 4; + + @Override + protected void ensurePeerReady() { + String enabled = System.getenv("FORY_JAVASCRIPT_JAVA_CI"); + if (!"1".equals(enabled)) { + throw new SkipException("Skipping JavaScriptXlangTest: FORY_JAVASCRIPT_JAVA_CI not set to 1"); + } + boolean nodeInstalled = true; + try { + Process process = new ProcessBuilder("node", "--version").start(); + int exitCode = process.waitFor(); + if (exitCode != 0) { + nodeInstalled = false; + } + } catch (IOException | InterruptedException e) { + nodeInstalled = false; + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + } + if (!nodeInstalled) { + throw new SkipException("Skipping JavaScriptXlangTest: nodejs not installed"); + } + } + + @Override + protected CommandContext buildCommandContext(String caseName, Path dataFile) { + List command = new ArrayList<>(RUST_BASE_COMMAND); + // jest use regexp to match the castName. And '$' at end to ignore matching error. + command.set(NODE_TESTCASE_INDEX, caseName + "$"); + ImmutableMap env = envBuilder(dataFile).build(); + return new CommandContext(command, env, new File("../../javascript")); + } + + // ============================================================================ + // Test methods - duplicated from XlangTestBase for Maven Surefire discovery + // ============================================================================ + + @Test + public void testBuffer() throws java.io.IOException { + super.testBuffer(); + } + + @Test + public void testBufferVar() throws java.io.IOException { + super.testBufferVar(); + } + + @Test + public void testMurmurHash3() throws java.io.IOException { + super.testMurmurHash3(); + } + + @Test + public void testStringSerializer() throws Exception { + super.testStringSerializer(); + } + + @Test + public void testCrossLanguageSerializer() throws Exception { + super.testCrossLanguageSerializer(); + } + + @Test(dataProvider = "enableCodegen") + public void testSimpleStruct(boolean enableCodegen) throws java.io.IOException { + super.testSimpleStruct(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testSimpleNamedStruct(boolean enableCodegen) throws java.io.IOException { + super.testSimpleNamedStruct(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testList(boolean enableCodegen) throws java.io.IOException { + super.testList(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testMap(boolean enableCodegen) throws java.io.IOException { + super.testMap(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testInteger(boolean enableCodegen) throws java.io.IOException { + super.testInteger(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testItem(boolean enableCodegen) throws java.io.IOException { + super.testItem(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testColor(boolean enableCodegen) throws java.io.IOException { + super.testColor(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testStructWithList(boolean enableCodegen) throws java.io.IOException { + super.testStructWithList(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testStructWithMap(boolean enableCodegen) throws java.io.IOException { + super.testStructWithMap(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testSkipIdCustom(boolean enableCodegen) throws java.io.IOException { + super.testSkipIdCustom(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testSkipNameCustom(boolean enableCodegen) throws java.io.IOException { + super.testSkipNameCustom(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testConsistentNamed(boolean enableCodegen) throws java.io.IOException { + super.testConsistentNamed(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testStructVersionCheck(boolean enableCodegen) throws java.io.IOException { + super.testStructVersionCheck(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testPolymorphicList(boolean enableCodegen) throws java.io.IOException { + super.testPolymorphicList(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testPolymorphicMap(boolean enableCodegen) throws java.io.IOException { + super.testPolymorphicMap(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testOneStringFieldSchemaConsistent(boolean enableCodegen) throws java.io.IOException { + super.testOneStringFieldSchemaConsistent(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testOneStringFieldCompatible(boolean enableCodegen) throws java.io.IOException { + super.testOneStringFieldCompatible(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testTwoStringFieldCompatible(boolean enableCodegen) throws java.io.IOException { + super.testTwoStringFieldCompatible(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testSchemaEvolutionCompatible(boolean enableCodegen) throws java.io.IOException { + super.testSchemaEvolutionCompatible(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testOneEnumFieldSchemaConsistent(boolean enableCodegen) throws java.io.IOException { + super.testOneEnumFieldSchemaConsistent(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testOneEnumFieldCompatible(boolean enableCodegen) throws java.io.IOException { + super.testOneEnumFieldCompatible(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testTwoEnumFieldCompatible(boolean enableCodegen) throws java.io.IOException { + super.testTwoEnumFieldCompatible(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testEnumSchemaEvolutionCompatible(boolean enableCodegen) throws java.io.IOException { + super.testEnumSchemaEvolutionCompatible(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testNullableFieldSchemaConsistentNotNull(boolean enableCodegen) + throws java.io.IOException { + super.testNullableFieldSchemaConsistentNotNull(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testNullableFieldSchemaConsistentNull(boolean enableCodegen) + throws java.io.IOException { + super.testNullableFieldSchemaConsistentNull(enableCodegen); + } + + @Override + @Test(dataProvider = "enableCodegen") + public void testNullableFieldCompatibleNotNull(boolean enableCodegen) throws java.io.IOException { + super.testNullableFieldCompatibleNotNull(enableCodegen); + } + + @Override + @Test(dataProvider = "enableCodegen") + public void testNullableFieldCompatibleNull(boolean enableCodegen) throws java.io.IOException { + super.testNullableFieldCompatibleNull(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testUnionXlang(boolean enableCodegen) throws java.io.IOException { + super.testUnionXlang(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testRefSchemaConsistent(boolean enableCodegen) throws java.io.IOException { + super.testRefSchemaConsistent(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testRefCompatible(boolean enableCodegen) throws java.io.IOException { + super.testRefCompatible(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testCircularRefSchemaConsistent(boolean enableCodegen) throws java.io.IOException { + super.testCircularRefSchemaConsistent(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testCircularRefCompatible(boolean enableCodegen) throws java.io.IOException { + super.testCircularRefCompatible(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testUnsignedSchemaConsistent(boolean enableCodegen) throws java.io.IOException { + super.testUnsignedSchemaConsistent(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testUnsignedSchemaConsistentSimple(boolean enableCodegen) throws java.io.IOException { + super.testUnsignedSchemaConsistentSimple(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testUnsignedSchemaCompatible(boolean enableCodegen) throws java.io.IOException { + super.testUnsignedSchemaCompatible(enableCodegen); + } +} diff --git a/javascript/jest.config.js b/javascript/jest.config.js index 3605969257..2dbfe8a426 100644 --- a/javascript/jest.config.js +++ b/javascript/jest.config.js @@ -29,6 +29,7 @@ module.exports = { "!**/build/**", "!packages/fory/lib/murmurHash3.ts" ], + testPathIgnorePatterns : !process.env["DATA_FILE"] ? ["test/crossLanguage.test.ts"] : [], transform: { '\\.ts$': ['ts-jest', { tsconfig: { diff --git a/javascript/packages/fory/index.ts b/javascript/packages/fory/index.ts index 3200c27ca9..ad7f7fc924 100644 --- a/javascript/packages/fory/index.ts +++ b/javascript/packages/fory/index.ts @@ -26,6 +26,7 @@ import { import { Serializer, InternalSerializerType, Mode } from "./lib/type"; import Fory from "./lib/fory"; import { BinaryReader } from "./lib/reader"; +import { BinaryWriter } from "./lib/writer"; export { Serializer, @@ -35,6 +36,7 @@ export { StructTypeInfo, Type, Mode, + BinaryWriter, BinaryReader, }; diff --git a/javascript/packages/fory/lib/classResolver.ts b/javascript/packages/fory/lib/classResolver.ts index c646d64420..806c12662d 100644 --- a/javascript/packages/fory/lib/classResolver.ts +++ b/javascript/packages/fory/lib/classResolver.ts @@ -87,6 +87,14 @@ export default class ClassResolver { this.setSerializer = this.getSerializerById((TypeId.SET)); this.arraySerializer = this.getSerializerById((TypeId.ARRAY)); this.mapSerializer = this.getSerializerById((TypeId.MAP)); + this.uint8ArraySerializer = this.getSerializerById(TypeId.UINT8_ARRAY); + this.uint16ArraySerializer = this.getSerializerById(TypeId.UINT16_ARRAY); + this.uint32ArraySerializer = this.getSerializerById(TypeId.UINT32_ARRAY); + this.uint64ArraySerializer = this.getSerializerById(TypeId.UINT64_ARRAY); + this.int8ArraySerializer = this.getSerializerById(TypeId.INT8_ARRAY); + this.int16ArraySerializer = this.getSerializerById(TypeId.INT16_ARRAY); + this.int32ArraySerializer = this.getSerializerById(TypeId.INT32_ARRAY); + this.int64ArraySerializer = this.getSerializerById(TypeId.INT64_ARRAY); } private numberSerializer: null | Serializer = null; @@ -97,6 +105,14 @@ export default class ClassResolver { private setSerializer: null | Serializer = null; private arraySerializer: null | Serializer = null; private mapSerializer: null | Serializer = null; + private uint8ArraySerializer: null | Serializer = null; + private uint16ArraySerializer: null | Serializer = null; + private uint32ArraySerializer: null | Serializer = null; + private uint64ArraySerializer: null | Serializer = null; + private int8ArraySerializer: null | Serializer = null; + private int16ArraySerializer: null | Serializer = null; + private int32ArraySerializer: null | Serializer = null; + private int64ArraySerializer: null | Serializer = null; constructor(private fory: Fory) { } @@ -178,6 +194,38 @@ export default class ClassResolver { return this.stringSerializer; } + if (v instanceof Uint8Array) { + return this.uint8ArraySerializer; + } + + if (v instanceof Uint16Array) { + return this.uint16ArraySerializer; + } + + if (v instanceof Uint32Array) { + return this.uint32ArraySerializer; + } + + if (v instanceof BigUint64Array) { + return this.uint64ArraySerializer; + } + + if (v instanceof Int8Array) { + return this.int8ArraySerializer; + } + + if (v instanceof Int16Array) { + return this.int16ArraySerializer; + } + + if (v instanceof Int32Array) { + return this.int32ArraySerializer; + } + + if (v instanceof BigInt64Array) { + return this.int64ArraySerializer; + } + if (Array.isArray(v)) { return this.arraySerializer; } diff --git a/javascript/packages/fory/lib/gen/typedArray.ts b/javascript/packages/fory/lib/gen/typedArray.ts index ed0db69f20..11ffa9d1ed 100644 --- a/javascript/packages/fory/lib/gen/typedArray.ts +++ b/javascript/packages/fory/lib/gen/typedArray.ts @@ -77,6 +77,10 @@ CodegenRegistry.register(InternalSerializerType.INT8_ARRAY, build(Type.int8())); CodegenRegistry.register(InternalSerializerType.INT16_ARRAY, build(Type.int16())); CodegenRegistry.register(InternalSerializerType.INT32_ARRAY, build(Type.int32())); CodegenRegistry.register(InternalSerializerType.INT64_ARRAY, build(Type.int64())); +CodegenRegistry.register(InternalSerializerType.UINT8_ARRAY, build(Type.uint8())); +CodegenRegistry.register(InternalSerializerType.UINT16_ARRAY, build(Type.uint16())); +CodegenRegistry.register(InternalSerializerType.UINT32_ARRAY, build(Type.uint32())); +CodegenRegistry.register(InternalSerializerType.UINT64_ARRAY, build(Type.uint64())); CodegenRegistry.register(InternalSerializerType.FLOAT16_ARRAY, build(Type.float16())); CodegenRegistry.register(InternalSerializerType.FLOAT32_ARRAY, build(Type.float32())); CodegenRegistry.register(InternalSerializerType.FLOAT64_ARRAY, build(Type.float64())); diff --git a/javascript/packages/fory/lib/type.ts b/javascript/packages/fory/lib/type.ts index 185b83d981..ebbc3a7798 100644 --- a/javascript/packages/fory/lib/type.ts +++ b/javascript/packages/fory/lib/type.ts @@ -167,6 +167,10 @@ export enum InternalSerializerType { INT16_ARRAY, INT32_ARRAY, INT64_ARRAY, + UINT8_ARRAY, + UINT16_ARRAY, + UINT32_ARRAY, + UINT64_ARRAY, FLOAT16_ARRAY, FLOAT32_ARRAY, FLOAT64_ARRAY, diff --git a/javascript/packages/fory/lib/typeInfo.ts b/javascript/packages/fory/lib/typeInfo.ts index 6a73d88e03..74d63fa856 100644 --- a/javascript/packages/fory/lib/typeInfo.ts +++ b/javascript/packages/fory/lib/typeInfo.ts @@ -707,6 +707,34 @@ export const Type = { ); }, + uint8Array() { + return TypeInfo.fromNonParam( + InternalSerializerType.UINT8_ARRAY as const, + (TypeId.INT8_ARRAY), + + ); + }, + uint16Array() { + return TypeInfo.fromNonParam( + InternalSerializerType.UINT16_ARRAY as const, + (TypeId.INT16_ARRAY), + + ); + }, + uint32Array() { + return TypeInfo.fromNonParam( + InternalSerializerType.UINT32_ARRAY as const, + (TypeId.UINT32_ARRAY), + + ); + }, + uint64Array() { + return TypeInfo.fromNonParam( + InternalSerializerType.UINT64_ARRAY as const, + (TypeId.INT64_ARRAY), + + ); + }, float16Array() { return TypeInfo.fromNonParam( InternalSerializerType.FLOAT16_ARRAY as const, diff --git a/javascript/packages/fory/lib/writer/index.ts b/javascript/packages/fory/lib/writer/index.ts index 295f576797..841f993991 100644 --- a/javascript/packages/fory/lib/writer/index.ts +++ b/javascript/packages/fory/lib/writer/index.ts @@ -80,6 +80,11 @@ export class BinaryWriter { this.reserved = 0; } + bool(bool: boolean) { + this.dataView.setUint8(this.cursor, bool ? 1 : 0); + this.cursor++; + } + uint8(v: number) { this.dataView.setUint8(this.cursor, v); this.cursor++; diff --git a/javascript/test/crossLanguage.test.ts b/javascript/test/crossLanguage.test.ts new file mode 100644 index 0000000000..8ad104d9a2 --- /dev/null +++ b/javascript/test/crossLanguage.test.ts @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import Fory, { + TypeInfo, + InternalSerializerType, + BinaryWriter, +} from "../packages/fory/index"; +import { describe, expect, test } from "@jest/globals"; +import * as fs from "node:fs"; + +const Byte = { + MAX_VALUE: 127, + MIN_VALUE: -128, +} + +const Short = { + MAX_VALUE: 32767, + MIN_VALUE: -32768, +} + +const Integer = { + MAX_VALUE: 2147483647, + MIN_VALUE: -2147483648, +} + +const Long = { + MAX_VALUE: BigInt("9223372036854775807"), + MIN_VALUE: BigInt("-9223372036854775808"), +} + +describe("bool", () => { + const dataFile = process.env["DATA_FILE"]; + if (!dataFile) { + return; + } + function writeToFile(buffer: Buffer) { + fs.writeFileSync(dataFile!, buffer); + } + const content = fs.readFileSync(dataFile); + + test("test_buffer", () => { + const buffer = new BinaryWriter(); + buffer.reserve(32); + buffer.bool(true); + buffer.uint8(Byte.MAX_VALUE); + buffer.int16(Short.MAX_VALUE); + buffer.int32(Integer.MAX_VALUE); + buffer.int64(Long.MAX_VALUE); + buffer.float32(-1.1); + buffer.float64(-1.1); + buffer.varUInt32(100); + const bytes = ['a'.charCodeAt(0), 'b'.charCodeAt(0)]; + buffer.int32(bytes.length); + buffer.buffer(new Uint8Array(bytes)); + writeToFile(buffer.dump() as Buffer); + }); + test("test_buffer_var", () => { + // todo + }); + test("test_murmurhash3", () => { + // todo + }); + test("test_string_serializer", () => { + // todo + }); + test("test_cross_language_serializer", () => { + // todo + }); + test("test_simple_struct", () => { + // todo + }); + test("test_named_simple_struct", () => { + // todo + }); + test("test_list", () => { + // todo + }); + test("test_map", () => { + // todo + }); + test("test_integer", () => { + // todo + }); + test("test_item", () => { + // todo + }); + test("test_color", () => { + // todo + }); + test("test_struct_with_list", () => { + // todo + }); + test("test_struct_with_map", () => { + // todo + }); + test("test_skip_id_custom", () => { + // todo + }); + test("test_skip_name_custom", () => { + // todo + }); + test("test_consistent_named", () => { + // todo + }); + test("test_struct_version_check", () => { + // todo + }); + test("test_polymorphic_list", () => { + // todo + }); + test("test_polymorphic_map", () => { + // todo + }); + test("test_one_string_field_schema", () => { + // todo + }); + test("test_one_string_field_compatible", () => { + // todo + }); + test("test_two_string_field_compatible", () => { + // todo + }); + test("test_schema_evolution_compatible", () => { + // todo + }); + test("test_one_enum_field_schema", () => { + // todo + }); + test("test_one_enum_field_compatible", () => { + // todo + }); + test("test_two_enum_field_compatible", () => { + // todo + }); + test("test_enum_schema_evolution_compatible", () => { + // todo + }); + test("test_nullable_field_schema_consistent_not_null", () => { + // todo + }); + test("test_nullable_field_schema_consistent_null", () => { + // todo + }); + test("test_nullable_field_compatible_not_null", () => { + // todo + }); + test("test_nullable_field_compatible_null", () => { + // todo + }); + test("test_ref_schema_consistent", () => { + // todo + }); + test("test_ref_compatible", () => { + // todo + }); + test("test_circular_ref_schema_consistent", () => { + // todo + }); + test("test_circular_ref_compatible", () => { + // todo + }); + test("test_unsigned_schema_consistent_simple", () => { + // todo + }); + test("test_unsigned_schema_consistent", () => { + // todo + }); + test("test_unsigned_schema_compatible", () => { + // todo + }); +}); diff --git a/javascript/test/hps.test.ts b/javascript/test/hps.test.ts index cb040cebf3..e58faca125 100644 --- a/javascript/test/hps.test.ts +++ b/javascript/test/hps.test.ts @@ -17,7 +17,7 @@ * under the License. */ -import { BinaryReader } from '@apache-fory/fory'; +import { BinaryReader } from '../packages/fory/index'; import hps from '../packages/hps/index'; import { describe, expect, test } from '@jest/globals';