diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a5e6f47ec3..9d33bb765e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -478,6 +478,7 @@ jobs: run: | cd javascript npm install + npm run build - name: Run JavaScript Xlang Test env: FORY_JAVASCRIPT_JAVA_CI: "1" @@ -486,6 +487,8 @@ jobs: mvn -T16 --no-transfer-progress clean install -DskipTests -Dmaven.javadoc.skip=true -Dmaven.source.skip=true cd fory-core mvn --no-transfer-progress test -Dtest=org.apache.fory.xlang.JavaScriptXlangTest -DforkCount=0 + - name: Run JavaScript IDL Tests + run: ./integration_tests/idl_tests/run_javascript_tests.sh rust: name: Rust CI diff --git a/compiler/README.md b/compiler/README.md index c348248035..e4510475e7 100644 --- a/compiler/README.md +++ b/compiler/README.md @@ -4,7 +4,7 @@ The FDL compiler generates cross-language serialization code from schema definit ## Features -- **Multi-language code generation**: Java, Python, Go, Rust, C++, C# +- **Multi-language code generation**: Java, Python, Go, Rust, C++, C#, Javascript, and Swift - **Rich type system**: Primitives, enums, messages, lists, maps - **Cross-language serialization**: Generated code works seamlessly with Apache Fory - **Type ID and namespace support**: Both numeric IDs and name-based type registration @@ -64,16 +64,16 @@ message Cat [id=103] { foryc schema.fdl --output ./generated # Generate for specific languages -foryc schema.fdl --lang java,python,csharp --output ./generated +foryc schema.fdl --lang java,python,csharp,javascript --output ./generated # Override package name foryc schema.fdl --package myapp.models --output ./generated # Language-specific output directories (protoc-style) -foryc schema.fdl --java_out=./src/main/java --python_out=./python/src --csharp_out=./csharp/src/Generated +foryc schema.fdl --java_out=./src/main/java --python_out=./python/src --csharp_out=./csharp/src/Generated --javascript_out=./javascript # Combine with other options -foryc schema.fdl --java_out=./gen --go_out=./gen/go --csharp_out=./gen/csharp -I ./proto +foryc schema.fdl --java_out=./gen --go_out=./gen/go --csharp_out=./gen/csharp --javascript_out=./gen/js -I ./proto ``` ### 3. Use Generated Code @@ -185,19 +185,19 @@ message Config { ... } // Registered as "package.Config" ### Primitive Types -| FDL Type | Java | Python | Go | Rust | C++ | C# | -| ----------- | ----------- | ------------------- | ----------- | ----------------------- | ---------------------- | ---------------- | -| `bool` | `boolean` | `bool` | `bool` | `bool` | `bool` | `bool` | -| `int8` | `byte` | `pyfory.int8` | `int8` | `i8` | `int8_t` | `sbyte` | -| `int16` | `short` | `pyfory.int16` | `int16` | `i16` | `int16_t` | `short` | -| `int32` | `int` | `pyfory.int32` | `int32` | `i32` | `int32_t` | `int` | -| `int64` | `long` | `pyfory.int64` | `int64` | `i64` | `int64_t` | `long` | -| `float32` | `float` | `pyfory.float32` | `float32` | `f32` | `float` | `float` | -| `float64` | `double` | `pyfory.float64` | `float64` | `f64` | `double` | `double` | -| `string` | `String` | `str` | `string` | `String` | `std::string` | `string` | -| `bytes` | `byte[]` | `bytes` | `[]byte` | `Vec` | `std::vector` | `byte[]` | -| `date` | `LocalDate` | `datetime.date` | `time.Time` | `chrono::NaiveDate` | `fory::Date` | `DateOnly` | -| `timestamp` | `Instant` | `datetime.datetime` | `time.Time` | `chrono::NaiveDateTime` | `fory::Timestamp` | `DateTimeOffset` | +| FDL Type | Java | Python | Go | Rust | C++ | C# | JavaScript | +| ----------- | ----------- | ------------------- | ----------- | ----------------------- | ---------------------- | ---------------- | ------------------ | +| `bool` | `boolean` | `bool` | `bool` | `bool` | `bool` | `bool` | `boolean` | +| `int8` | `byte` | `pyfory.int8` | `int8` | `i8` | `int8_t` | `sbyte` | `number` | +| `int16` | `short` | `pyfory.int16` | `int16` | `i16` | `int16_t` | `short` | `number` | +| `int32` | `int` | `pyfory.int32` | `int32` | `i32` | `int32_t` | `int` | `number` | +| `int64` | `long` | `pyfory.int64` | `int64` | `i64` | `int64_t` | `long` | `bigint \| number` | +| `float32` | `float` | `pyfory.float32` | `float32` | `f32` | `float` | `float` | `number` | +| `float64` | `double` | `pyfory.float64` | `float64` | `f64` | `double` | `double` | `number` | +| `string` | `String` | `str` | `string` | `String` | `std::string` | `string` | `string` | +| `bytes` | `byte[]` | `bytes` | `[]byte` | `Vec` | `std::vector` | `byte[]` | `Uint8Array` | +| `date` | `LocalDate` | `datetime.date` | `time.Time` | `chrono::NaiveDate` | `fory::Date` | `DateOnly` | `Date` | +| `timestamp` | `Instant` | `datetime.datetime` | `time.Time` | `chrono::NaiveDateTime` | `fory::Timestamp` | `DateTimeOffset` | `Date` | ### Collection Types @@ -285,7 +285,8 @@ fory_compiler/ ├── go.py # Go struct generator ├── rust.py # Rust struct generator ├── cpp.py # C++ struct generator - └── csharp.py # C# class generator + ├── csharp.py # C# class generator + └── javascript.py # JavaScript interface generator ``` ### FDL Frontend @@ -422,6 +423,25 @@ cd integration_tests/idl_tests ./run_csharp_tests.sh ``` +### JavaScript + +Generates interfaces with: + +- `export interface` declarations for messages +- `export enum` declarations for enums +- Discriminated unions with case enums +- Registration helper function + +```javascript +export interface Cat { + friend?: Dog | undefined; + name?: string | undefined; + tags: string[]; + scores: Record; + lives: number; +} +``` + ## CLI Reference ``` @@ -431,7 +451,7 @@ Arguments: FILES FDL files to compile Options: - --lang TEXT Target languages (java,python,cpp,rust,go,csharp or "all") + --lang TEXT Target languages (java,python,cpp,rust,go,csharp,javascript or "all") Default: all --output, -o PATH Output directory Default: ./generated diff --git a/compiler/fory_compiler/cli.py b/compiler/fory_compiler/cli.py index 96325eaa8f..f558dc309b 100644 --- a/compiler/fory_compiler/cli.py +++ b/compiler/fory_compiler/cli.py @@ -264,7 +264,7 @@ def parse_args(args: Optional[List[str]] = None) -> argparse.Namespace: "--lang", type=str, default="all", - help="Comma-separated list of target languages (java,python,cpp,rust,go,csharp,swift). Default: all", + help="Comma-separated list of target languages (java,python,cpp,rust,go,csharp,javascript,swift). Default: all", ) parser.add_argument( @@ -343,6 +343,14 @@ def parse_args(args: Optional[List[str]] = None) -> argparse.Namespace: help="Generate C# code in DST_DIR", ) + parser.add_argument( + "--javascript_out", + type=Path, + default=None, + metavar="DST_DIR", + help="Generate JavaScript code in DST_DIR", + ) + parser.add_argument( "--swift_out", type=Path, @@ -650,6 +658,7 @@ def cmd_compile(args: argparse.Namespace) -> int: "go": args.go_out, "rust": args.rust_out, "csharp": args.csharp_out, + "javascript": args.javascript_out, "swift": args.swift_out, } diff --git a/compiler/fory_compiler/generators/__init__.py b/compiler/fory_compiler/generators/__init__.py index 47adaf6feb..83d33d6225 100644 --- a/compiler/fory_compiler/generators/__init__.py +++ b/compiler/fory_compiler/generators/__init__.py @@ -24,6 +24,7 @@ from fory_compiler.generators.rust import RustGenerator from fory_compiler.generators.go import GoGenerator from fory_compiler.generators.csharp import CSharpGenerator +from fory_compiler.generators.javascript import JavaScriptGenerator from fory_compiler.generators.swift import SwiftGenerator GENERATORS = { @@ -33,6 +34,7 @@ "rust": RustGenerator, "go": GoGenerator, "csharp": CSharpGenerator, + "javascript": JavaScriptGenerator, "swift": SwiftGenerator, } @@ -44,6 +46,7 @@ "RustGenerator", "GoGenerator", "CSharpGenerator", + "JavaScriptGenerator", "SwiftGenerator", "GENERATORS", ] diff --git a/compiler/fory_compiler/generators/javascript.py b/compiler/fory_compiler/generators/javascript.py new file mode 100644 index 0000000000..ac2229ae7f --- /dev/null +++ b/compiler/fory_compiler/generators/javascript.py @@ -0,0 +1,867 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""JavaScript/TypeScript code generator.""" + +from pathlib import Path +from typing import Dict, List, Optional, Set, Tuple, Union as TypingUnion + +from fory_compiler.frontend.utils import parse_idl_file +from fory_compiler.generators.base import BaseGenerator, GeneratedFile +from fory_compiler.ir.ast import ( + Enum, + Field, + FieldType, + ListType, + MapType, + Message, + NamedType, + PrimitiveType, + Schema, + Union, +) +from fory_compiler.ir.types import PrimitiveKind + + +class JavaScriptGenerator(BaseGenerator): + """Generates JavaScript/TypeScript type definitions and Fory registration helpers from IDL.""" + + language_name = "javascript" + file_extension = ".ts" + + # TypeScript/JavaScript reserved keywords that cannot be used as identifiers + TS_KEYWORDS = { + "abstract", + "any", + "as", + "asserts", + "async", + "await", + "bigint", + "boolean", + "break", + "case", + "catch", + "class", + "const", + "continue", + "debugger", + "declare", + "default", + "delete", + "do", + "else", + "enum", + "export", + "extends", + "false", + "finally", + "for", + "from", + "function", + "get", + "if", + "implements", + "import", + "in", + "infer", + "instanceof", + "interface", + "is", + "keyof", + "let", + "module", + "namespace", + "never", + "new", + "null", + "number", + "object", + "of", + "package", + "private", + "protected", + "public", + "readonly", + "require", + "return", + "set", + "static", + "string", + "super", + "switch", + "symbol", + "this", + "throw", + "true", + "try", + "type", + "typeof", + "undefined", + "unique", + "unknown", + "var", + "void", + "while", + "with", + "yield", + } + + # Mapping from FDL primitive types to TypeScript types + PRIMITIVE_MAP = { + PrimitiveKind.BOOL: "boolean", + PrimitiveKind.INT8: "number", + PrimitiveKind.INT16: "number", + PrimitiveKind.INT32: "number", + PrimitiveKind.VARINT32: "number", + PrimitiveKind.INT64: "bigint | number", + PrimitiveKind.VARINT64: "bigint | number", + PrimitiveKind.TAGGED_INT64: "bigint | number", + PrimitiveKind.UINT8: "number", + PrimitiveKind.UINT16: "number", + PrimitiveKind.UINT32: "number", + PrimitiveKind.VAR_UINT32: "number", + PrimitiveKind.UINT64: "bigint | number", + PrimitiveKind.VAR_UINT64: "bigint | number", + PrimitiveKind.TAGGED_UINT64: "bigint | number", + PrimitiveKind.FLOAT16: "number", + PrimitiveKind.BFLOAT16: "number", + PrimitiveKind.FLOAT32: "number", + PrimitiveKind.FLOAT64: "number", + PrimitiveKind.STRING: "string", + PrimitiveKind.BYTES: "Uint8Array", + PrimitiveKind.DATE: "Date", + PrimitiveKind.TIMESTAMP: "Date", + PrimitiveKind.DURATION: "number", + PrimitiveKind.DECIMAL: "number", + PrimitiveKind.ANY: "any", + } + + # Mapping from FDL primitive types to Fory JS runtime Type.xxx() calls + PRIMITIVE_RUNTIME_MAP = { + PrimitiveKind.BOOL: "Type.bool()", + PrimitiveKind.INT8: "Type.int8()", + PrimitiveKind.INT16: "Type.int16()", + PrimitiveKind.INT32: "Type.int32()", + PrimitiveKind.VARINT32: "Type.varInt32()", + PrimitiveKind.INT64: "Type.int64()", + PrimitiveKind.VARINT64: "Type.varInt64()", + PrimitiveKind.TAGGED_INT64: "Type.sliInt64()", + PrimitiveKind.UINT8: "Type.uint8()", + PrimitiveKind.UINT16: "Type.uint16()", + PrimitiveKind.UINT32: "Type.uint32()", + PrimitiveKind.VAR_UINT32: "Type.varUInt32()", + PrimitiveKind.UINT64: "Type.uint64()", + PrimitiveKind.VAR_UINT64: "Type.varUInt64()", + PrimitiveKind.TAGGED_UINT64: "Type.taggedUInt64()", + PrimitiveKind.FLOAT16: "Type.float16()", + PrimitiveKind.BFLOAT16: "Type.bfloat16()", + PrimitiveKind.FLOAT32: "Type.float32()", + PrimitiveKind.FLOAT64: "Type.float64()", + PrimitiveKind.STRING: "Type.string()", + PrimitiveKind.BYTES: "Type.binary()", + PrimitiveKind.DATE: "Type.date()", + PrimitiveKind.TIMESTAMP: "Type.timestamp()", + PrimitiveKind.DURATION: "Type.duration()", + PrimitiveKind.DECIMAL: "Type.float64()", + PrimitiveKind.ANY: "Type.any()", + } + + def __init__(self, schema: Schema, options): + super().__init__(schema, options) + self.indent_str = " " # TypeScript uses 2 spaces + self._qualified_type_names: Dict[int, str] = {} + self._build_qualified_type_name_index() + + def _build_qualified_type_name_index(self) -> None: + """Build an index mapping type object ids to their qualified names.""" + for enum in self.schema.enums: + self._qualified_type_names[id(enum)] = enum.name + for union in self.schema.unions: + self._qualified_type_names[id(union)] = union.name + + def visit_message(message: Message, parents: List[str]) -> None: + path = ".".join(parents + [message.name]) + self._qualified_type_names[id(message)] = path + for nested_enum in message.nested_enums: + self._qualified_type_names[id(nested_enum)] = ( + f"{path}.{nested_enum.name}" + ) + for nested_union in message.nested_unions: + self._qualified_type_names[id(nested_union)] = ( + f"{path}.{nested_union.name}" + ) + for nested_msg in message.nested_messages: + visit_message(nested_msg, parents + [message.name]) + + for message in self.schema.messages: + visit_message(message, []) + + def safe_identifier(self, name: str) -> str: + """Escape identifiers that collide with TypeScript reserved words.""" + if name in self.TS_KEYWORDS: + return f"{name}_" + return name + + def safe_type_identifier(self, name: str) -> str: + """Escape type names that collide with TypeScript reserved words.""" + return self.safe_identifier(name) + + def safe_member_name(self, name: str) -> str: + """Generate a safe camelCase member name.""" + return self.safe_identifier(self.to_camel_case(name)) + + def _nested_type_names_for_message(self, message: Message) -> Set[str]: + """Collect safe type names of nested types to detect collisions.""" + names: Set[str] = set() + for nested in ( + list(message.nested_enums) + + list(message.nested_unions) + + list(message.nested_messages) + ): + names.add(self.safe_type_identifier(nested.name)) + return names + + def _field_member_name( + self, + field: Field, + message: Message, + used_names: Set[str], + ) -> str: + """Produce a unique safe member name for a field, avoiding collisions.""" + base = self.safe_member_name(field.name) + nested_type_names = self._nested_type_names_for_message(message) + if base in nested_type_names: + base = f"{base}Value" + + candidate = base + suffix = 1 + while candidate in used_names: + candidate = f"{base}{suffix}" + suffix += 1 + used_names.add(candidate) + return candidate + + def is_imported_type(self, type_def: object) -> bool: + """Return True if a type definition comes from an imported IDL file.""" + if not self.schema.source_file: + return False + location = getattr(type_def, "location", None) + if location is None or not location.file: + return False + try: + return ( + Path(location.file).resolve() != Path(self.schema.source_file).resolve() + ) + except Exception: + return location.file != self.schema.source_file + + def split_imported_types( + self, items: List[object] + ) -> Tuple[List[object], List[object]]: + imported: List[object] = [] + local: List[object] = [] + for item in items: + if self.is_imported_type(item): + imported.append(item) + else: + local.append(item) + return imported, local # Return (imported, local) tuple + + def get_module_name(self) -> str: + """Get the TypeScript module name from source file or package.""" + if self.schema.source_file and not self.schema.source_file.startswith("<"): + return Path(self.schema.source_file).stem + if self.package: + return self.package.replace(".", "_") + return "generated" + + def get_registration_function_name(self) -> str: + """Get the name of the registration function.""" + return f"register{self.to_pascal_case(self.get_module_name())}Types" + + def _normalize_import_path(self, path_str: str) -> str: + if not path_str: + return path_str + try: + return str(Path(path_str).resolve()) + except Exception: + return path_str + + def _load_schema(self, file_path: str) -> Optional[Schema]: + if not file_path: + return None + if not hasattr(self, "_schema_cache"): + self._schema_cache: Dict[Path, Schema] = {} + path = Path(file_path).resolve() + if path in self._schema_cache: + return self._schema_cache[path] + try: + schema = parse_idl_file(path) + except Exception: + return None + self._schema_cache[path] = schema + return schema + + def _module_name_for_schema(self, schema: Schema) -> str: + """Derive a module name from another schema.""" + if schema.source_file and not schema.source_file.startswith("<"): + return Path(schema.source_file).stem + if schema.package: + return schema.package.replace(".", "_") + return "generated" + + def _registration_fn_for_schema(self, schema: Schema) -> str: + """Derive the registration function name for an imported schema.""" + mod = self._module_name_for_schema(schema) + return f"register{self.to_pascal_case(mod)}Types" + + def _collect_imported_registrations(self) -> List[Tuple[str, str]]: + """Collect (module_path, registration_fn) pairs for imported schemas.""" + file_info: Dict[str, Tuple[str, str]] = {} + for type_def in self.schema.enums + self.schema.unions + self.schema.messages: + if not self.is_imported_type(type_def): + continue + location = getattr(type_def, "location", None) + file_path = getattr(location, "file", None) if location else None + if not file_path: + continue + normalized = self._normalize_import_path(file_path) + if normalized in file_info: + continue + imported_schema = self._load_schema(file_path) + if imported_schema is None: + continue + reg_fn = self._registration_fn_for_schema(imported_schema) + mod_name = self._module_name_for_schema(imported_schema) + file_info[normalized] = (f"./{mod_name}", reg_fn) + + ordered: List[Tuple[str, str]] = [] + used: Set[str] = set() + + if self.schema.source_file: + base_dir = Path(self.schema.source_file).resolve().parent + for imp in self.schema.imports: + candidate = self._normalize_import_path( + str((base_dir / imp.path).resolve()) + ) + if candidate in file_info and candidate not in used: + ordered.append(file_info[candidate]) + used.add(candidate) + + for key in sorted(file_info.keys()): + if key in used: + continue + ordered.append(file_info[key]) + + deduped: List[Tuple[str, str]] = [] + seen: Set[Tuple[str, str]] = set() + for item in ordered: + if item in seen: + continue + seen.add(item) + deduped.append(item) + return deduped + + def _resolve_named_type( + self, name: str, parent_stack: Optional[List[Message]] = None + ) -> Optional[TypingUnion[Message, Enum, Union]]: + """Resolve a named type reference to its definition.""" + parent_stack = parent_stack or [] + if "." in name: + return self.schema.get_type(name) + for msg in reversed(parent_stack): + nested = msg.get_nested_type(name) + if nested is not None: + return nested + return self.schema.get_type(name) + + def generate_type( + self, + field_type: FieldType, + nullable: bool = False, + parent_stack: Optional[List[Message]] = None, + ) -> str: + """Generate TypeScript type string for a field type.""" + parent_stack = parent_stack or [] + type_str = "" + + if isinstance(field_type, PrimitiveType): + if field_type.kind not in self.PRIMITIVE_MAP: + raise ValueError( + f"Unsupported primitive type for TypeScript: {field_type.kind}" + ) + type_str = self.PRIMITIVE_MAP[field_type.kind] + elif isinstance(field_type, NamedType): + # Check if this NamedType matches a primitive type name + primitive_name = field_type.name.lower() + # Map common shorthand names to primitive kinds + shorthand_map = { + "float": PrimitiveKind.FLOAT32, + "double": PrimitiveKind.FLOAT64, + } + if primitive_name in shorthand_map: + type_str = self.PRIMITIVE_MAP.get(shorthand_map[primitive_name], "any") + else: + # Check if it matches any primitive kind directly + for primitive_kind, ts_type in self.PRIMITIVE_MAP.items(): + if primitive_kind.value == primitive_name: + type_str = ts_type + break + if not type_str: + resolved = self._resolve_named_type(field_type.name, parent_stack) + if resolved is not None: + type_str = self.safe_type_identifier(resolved.name) + else: + type_str = self.safe_type_identifier( + self.to_pascal_case(field_type.name) + ) + elif isinstance(field_type, ListType): + element_type = self.generate_type( + field_type.element_type, + nullable=field_type.element_optional, + parent_stack=parent_stack, + ) + type_str = f"{element_type}[]" + elif isinstance(field_type, MapType): + key_type = self.generate_type( + field_type.key_type, + nullable=False, + parent_stack=parent_stack, + ) + value_type = self.generate_type( + field_type.value_type, + nullable=False, + parent_stack=parent_stack, + ) + if key_type in ("string", "number"): + type_str = f"Record<{key_type}, {value_type}>" + else: + type_str = f"Map<{key_type}, {value_type}>" + else: + type_str = "any" + + if nullable: + type_str += " | undefined" + + return type_str + + def generate_imports(self) -> List[str]: + """Generate import statements for imported types and registration functions.""" + lines: List[str] = [] + imported_regs = self._collect_imported_registrations() + + # Collect all imported types used in this schema + imported_types_by_module: Dict[str, Set[str]] = {} + + for type_def in self.schema.enums + self.schema.unions + self.schema.messages: + if not self.is_imported_type(type_def): + continue + + location = getattr(type_def, "location", None) + file_path = getattr(location, "file", None) if location else None + if not file_path: + continue + + imported_schema = self._load_schema(file_path) + if imported_schema is None: + continue + + mod_name = self._module_name_for_schema(imported_schema) + mod_path = f"./{mod_name}" + + if mod_path not in imported_types_by_module: + imported_types_by_module[mod_path] = set() + + imported_types_by_module[mod_path].add( + self.safe_type_identifier(type_def.name) + ) + + # If it's a union, also import the Case enum + if isinstance(type_def, Union): + imported_types_by_module[mod_path].add( + self.safe_type_identifier(f"{type_def.name}Case") + ) + + # Add registration functions to the imports + for mod_path, reg_fn in imported_regs: + if mod_path not in imported_types_by_module: + imported_types_by_module[mod_path] = set() + imported_types_by_module[mod_path].add(reg_fn) + + # Generate import statements + for mod_path, types in sorted(imported_types_by_module.items()): + if types: + types_str = ", ".join(sorted(types)) + lines.append(f"import {{ {types_str} }} from '{mod_path}';") + + return lines + + def generate(self) -> List[GeneratedFile]: + """Generate TypeScript files for the schema.""" + return [self.generate_file()] + + def generate_file(self) -> GeneratedFile: + """Generate a single TypeScript module with all types.""" + lines: List[str] = [] + + # License header + lines.append(self.get_license_header("//")) + lines.append("") + + # Add package comment if present + if self.package: + lines.append(f"// Package: {self.package}") + lines.append("") + + # Generate enums (top-level only) + _, local_enums = self.split_imported_types(self.schema.enums) + if local_enums: + lines.append("// Enums") + lines.append("") + for enum in local_enums: + lines.extend(self.generate_enum(enum)) + lines.append("") + + # Generate unions (top-level only) + _, local_unions = self.split_imported_types(self.schema.unions) + if local_unions: + lines.append("// Unions") + lines.append("") + for union in local_unions: + lines.extend(self.generate_union(union)) + lines.append("") + + # Generate messages (including nested types) + _, local_messages = self.split_imported_types(self.schema.messages) + if local_messages: + lines.append("// Messages") + lines.append("") + for message in local_messages: + lines.extend(self.generate_message(message, indent=0)) + lines.append("") + + # Generate registration function + lines.extend(self.generate_registration()) + lines.append("") + + # Add imports at the top + imports = self.generate_imports() + if imports: + # Insert after package comment or license + insert_idx = 0 + for i, line in enumerate(lines): + if line.startswith("// Package:") or line.startswith("// Licensed"): + insert_idx = i + 2 + + lines.insert(insert_idx, "") + for imp in reversed(imports): + lines.insert(insert_idx, imp) + lines.insert(insert_idx, "") + + return GeneratedFile( + path=f"{self.get_module_name()}{self.file_extension}", + content="\n".join(lines), + ) + + def generate_enum(self, enum: Enum, indent: int = 0) -> List[str]: + """Generate a TypeScript enum.""" + lines: List[str] = [] + ind = self.indent_str * indent + comment = self.format_type_id_comment(enum, f"{ind}//") + if comment: + lines.append(comment) + + enum_name = self.safe_type_identifier(enum.name) + lines.append(f"{ind}export enum {enum_name} {{") + for value in enum.values: + stripped_name = self.strip_enum_prefix(enum.name, value.name) + value_name = self.safe_identifier(stripped_name) + lines.append(f"{ind}{self.indent_str}{value_name} = {value.value},") + lines.append(f"{ind}}}") + + return lines + + def generate_message( + self, + message: Message, + indent: int = 0, + parent_stack: Optional[List[Message]] = None, + ) -> List[str]: + """Generate a TypeScript interface for a message.""" + lines: List[str] = [] + ind = self.indent_str * indent + parent_stack = parent_stack or [] + lineage = parent_stack + [message] + type_name = self.safe_type_identifier(message.name) + + comment = self.format_type_id_comment(message, f"{ind}//") + if comment: + lines.append(comment) + + lines.append(f"{ind}export interface {type_name} {{") + + # Generate fields with safe, deduplicated names + used_field_names: Set[str] = set() + for field in message.fields: + field_name = self._field_member_name(field, message, used_field_names) + field_type = self.generate_type( + field.field_type, + nullable=field.optional, + parent_stack=lineage, + ) + optional_marker = "?" if field.optional else "" + lines.append( + f"{ind}{self.indent_str}{field_name}{optional_marker}: {field_type};" + ) + + lines.append(f"{ind}}}") + + # Generate nested enums after parent interface + for nested_enum in message.nested_enums: + lines.append("") + lines.extend(self.generate_enum(nested_enum, indent=indent)) + + # Generate nested unions after parent interface + for nested_union in message.nested_unions: + lines.append("") + lines.extend(self.generate_union(nested_union, indent=indent)) + + # Generate nested messages after parent interface + for nested_msg in message.nested_messages: + lines.append("") + lines.extend( + self.generate_message(nested_msg, indent=indent, parent_stack=lineage) + ) + + return lines + + def generate_union( + self, + union: Union, + indent: int = 0, + parent_stack: Optional[List[Message]] = None, + ) -> List[str]: + """Generate a TypeScript discriminated union.""" + lines: List[str] = [] + ind = self.indent_str * indent + union_name = self.safe_type_identifier(union.name) + + comment = self.format_type_id_comment(union, f"{ind}//") + if comment: + lines.append(comment) + + # Generate case enum + case_enum_name = self.safe_type_identifier(f"{union.name}Case") + lines.append(f"{ind}export enum {case_enum_name} {{") + for field in union.fields: + case_name = self.safe_identifier(self.to_upper_snake_case(field.name)) + lines.append(f"{ind}{self.indent_str}{case_name} = {field.number},") + lines.append(f"{ind}}}") + lines.append("") + + # Generate union type as discriminated union + union_cases = [] + for field in union.fields: + field_type_str = self.generate_type( + field.field_type, + nullable=False, + parent_stack=parent_stack, + ) + case_value = self.safe_identifier(self.to_upper_snake_case(field.name)) + union_cases.append( + f"{ind}{self.indent_str}| ( {{ case: {case_enum_name}.{case_value}; value: {field_type_str} }} )" + ) + + lines.append(f"{ind}export type {union_name} =") + lines.extend(union_cases) + lines.append(f"{ind}{self.indent_str};") + + return lines + + def _field_type_expr( + self, + field_type: FieldType, + parent_stack: Optional[List[Message]] = None, + ) -> str: + """Return the Fory JS runtime ``Type.xxx()`` expression for a field type.""" + parent_stack = parent_stack or [] + if isinstance(field_type, PrimitiveType): + expr = self.PRIMITIVE_RUNTIME_MAP.get(field_type.kind) + if expr is None: + return "Type.any()" + return expr + elif isinstance(field_type, NamedType): + # Check for primitive-like shorthand names (e.g. "float", "double") + lower = field_type.name.lower() + shorthand_map = { + "float": PrimitiveKind.FLOAT32, + "double": PrimitiveKind.FLOAT64, + } + if lower in shorthand_map: + return self.PRIMITIVE_RUNTIME_MAP[shorthand_map[lower]] + for pk in PrimitiveKind: + if pk.value == lower: + return self.PRIMITIVE_RUNTIME_MAP.get(pk, "Type.any()") + + # Named type — could be a Message, Enum, or Union + resolved = self._resolve_named_type(field_type.name, parent_stack) + if isinstance(resolved, Enum): + return "Type.int32()" + if isinstance(resolved, Union): + return "Type.any()" + if isinstance(resolved, Message): + if self.should_register_by_id(resolved): + return f"Type.struct({resolved.type_id})" + ns = self.schema.package or "default" + qname = self._qualified_type_names.get(id(resolved), resolved.name) + return f'Type.struct("{ns}.{qname}")' + # Unresolved — fall back to any + return "Type.any()" + elif isinstance(field_type, ListType): + inner = self._field_type_expr(field_type.element_type, parent_stack) + return f"Type.array({inner})" + elif isinstance(field_type, MapType): + key = self._field_type_expr(field_type.key_type, parent_stack) + value = self._field_type_expr(field_type.value_type, parent_stack) + return f"Type.map({key}, {value})" + return "Type.any()" + + def _register_type_line( + self, + type_def: TypingUnion[Message, Enum, Union], + target_var: str = "fory", + parent_stack: Optional[List[Message]] = None, + ) -> str: + """Return a single ``fory.registerSerializer(Type.struct(...))`` statement.""" + if isinstance(type_def, Union): + return "" + if not isinstance(type_def, Message): + return "" + + field_parent_stack = (parent_stack or []) + [type_def] + props_parts: List[str] = [] + for field in type_def.fields: + member = self.safe_member_name(field.name) + expr = self._field_type_expr(field.field_type, field_parent_stack) + if field.optional: + expr += ".setNullable(true)" + props_parts.append(f"{member}: {expr}") + + props_str = ", ".join(props_parts) + props_arg = f", {{ {props_str} }}" if props_parts else "" + + if self.should_register_by_id(type_def): + name_info = str(type_def.type_id) + else: + ns = self.schema.package or "default" + qname = self._qualified_type_names.get(id(type_def), type_def.name) + name_info = f'{{ namespace: "{ns}", typeName: "{qname}" }}' + + return f"{target_var}.registerSerializer(Type.struct({name_info}{props_arg}));" + + def _resolve_field_deps( + self, + message: Message, + parent_stack: List[Message], + ) -> List[Message]: + """Return the local Message objects that *message* directly references + in its fields (excluding self-references and imported types).""" + lineage = parent_stack + [message] + deps: List[Message] = [] + seen: Set[int] = set() + + def visit(ft: FieldType) -> None: + if isinstance(ft, NamedType): + resolved = self._resolve_named_type(ft.name, lineage) + if ( + isinstance(resolved, Message) + and not self.is_imported_type(resolved) + and id(resolved) != id(message) + and id(resolved) not in seen + ): + seen.add(id(resolved)) + deps.append(resolved) + elif isinstance(ft, ListType): + visit(ft.element_type) + elif isinstance(ft, MapType): + visit(ft.key_type) + visit(ft.value_type) + + for field in message.fields: + visit(field.field_type) + return deps + + def generate_registration(self) -> List[str]: + """Generate a registration function that registers all local and + imported types with a Fory instance. + + Types are emitted in dependency order (leaf types first) via a + simple DFS so that the Fory JS runtime does not prematurely + register bare ``Type.struct(id)`` references with empty fields.""" + lines: List[str] = [] + fn_name = self.get_registration_function_name() + imported_regs = self._collect_imported_registrations() + + lines.append("// Registration helper") + lines.append(f"export function {fn_name}(fory: any, Type: any): void {{") + + # Delegate to imported registration functions first + for _module_path, reg_fn in imported_regs: + if reg_fn == fn_name: + continue + lines.append(f" {reg_fn}(fory, Type);") + + # DFS emit: visit dependencies before the type itself. + # The visited set also breaks cycles (e.g. self-referential trees). + emitted: Set[int] = set() + # Pre-build a mapping from message id -> parent_stack so that + # dependencies emitted out of tree order still get the right context. + parent_map: Dict[int, List[Message]] = {} + + def build_parent_map(msg: Message, parents: List[Message]) -> None: + parent_map[id(msg)] = parents + for nested_msg in msg.nested_messages: + build_parent_map(nested_msg, parents + [msg]) + + for message in self.schema.messages: + if not self.is_imported_type(message): + build_parent_map(message, []) + + def emit_message(msg: Message) -> None: + if id(msg) in emitted or self.is_imported_type(msg): + return + # Mark visited early to break cycles (e.g. Node <-> Edge) + emitted.add(id(msg)) + parents = parent_map.get(id(msg), []) + # Emit field-level struct dependencies first + for dep in self._resolve_field_deps(msg, parents): + emit_message(dep) + reg_line = self._register_type_line(msg, "fory", parents) + if reg_line: + lines.append(f" {reg_line}") + for nested_msg in msg.nested_messages: + emit_message(nested_msg) + + for message in self.schema.messages: + if self.is_imported_type(message): + continue + emit_message(message) + + lines.append("}") + + return lines diff --git a/compiler/fory_compiler/tests/test_generated_code.py b/compiler/fory_compiler/tests/test_generated_code.py index 8fff024e3f..738094054b 100644 --- a/compiler/fory_compiler/tests/test_generated_code.py +++ b/compiler/fory_compiler/tests/test_generated_code.py @@ -32,6 +32,7 @@ from fory_compiler.generators.python import PythonGenerator from fory_compiler.generators.rust import RustGenerator from fory_compiler.generators.csharp import CSharpGenerator +from fory_compiler.generators.javascript import JavaScriptGenerator from fory_compiler.generators.swift import SwiftGenerator from fory_compiler.ir.ast import Schema @@ -43,6 +44,7 @@ RustGenerator, GoGenerator, CSharpGenerator, + JavaScriptGenerator, SwiftGenerator, ) diff --git a/compiler/fory_compiler/tests/test_javascript_codegen.py b/compiler/fory_compiler/tests/test_javascript_codegen.py new file mode 100644 index 0000000000..e3e2577f85 --- /dev/null +++ b/compiler/fory_compiler/tests/test_javascript_codegen.py @@ -0,0 +1,434 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Tests for JavaScript code generation.""" + +from pathlib import Path +from textwrap import dedent + +from fory_compiler.frontend.fdl.lexer import Lexer +from fory_compiler.frontend.fdl.parser import Parser +from fory_compiler.generators.base import GeneratorOptions +from fory_compiler.generators.javascript import JavaScriptGenerator +from fory_compiler.ir.ast import Schema + + +def parse_fdl(source: str) -> Schema: + return Parser(Lexer(source).tokenize()).parse() + + +def generate_javascript(source: str) -> str: + schema = parse_fdl(source) + options = GeneratorOptions(output_dir=Path("/tmp")) + generator = JavaScriptGenerator(schema, options) + files = generator.generate() + assert len(files) == 1, f"Expected 1 file, got {len(files)}" + return files[0].content + + +def test_javascript_enum_generation(): + """Test that enums are properly generated.""" + source = dedent( + """ + package example; + + enum Color [id=101] { + RED = 0; + GREEN = 1; + BLUE = 2; + } + """ + ) + output = generate_javascript(source) + + # Check enum definition + assert "export enum Color" in output + assert "RED = 0" in output + assert "GREEN = 1" in output + assert "BLUE = 2" in output + assert "Type ID 101" in output + + +def test_javascript_message_generation(): + """Test that messages are properly generated as interfaces.""" + source = dedent( + """ + package example; + + message Person [id=102] { + string name = 1; + int32 age = 2; + optional string email = 3; + } + """ + ) + output = generate_javascript(source) + + # Check interface definition + assert "export interface Person" in output + assert "name: string;" in output + assert "age: number;" in output + assert "email?: string | undefined;" in output + assert "Type ID 102" in output + + +def test_javascript_nested_message(): + """Test that nested messages are properly generated.""" + source = dedent( + """ + package example; + + message Person [id=100] { + string name = 1; + + message Address [id=101] { + string street = 1; + string city = 2; + } + + Address address = 2; + } + """ + ) + output = generate_javascript(source) + + # Check nested interface + assert "export interface Person" in output + assert "export interface Address" in output + assert "street: string;" in output + assert "city: string;" in output + + +def test_javascript_nested_enum(): + """Test that nested enums are properly generated.""" + source = dedent( + """ + package example; + + message Person [id=100] { + string name = 1; + + enum PhoneType [id=101] { + MOBILE = 0; + HOME = 1; + } + } + """ + ) + output = generate_javascript(source) + + # Check nested enum + assert "export enum PhoneType" in output + assert "MOBILE = 0" in output + assert "HOME = 1" in output + + +def test_javascript_nested_enum_registration_uses_simple_name(): + """Test that nested enums are registered with simple names, not qualified names.""" + source = dedent( + """ + package example; + + message Person [id=100] { + string name = 1; + + enum PhoneType [id=101] { + MOBILE = 0; + HOME = 1; + } + } + """ + ) + output = generate_javascript(source) + + # Enums are skipped during registration in JavaScript (they are numeric + # values at runtime and don't need separate Fory registration). + assert "fory.register('PhoneType'" not in output + # Messages are registered via fory.registerSerializer(Type.struct(...)). + assert "fory.registerSerializer(Type.struct(100" in output + # Ensure qualified names are NOT used + assert "Person.PhoneType" not in output + + +def test_javascript_union_generation(): + """Test that unions are properly generated as discriminated unions.""" + source = dedent( + """ + package example; + + message Dog [id=101] { + string name = 1; + int32 bark_volume = 2; + } + + message Cat [id=102] { + string name = 1; + int32 lives = 2; + } + + union Animal [id=103] { + Dog dog = 1; + Cat cat = 2; + } + """ + ) + output = generate_javascript(source) + + # Check union generation + assert "export enum AnimalCase" in output + assert "DOG = 1" in output + assert "CAT = 2" in output + assert "export type Animal" in output + assert "AnimalCase.DOG" in output + assert "AnimalCase.CAT" in output + assert "Type ID 103" in output + + +def test_javascript_collection_types(): + """Test that collection types are properly mapped.""" + source = dedent( + """ + package example; + + message Data [id=100] { + repeated string items = 1; + map config = 2; + } + """ + ) + output = generate_javascript(source) + + # Check collection types + assert "items: string[];" in output + assert "config: Record;" in output + + +def test_javascript_map_key_fallback_to_map(): + """Test that map keys not valid for Record use Map instead.""" + source = dedent( + """ + package example; + + message Data [id=100] { + map str_key = 1; + map num_key = 2; + map bigint_key = 3; + map bigint_key2 = 4; + } + """ + ) + output = generate_javascript(source) + + # string and number keys -> Record + assert "strKey: Record;" in output + assert "numKey: Record;" in output + + # bigint | number keys -> Map (bigint is not a valid Record key) + assert "bigintKey: Map;" in output + assert "bigintKey2: Map;" in output + + +def test_javascript_primitive_types(): + """Test that all primitive types are properly mapped.""" + source = dedent( + """ + package example; + + message AllTypes [id=100] { + bool f_bool = 1; + int32 f_int32 = 2; + int64 f_int64 = 3; + uint32 f_uint32 = 4; + uint64 f_uint64 = 5; + float f_float = 6; + double f_double = 7; + string f_string = 8; + bytes f_bytes = 9; + } + """ + ) + output = generate_javascript(source) + + # Check type mappings (field names are converted to camelCase) + assert "fBool: boolean;" in output + assert "fInt32: number;" in output + assert "fInt64: bigint | number;" in output + assert "fUint32: number;" in output + assert "fUint64: bigint | number;" in output + assert "fFloat: number;" in output + assert "fDouble: number;" in output + assert "fString: string;" in output + assert "fBytes: Uint8Array;" in output + + +def test_javascript_file_structure(): + """Test that generated file has proper structure.""" + source = dedent( + """ + package example.v1; + + enum Status [id=100] { + UNKNOWN = 0; + ACTIVE = 1; + } + + message Request [id=101] { + string query = 1; + } + + union Response [id=102] { + string result = 1; + string error = 2; + } + """ + ) + output = generate_javascript(source) + + # Check license header + assert "Apache Software Foundation (ASF)" in output + assert "Licensed" in output + + # Check package comment + assert "Package: example.v1" in output + + # Check section comments + assert "// Enums" in output + assert "// Messages" in output + assert "// Unions" in output + assert "// Registration helper" in output + + # Check registration function (uses full package path to avoid collisions) + assert "export function registerExampleV1Types" in output + + +def test_javascript_field_naming(): + """Test that field names are converted to camelCase.""" + source = dedent( + """ + package example; + + message Person [id=100] { + string first_name = 1; + string last_name = 2; + int32 phone_number = 3; + } + """ + ) + output = generate_javascript(source) + + # Check that field names are properly converted to camelCase + assert "firstName:" in output + assert "lastName:" in output + assert "phoneNumber:" in output + # Ensure snake_case is not used + assert "first_name:" not in output + assert "last_name:" not in output + assert "phone_number:" not in output + + +def test_javascript_no_runtime_dependencies(): + """Test that generated code has no gRPC runtime dependencies.""" + source = dedent( + """ + package example; + + message Request [id=100] { + string query = 1; + } + """ + ) + output = generate_javascript(source) + + # Should not reference gRPC + assert "@grpc" not in output + assert "grpc-js" not in output + assert "require('grpc" not in output + assert "import.*grpc" not in output + + +def test_javascript_file_extension(): + """Test that output file has correct extension.""" + source = dedent( + """ + package example; + + message Test [id=100] { + string value = 1; + } + """ + ) + + schema = parse_fdl(source) + options = GeneratorOptions(output_dir=Path("/tmp")) + generator = JavaScriptGenerator(schema, options) + files = generator.generate() + + assert len(files) == 1 + assert files[0].path.endswith(".js") or files[0].path.endswith(".ts"), ( + f"Unexpected file extension: {files[0].path}" + ) + + +def test_javascript_enum_value_stripping(): + """Test that enum value prefixes are stripped correctly.""" + source = dedent( + """ + package example; + + enum PhoneType [id=100] { + PHONE_TYPE_MOBILE = 0; + PHONE_TYPE_HOME = 1; + PHONE_TYPE_WORK = 2; + } + """ + ) + output = generate_javascript(source) + + # Prefixes should be stripped + assert "MOBILE = 0" in output + assert "HOME = 1" in output + assert "WORK = 2" in output + + +def test_javascript_qualified_nested_type_resolved(): + """Test that qualified nested type refs (Outer.Inner) emit the simple name.""" + source = dedent( + """ + package example; + + message Outer [id=100] { + string label = 1; + + message Inner [id=101] { + int32 value = 1; + } + + Inner nested = 2; + } + + message Consumer [id=102] { + Outer.Inner item = 1; + } + """ + ) + output = generate_javascript(source) + + # Nested type is flattened to a top-level export + assert "export interface Inner" in output + + assert "item: Inner;" in output + assert "Outer.Inner" not in output diff --git a/docs/compiler/compiler-guide.md b/docs/compiler/compiler-guide.md index f58eaee0ca..702f771d94 100644 --- a/docs/compiler/compiler-guide.md +++ b/docs/compiler/compiler-guide.md @@ -64,6 +64,7 @@ Compile options: | `--go_out=DST_DIR` | Generate Go code in DST_DIR | (none) | | `--rust_out=DST_DIR` | Generate Rust code in DST_DIR | (none) | | `--csharp_out=DST_DIR` | Generate C# code in DST_DIR | (none) | +| `--javascript_out=DST_DIR` | Generate JavaScript code in DST_DIR | (none) | | `--swift_out=DST_DIR` | Generate Swift code in DST_DIR | (none) | | `--go_nested_type_style` | Go nested type naming: `camelcase` or `underscore` | `underscore` | | `--swift_namespace_style` | Swift namespace style: `enum` or `flatten` | `enum` | @@ -115,7 +116,7 @@ foryc schema.fdl **Compile for specific languages:** ```bash -foryc schema.fdl --lang java,python,csharp,swift +foryc schema.fdl --lang java,python,csharp,javascript,swift ``` **Specify output directory:** @@ -162,7 +163,7 @@ foryc src/main.fdl -I libs/common,libs/types --proto_path third_party/ foryc schema.fdl --java_out=./src/main/java # Generate multiple languages to different directories -foryc schema.fdl --java_out=./java/gen --python_out=./python/src --go_out=./go/gen --csharp_out=./csharp/gen --swift_out=./swift/gen +foryc schema.fdl --java_out=./java/gen --python_out=./python/src --go_out=./go/gen --csharp_out=./csharp/gen --javascript_out=./javascript/src --swift_out=./swift/gen # Combine with import paths foryc schema.fdl --java_out=./gen/java -I proto/ -I common/ @@ -231,15 +232,16 @@ Compiling src/main.fdl... ## Supported Languages -| Language | Flag | Output Extension | Description | -| -------- | -------- | ---------------- | ---------------------------- | -| Java | `java` | `.java` | POJOs with Fory annotations | -| Python | `python` | `.py` | Dataclasses with type hints | -| Go | `go` | `.go` | Structs with struct tags | -| Rust | `rust` | `.rs` | Structs with derive macros | -| C++ | `cpp` | `.h` | Structs with FORY macros | -| C# | `csharp` | `.cs` | Classes with Fory attributes | -| Swift | `swift` | `.swift` | `@ForyObject` Swift models | +| Language | Flag | Output Extension | Description | +| ---------- | ------------ | ---------------- | ------------------------------------- | +| Java | `java` | `.java` | POJOs with Fory annotations | +| Python | `python` | `.py` | Dataclasses with type hints | +| Go | `go` | `.go` | Structs with struct tags | +| Rust | `rust` | `.rs` | Structs with derive macros | +| C++ | `cpp` | `.h` | Structs with FORY macros | +| C# | `csharp` | `.cs` | Classes with Fory attributes | +| JavaScript | `javascript` | `.ts` | Interfaces with registration function | +| Swift | `swift` | `.swift` | `@ForyObject` Swift models | ## Output Structure @@ -309,6 +311,20 @@ generated/ - Namespace matches package (dots to `::`) - Header guards and forward declarations +### JavaScript + +``` +generated/ +└── javascript/ + └── example.ts +``` + +- Single `.ts` file per schema +- `export interface` declarations for messages +- `export enum` declarations for enums +- Discriminated unions with case enums +- Registration helper function included + ### C\# ``` diff --git a/docs/compiler/generated-code.md b/docs/compiler/generated-code.md index dcf9bce3b1..3470b84d11 100644 --- a/docs/compiler/generated-code.md +++ b/docs/compiler/generated-code.md @@ -744,6 +744,50 @@ public static class AddressbookForyRegistration When explicit type IDs are not provided, generated registration uses computed numeric IDs (same behavior as other targets). +## JavaScript + +### Output Layout + +JavaScript output is one `.ts` file per schema, for example: + +- `/addressbook.ts` + +### Type Generation + +Messages generate `export interface` declarations with camelCase field names: + +```typescript +export interface Person { + name: string; + id: number; + phones: PhoneNumber[]; + pet?: Animal | undefined; +} +``` + +Enums generate `export enum` declarations: + +```typescript +export enum PhoneType { + PHONE_TYPE_MOBILE = 0, + PHONE_TYPE_HOME = 1, + PHONE_TYPE_WORK = 2, +} +``` + +Unions generate a discriminated union with a case enum: + +```typescript +export enum AnimalCase { + DOG = 1, + CAT = 2, +} + +export type Animal = + | { case: AnimalCase.DOG; value: Dog } + | { case: AnimalCase.CAT; value: Cat }; +``` + ## Swift ### Output Layout @@ -826,24 +870,26 @@ If `option enable_auto_type_id = false;` is set, generated code uses name-based ### Nested Type Shape -| Language | Nested type form | -| -------- | ------------------------------ | -| Java | `Person.PhoneNumber` | -| Python | `Person.PhoneNumber` | -| Rust | `person::PhoneNumber` | -| C++ | `Person::PhoneNumber` | -| Go | `Person_PhoneNumber` (default) | -| C# | `Person.PhoneNumber` | -| Swift | `Person.PhoneNumber` | +| Language | Nested type form | +| ---------- | ------------------------------ | +| Java | `Person.PhoneNumber` | +| Python | `Person.PhoneNumber` | +| Rust | `person::PhoneNumber` | +| C++ | `Person::PhoneNumber` | +| Go | `Person_PhoneNumber` (default) | +| C# | `Person.PhoneNumber` | +| JavaScript | `PhoneNumber` (flat) | +| Swift | `Person.PhoneNumber` | ### Byte Helper Naming -| Language | Helpers | -| -------- | ------------------------- | -| Java | `toBytes` / `fromBytes` | -| Python | `to_bytes` / `from_bytes` | -| Rust | `to_bytes` / `from_bytes` | -| C++ | `to_bytes` / `from_bytes` | -| Go | `ToBytes` / `FromBytes` | -| C# | `ToBytes` / `FromBytes` | -| Swift | `toBytes` / `fromBytes` | +| Language | Helpers | +| ---------- | ------------------------- | +| Java | `toBytes` / `fromBytes` | +| Python | `to_bytes` / `from_bytes` | +| Rust | `to_bytes` / `from_bytes` | +| C++ | `to_bytes` / `from_bytes` | +| Go | `ToBytes` / `FromBytes` | +| C# | `ToBytes` / `FromBytes` | +| JavaScript | (via `fory.serialize()`) | +| Swift | `toBytes` / `fromBytes` | diff --git a/docs/compiler/index.md b/docs/compiler/index.md index 63c0a0659d..52408c7ffe 100644 --- a/docs/compiler/index.md +++ b/docs/compiler/index.md @@ -21,7 +21,7 @@ license: | Fory IDL is a schema definition language for Apache Fory that enables type-safe cross-language serialization. Define your data structures once and generate -native data structure code for Java, Python, Go, Rust, C++, C#, and Swift. +native data structure code for Java, Python, Go, Rust, C++, C#, Swift, and JavaScript. ## Example Schema @@ -101,6 +101,7 @@ Generated code uses native language constructs: - Rust: Structs with `#[derive(ForyObject)]` - C++: Structs with `FORY_STRUCT` macros - C#: Classes with `[ForyObject]` and registration helpers +- JavaScript: Interfaces with registration function - Swift: `@ForyObject` models with `@ForyField` metadata and registration helpers ## Quick Start @@ -139,7 +140,7 @@ message Person { foryc example.fdl --output ./generated # Generate for specific languages -foryc example.fdl --lang java,python,csharp,swift --output ./generated +foryc example.fdl --lang java,python,csharp,javascript,swift --output ./generated ``` ### 4. Use Generated Code @@ -194,11 +195,11 @@ message Example { Fory IDL types map to native types in each language: -| Fory IDL Type | Java | Python | Go | Rust | C++ | C# | Swift | -| ------------- | --------- | -------------- | -------- | -------- | ------------- | -------- | -------- | -| `int32` | `int` | `pyfory.int32` | `int32` | `i32` | `int32_t` | `int` | `Int32` | -| `string` | `String` | `str` | `string` | `String` | `std::string` | `string` | `String` | -| `bool` | `boolean` | `bool` | `bool` | `bool` | `bool` | `bool` | `Bool` | +| Fory IDL Type | Java | Python | Go | Rust | C++ | C# | JavaScript | Swift | +| ------------- | --------- | -------------- | -------- | -------- | ------------- | -------- | ---------- | -------- | +| `int32` | `int` | `pyfory.int32` | `int32` | `i32` | `int32_t` | `int` | `number` | `Int32` | +| `string` | `String` | `str` | `string` | `String` | `std::string` | `string` | `string` | `String` | +| `bool` | `boolean` | `bool` | `bool` | `bool` | `bool` | `bool` | `boolean` | `Bool` | See [Type System](schema-idl.md#type-system) for complete mappings. diff --git a/docs/compiler/schema-idl.md b/docs/compiler/schema-idl.md index c66803be6b..382cb65179 100644 --- a/docs/compiler/schema-idl.md +++ b/docs/compiler/schema-idl.md @@ -93,14 +93,15 @@ package com.example.models alias models_v1; **Language Mapping:** -| Language | Package Usage | -| -------- | --------------------------------- | -| Java | Java package | -| Python | Module name (dots to underscores) | -| Go | Package name (last component) | -| Rust | Module name (dots to underscores) | -| C++ | Namespace (dots to `::`) | -| C# | Namespace | +| Language | Package Usage | +| ---------- | --------------------------------- | +| Java | Java package | +| Python | Module name (dots to underscores) | +| Go | Package name (last component) | +| Rust | Module name (dots to underscores) | +| C++ | Namespace (dots to `::`) | +| C# | Namespace | +| JavaScript | Module name (last segment) | ## File-Level Options @@ -542,13 +543,14 @@ FDL does not support `option ...;` statements inside enum bodies. ### Language Mapping -| Language | Implementation | -| -------- | -------------------------------------- | -| Java | `enum Status { UNKNOWN, ACTIVE, ... }` | -| Python | `class Status(IntEnum): UNKNOWN = 0` | -| Go | `type Status int32` with constants | -| Rust | `#[repr(i32)] enum Status { Unknown }` | -| C++ | `enum class Status : int32_t { ... }` | +| Language | Implementation | +| ---------- | -------------------------------------- | +| Java | `enum Status { UNKNOWN, ACTIVE, ... }` | +| Python | `class Status(IntEnum): UNKNOWN = 0` | +| Go | `type Status int32` with constants | +| Rust | `#[repr(i32)] enum Status { Unknown }` | +| C++ | `enum class Status : int32_t { ... }` | +| JavaScript | `export enum Status { UNKNOWN, ... }` | ### Enum Prefix Stripping @@ -565,13 +567,14 @@ enum DeviceTier { **Generated code:** -| Language | Output | Style | -| -------- | ----------------------------------------- | -------------- | -| Java | `UNKNOWN, TIER1, TIER2` | Scoped enum | -| Rust | `Unknown, Tier1, Tier2` | Scoped enum | -| C++ | `UNKNOWN, TIER1, TIER2` | Scoped enum | -| Python | `UNKNOWN, TIER1, TIER2` | Scoped IntEnum | -| Go | `DeviceTierUnknown, DeviceTierTier1, ...` | Unscoped const | +| Language | Output | Style | +| ---------- | ----------------------------------------- | -------------- | +| Java | `UNKNOWN, TIER1, TIER2` | Scoped enum | +| Rust | `Unknown, Tier1, Tier2` | Scoped enum | +| C++ | `UNKNOWN, TIER1, TIER2` | Scoped enum | +| Python | `UNKNOWN, TIER1, TIER2` | Scoped IntEnum | +| Go | `DeviceTierUnknown, DeviceTierTier1, ...` | Unscoped const | +| JavaScript | `UNKNOWN, TIER1, TIER2` | Scoped enum | **Note:** The prefix is only stripped if the remainder is a valid identifier. For example, `DEVICE_TIER_1` is kept unchanged because `1` is not a valid identifier name. @@ -641,13 +644,14 @@ message Person { // Auto-generated when enable_auto_type_id = true ### Language Mapping -| Language | Implementation | -| -------- | ----------------------------------- | -| Java | POJO class with getters/setters | -| Python | `@dataclass` class | -| Go | Struct with exported fields | -| Rust | Struct with `#[derive(ForyObject)]` | -| C++ | Struct with `FORY_STRUCT` macro | +| Language | Implementation | +| ---------- | ----------------------------------- | +| Java | POJO class with getters/setters | +| Python | `@dataclass` class | +| Go | Struct with exported fields | +| Rust | Struct with `#[derive(ForyObject)]` | +| C++ | Struct with `FORY_STRUCT` macro | +| JavaScript | `export interface` declaration | Type IDs control cross-language registration for messages, unions, and enums. See [Type IDs](#type-ids) for auto-generation, aliases, and collision handling. @@ -764,13 +768,14 @@ message OtherMessage { ### Language-Specific Generation -| Language | Nested Type Generation | -| -------- | --------------------------------------------------------------------------------- | -| Java | Static inner classes (`SearchResponse.Result`) | -| Python | Nested classes within dataclass | -| Go | Flat structs with underscore (`SearchResponse_Result`, configurable to camelcase) | -| Rust | Nested modules (`search_response::Result`) | -| C++ | Nested classes (`SearchResponse::Result`) | +| Language | Nested Type Generation | +| ---------- | --------------------------------------------------------------------------------- | +| Java | Static inner classes (`SearchResponse.Result`) | +| Python | Nested classes within dataclass | +| Go | Flat structs with underscore (`SearchResponse_Result`, configurable to camelcase) | +| Rust | Nested modules (`search_response::Result`) | +| C++ | Nested classes (`SearchResponse::Result`) | +| JavaScript | Flat names (`Result`) | **Note:** Go defaults to underscore-separated nested names; set `option go_nested_type_style = "camelcase";` to use concatenated names. Rust emits nested modules for nested types. @@ -866,13 +871,14 @@ message User { **Generated Code:** -| Language | Non-optional | Optional | -| -------- | ------------------ | ----------------------------------------------- | -| Java | `String name` | `String email` with `@ForyField(nullable=true)` | -| Python | `name: str` | `name: Optional[str]` | -| Go | `Name string` | `Name *string` | -| Rust | `name: String` | `name: Option` | -| C++ | `std::string name` | `std::optional name` | +| Language | Non-optional | Optional | +| ---------- | ------------------ | ----------------------------------------------- | +| Java | `String name` | `String email` with `@ForyField(nullable=true)` | +| Python | `name: str` | `name: Optional[str]` | +| Go | `Name string` | `Name *string` | +| Rust | `name: String` | `name: Option` | +| C++ | `std::string name` | `std::optional name` | +| JavaScript | `name: string` | `name?: string \| undefined` | **Default Values:** @@ -901,13 +907,14 @@ message Node { **Generated Code:** -| Language | Without `ref` | With `ref` | -| -------- | -------------- | ----------------------------------------- | -| Java | `Node parent` | `Node parent` with `@ForyField(ref=true)` | -| Python | `parent: Node` | `parent: Node = pyfory.field(ref=True)` | -| Go | `Parent Node` | `Parent *Node` with `fory:"ref"` | -| Rust | `parent: Node` | `parent: Arc` | -| C++ | `Node parent` | `std::shared_ptr parent` | +| Language | Without `ref` | With `ref` | +| ---------- | -------------- | ----------------------------------------- | +| Java | `Node parent` | `Node parent` with `@ForyField(ref=true)` | +| Python | `parent: Node` | `parent: Node = pyfory.field(ref=True)` | +| Go | `Parent Node` | `Parent *Node` with `fory:"ref"` | +| Rust | `parent: Node` | `parent: Arc` | +| C++ | `Node parent` | `std::shared_ptr parent` | +| JavaScript | `parent: Node` | `parent: Node` (no ref distinction) | Rust uses `Arc` by default; use `ref(thread_safe=false)` or `ref(weak=true)` to customize pointer types. For protobuf option syntax, see @@ -926,13 +933,14 @@ message Document { **Generated Code:** -| Language | Type | -| -------- | -------------------------- | -| Java | `List` | -| Python | `List[str]` | -| Go | `[]string` | -| Rust | `Vec` | -| C++ | `std::vector` | +| Language | Type | +| ---------- | -------------------------- | +| Java | `List` | +| Python | `List[str]` | +| Go | `[]string` | +| Rust | `Vec` | +| C++ | `std::vector` | +| JavaScript | `string[]` | ### Combining Modifiers @@ -1021,13 +1029,14 @@ collection behavior, and reference tracking (see #### Boolean -| Language | Type | Notes | -| -------- | --------------------- | ------------------ | -| Java | `boolean` / `Boolean` | Primitive or boxed | -| Python | `bool` | | -| Go | `bool` | | -| Rust | `bool` | | -| C++ | `bool` | | +| Language | Type | Notes | +| ---------- | --------------------- | ------------------ | +| Java | `boolean` / `Boolean` | Primitive or boxed | +| Python | `bool` | | +| Go | `bool` | | +| Rust | `bool` | | +| C++ | `bool` | | +| JavaScript | `boolean` | | #### Integer Types @@ -1042,12 +1051,12 @@ Fory IDL provides fixed-width signed integers (varint encoding for 32/64-bit by **Language Mapping (Signed):** -| Fory IDL | Java | Python | Go | Rust | C++ | -| -------- | ------- | -------------- | ------- | ----- | --------- | -| `int8` | `byte` | `pyfory.int8` | `int8` | `i8` | `int8_t` | -| `int16` | `short` | `pyfory.int16` | `int16` | `i16` | `int16_t` | -| `int32` | `int` | `pyfory.int32` | `int32` | `i32` | `int32_t` | -| `int64` | `long` | `pyfory.int64` | `int64` | `i64` | `int64_t` | +| Fory IDL | Java | Python | Go | Rust | C++ | Javascript | +| -------- | ------- | -------------- | ------- | ----- | --------- | ------------------ | +| `int8` | `byte` | `pyfory.int8` | `int8` | `i8` | `int8_t` | `number` | +| `int16` | `short` | `pyfory.int16` | `int16` | `i16` | `int16_t` | `number` | +| `int32` | `int` | `pyfory.int32` | `int32` | `i32` | `int32_t` | `number` | +| `int64` | `long` | `pyfory.int64` | `int64` | `i64` | `int64_t` | `bigint \| number` | Fory IDL provides fixed-width unsigned integers (varint encoding for 32/64-bit by default): @@ -1060,12 +1069,12 @@ Fory IDL provides fixed-width unsigned integers (varint encoding for 32/64-bit b **Language Mapping (Unsigned):** -| Fory IDL | Java | Python | Go | Rust | C++ | -| -------- | ------- | --------------- | -------- | ----- | ---------- | -| `uint8` | `short` | `pyfory.uint8` | `uint8` | `u8` | `uint8_t` | -| `uint16` | `int` | `pyfory.uint16` | `uint16` | `u16` | `uint16_t` | -| `uint32` | `long` | `pyfory.uint32` | `uint32` | `u32` | `uint32_t` | -| `uint64` | `long` | `pyfory.uint64` | `uint64` | `u64` | `uint64_t` | +| Fory IDL | Java | Python | Go | Rust | C++ | Javascript | +| -------- | ------- | --------------- | -------- | ----- | ---------- | ------------------ | +| `uint8` | `short` | `pyfory.uint8` | `uint8` | `u8` | `uint8_t` | `number` | +| `uint16` | `int` | `pyfory.uint16` | `uint16` | `u16` | `uint16_t` | `number` | +| `uint32` | `long` | `pyfory.uint32` | `uint32` | `u32` | `uint32_t` | `number` | +| `uint64` | `long` | `pyfory.uint64` | `uint64` | `u64` | `uint64_t` | `bigint \| number` | #### Integer Encoding Variants @@ -1090,62 +1099,67 @@ you need fixed-width or tagged encoding: **Language Mapping:** -| Fory IDL | Java | Python | Go | Rust | C++ | -| --------- | -------- | ---------------- | --------- | ----- | -------- | -| `float32` | `float` | `pyfory.float32` | `float32` | `f32` | `float` | -| `float64` | `double` | `pyfory.float64` | `float64` | `f64` | `double` | +| Fory IDL | Java | Python | Go | Rust | C++ | JavaScript | +| --------- | -------- | ---------------- | --------- | ----- | -------- | ---------- | +| `float32` | `float` | `pyfory.float32` | `float32` | `f32` | `float` | `number` | +| `float64` | `double` | `pyfory.float64` | `float64` | `f64` | `double` | `number` | #### String Type -| Language | Type | Notes | -| -------- | ------------- | --------------------- | -| Java | `String` | Immutable | -| Python | `str` | | -| Go | `string` | Immutable | -| Rust | `String` | Owned, heap-allocated | -| C++ | `std::string` | | +| Language | Type | Notes | +| ---------- | ------------- | --------------------- | +| Java | `String` | Immutable | +| Python | `str` | | +| Go | `string` | Immutable | +| Rust | `String` | Owned, heap-allocated | +| C++ | `std::string` | | +| JavaScript | `string` | | #### Bytes Type -| Language | Type | Notes | -| -------- | ---------------------- | --------- | -| Java | `byte[]` | | -| Python | `bytes` | Immutable | -| Go | `[]byte` | | -| Rust | `Vec` | | -| C++ | `std::vector` | | +| Language | Type | Notes | +| ---------- | ---------------------- | --------- | +| Java | `byte[]` | | +| Python | `bytes` | Immutable | +| Go | `[]byte` | | +| Rust | `Vec` | | +| C++ | `std::vector` | | +| JavaScript | `Uint8Array` | | #### Temporal Types ##### Date -| Language | Type | Notes | -| -------- | --------------------------- | ----------------------- | -| Java | `java.time.LocalDate` | | -| Python | `datetime.date` | | -| Go | `time.Time` | Time portion ignored | -| Rust | `chrono::NaiveDate` | Requires `chrono` crate | -| C++ | `fory::serialization::Date` | | +| Language | Type | Notes | +| ---------- | --------------------------- | ----------------------- | +| Java | `java.time.LocalDate` | | +| Python | `datetime.date` | | +| Go | `time.Time` | Time portion ignored | +| Rust | `chrono::NaiveDate` | Requires `chrono` crate | +| C++ | `fory::serialization::Date` | | +| JavaScript | `Date` | | ##### Timestamp -| Language | Type | Notes | -| -------- | -------------------------------- | ----------------------- | -| Java | `java.time.Instant` | UTC-based | -| Python | `datetime.datetime` | | -| Go | `time.Time` | | -| Rust | `chrono::NaiveDateTime` | Requires `chrono` crate | -| C++ | `fory::serialization::Timestamp` | | +| Language | Type | Notes | +| ---------- | -------------------------------- | ----------------------- | +| Java | `java.time.Instant` | UTC-based | +| Python | `datetime.datetime` | | +| Go | `time.Time` | | +| Rust | `chrono::NaiveDateTime` | Requires `chrono` crate | +| C++ | `fory::serialization::Timestamp` | | +| JavaScript | `Date` | | #### Any -| Language | Type | Notes | -| -------- | -------------- | -------------------- | -| Java | `Object` | Runtime type written | -| Python | `Any` | Runtime type written | -| Go | `any` | Runtime type written | -| Rust | `Box` | Runtime type written | -| C++ | `std::any` | Runtime type written | +| Language | Type | Notes | +| ---------- | -------------- | -------------------- | +| Java | `Object` | Runtime type written | +| Python | `Any` | Runtime type written | +| Go | `any` | Runtime type written | +| Rust | `Box` | Runtime type written | +| C++ | `std::any` | Runtime type written | +| JavaScript | `any` | Runtime type written | **Example:** @@ -1167,13 +1181,14 @@ message Envelope [id=122] { **Generated Code (`Envelope.payload`):** -| Language | Generated Field Type | -| -------- | ----------------------- | -| Java | `Object payload` | -| Python | `payload: Any` | -| Go | `Payload any` | -| Rust | `payload: Box` | -| C++ | `std::any payload` | +| Language | Generated Field Type | +| ---------- | ----------------------- | +| Java | `Object payload` | +| Python | `payload: Any` | +| Go | `Payload any` | +| Rust | `payload: Box` | +| C++ | `std::any payload` | +| JavaScript | `payload: any` | **Notes:** @@ -1224,10 +1239,10 @@ message Config { **Language Mapping:** -| Fory IDL | Java | Python | Go | Rust | C++ | -| -------------------- | ---------------------- | ----------------- | ------------------ | ----------------------- | -------------------------------- | -| `map` | `Map` | `Dict[str, int]` | `map[string]int32` | `HashMap` | `std::map` | -| `map` | `Map` | `Dict[str, User]` | `map[string]User` | `HashMap` | `std::map` | +| Fory IDL | Java | Python | Go | Rust | C++ | JavaScript | +| -------------------- | ---------------------- | ----------------- | ------------------ | ----------------------- | -------------------------------- | ------------------------ | +| `map` | `Map` | `Dict[str, int]` | `map[string]int32` | `HashMap` | `std::map` | `Record` | +| `map` | `Map` | `Dict[str, User]` | `map[string]User` | `HashMap` | `std::map` | `Record` | **Key Type Restrictions:** diff --git a/integration_tests/idl_tests/README.md b/integration_tests/idl_tests/README.md index ad0fc618c4..bc730345db 100644 --- a/integration_tests/idl_tests/README.md +++ b/integration_tests/idl_tests/README.md @@ -11,4 +11,5 @@ Run tests: - Rust: `./run_rust_tests.sh` - C++: `./run_cpp_tests.sh` - C#: `./run_csharp_tests.sh` +- JavaScript: `./run_javascript_tests.sh` - Swift: `./run_swift_tests.sh` diff --git a/integration_tests/idl_tests/generate_idl.py b/integration_tests/idl_tests/generate_idl.py index 6313508fd2..8e4f23d385 100755 --- a/integration_tests/idl_tests/generate_idl.py +++ b/integration_tests/idl_tests/generate_idl.py @@ -47,6 +47,7 @@ "go": REPO_ROOT / "integration_tests/idl_tests/go/generated", "rust": REPO_ROOT / "integration_tests/idl_tests/rust/src/generated", "csharp": REPO_ROOT / "integration_tests/idl_tests/csharp/IdlTests/Generated", + "javascript": REPO_ROOT / "integration_tests/idl_tests/javascript/generated", "swift": REPO_ROOT / "integration_tests/idl_tests/swift/idl_package/Sources/IdlGenerated/generated", } diff --git a/integration_tests/idl_tests/java/src/test/java/org/apache/fory/idl_tests/IdlRoundTripTest.java b/integration_tests/idl_tests/java/src/test/java/org/apache/fory/idl_tests/IdlRoundTripTest.java index 61ae1ca155..9f5a96d310 100644 --- a/integration_tests/idl_tests/java/src/test/java/org/apache/fory/idl_tests/IdlRoundTripTest.java +++ b/integration_tests/idl_tests/java/src/test/java/org/apache/fory/idl_tests/IdlRoundTripTest.java @@ -624,7 +624,7 @@ private List resolvePeers() { .filter(value -> !value.isEmpty()) .collect(Collectors.toList()); if (peers.contains("all")) { - return Arrays.asList("python", "go", "rust", "cpp", "swift"); + return Arrays.asList("python", "go", "rust", "cpp", "swift", "javascript"); } return peers; } @@ -684,6 +684,11 @@ private PeerCommand buildPeerCommand( command = Arrays.asList("swift", "test", "--filter", swiftTest); peerCommand.environment.put("ENABLE_FORY_DEBUG_OUTPUT", "1"); break; + case "javascript": + workDir = idlRoot.resolve("javascript"); + command = Arrays.asList("npx", "ts-node", "roundtrip.ts"); + peerCommand.environment.put("ENABLE_FORY_DEBUG_OUTPUT", "1"); + break; default: throw new IllegalArgumentException("Unknown peer language: " + peer); } diff --git a/integration_tests/idl_tests/javascript/.eslintrc.cjs b/integration_tests/idl_tests/javascript/.eslintrc.cjs new file mode 100644 index 0000000000..989c1e04a9 --- /dev/null +++ b/integration_tests/idl_tests/javascript/.eslintrc.cjs @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +module.exports = { + root: true, + ignorePatterns: ["**"], +}; diff --git a/integration_tests/idl_tests/javascript/.gitignore b/integration_tests/idl_tests/javascript/.gitignore new file mode 100644 index 0000000000..504afef81f --- /dev/null +++ b/integration_tests/idl_tests/javascript/.gitignore @@ -0,0 +1,2 @@ +node_modules/ +package-lock.json diff --git a/integration_tests/idl_tests/javascript/jest.config.js b/integration_tests/idl_tests/javascript/jest.config.js new file mode 100644 index 0000000000..7548378bb1 --- /dev/null +++ b/integration_tests/idl_tests/javascript/jest.config.js @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** @type {import('ts-jest').JestConfigWithTsJest} */ +module.exports = { + preset: 'ts-jest', + testEnvironment: 'node', + testMatch: ['**/test/**/*.test.ts'], +}; diff --git a/integration_tests/idl_tests/javascript/package.json b/integration_tests/idl_tests/javascript/package.json new file mode 100644 index 0000000000..1bf010337a --- /dev/null +++ b/integration_tests/idl_tests/javascript/package.json @@ -0,0 +1,21 @@ +{ + "name": "fory-idl-tests", + "version": "1.0.0", + "description": "Fory IDL integration tests for TypeScript", + "main": "index.js", + "scripts": { + "test": "jest", + "roundtrip": "ts-node roundtrip.ts" + }, + "dependencies": { + "@fory/fory": "file:../../../javascript/packages/fory" + }, + "devDependencies": { + "@types/jest": "^29.5.14", + "@types/node": "^20.11.0", + "jest": "^29.7.0", + "ts-jest": "^29.4.6", + "ts-node": "^10.9.2", + "typescript": "^4.9.5" + } +} diff --git a/integration_tests/idl_tests/javascript/roundtrip.ts b/integration_tests/idl_tests/javascript/roundtrip.ts new file mode 100644 index 0000000000..145aee077d --- /dev/null +++ b/integration_tests/idl_tests/javascript/roundtrip.ts @@ -0,0 +1,205 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * Cross-language roundtrip program for TypeScript IDL tests. + * + * This script is invoked by the Java IdlRoundTripTest as a peer process. + * It reads binary data files (written by Java), deserializes them, + * re-serializes the objects, and writes the bytes back to the same files. + * Java then reads the files back and verifies the roundtrip integrity. + * + * Environment variables: + * IDL_COMPATIBLE - "true" for compatible mode, "false" for schema_consistent + * DATA_FILE - AddressBook binary data file path + * DATA_FILE_AUTO_ID - Envelope (auto-id) binary data file path + * DATA_FILE_PRIMITIVES - PrimitiveTypes binary data file path + * DATA_FILE_COLLECTION - NumericCollections binary data file path + * DATA_FILE_COLLECTION_UNION - NumericCollectionUnion binary data file path + * DATA_FILE_COLLECTION_ARRAY - NumericCollectionsArray binary data file path + * DATA_FILE_COLLECTION_ARRAY_UNION - NumericCollectionArrayUnion binary data file path + * DATA_FILE_OPTIONAL_TYPES - OptionalHolder binary data file path + * DATA_FILE_TREE - TreeNode binary data file path (ref tracking) + * DATA_FILE_GRAPH - Graph binary data file path (ref tracking) + * DATA_FILE_FLATBUFFERS_MONSTER - Monster binary data file path + * DATA_FILE_FLATBUFFERS_TEST2 - Container binary data file path + */ + +import * as fs from "fs"; +import Fory, { Type } from "@fory/fory"; + +import { registerAddressbookTypes } from "./generated/addressbook"; +import { registerAutoIdTypes } from "./generated/auto_id"; +import { registerComplexPbTypes } from "./generated/complex_pb"; +import { registerCollectionTypes } from "./generated/collection"; +import { registerOptionalTypesTypes } from "./generated/optional_types"; +import { registerTreeTypes } from "./generated/tree"; +import { registerGraphTypes } from "./generated/graph"; +import { registerMonsterTypes } from "./generated/monster"; +import { registerComplexFbsTypes } from "./generated/complex_fbs"; + +// --------------------------------------------------------------------------- +// Capability: compatible mode +// --------------------------------------------------------------------------- +// The Fory JS runtime does not yet support compatible mode (class metadata / +// versioning is incomplete). Set this to `true` once the runtime adds +// compatible-mode support; all gated skips below will then run normally. +const SUPPORTS_COMPATIBLE_MODE = false; + +const compatible = process.env["IDL_COMPATIBLE"] === "true"; + +if (compatible && !SUPPORTS_COMPATIBLE_MODE) { + // Enumerate every data-file env var so the test output makes the exact + // coverage gap visible (mirrors how union-type skips are reported below). + const dataFileVars = [ + "DATA_FILE", + "DATA_FILE_AUTO_ID", + "DATA_FILE_PRIMITIVES", + "DATA_FILE_COLLECTION", + "DATA_FILE_COLLECTION_UNION", + "DATA_FILE_COLLECTION_ARRAY", + "DATA_FILE_COLLECTION_ARRAY_UNION", + "DATA_FILE_OPTIONAL_TYPES", + "DATA_FILE_TREE", + "DATA_FILE_GRAPH", + "DATA_FILE_FLATBUFFERS_MONSTER", + "DATA_FILE_FLATBUFFERS_TEST2", + ]; + const present = dataFileVars.filter((v) => process.env[v]); + console.log( + "TypeScript roundtrip: compatible mode is NOT SUPPORTED " + + "(SUPPORTS_COMPATIBLE_MODE = false)." + ); + for (const v of present) { + console.log(` SKIP [compatible]: ${v}`); + } + console.log( + ` 0/${present.length} compatible-mode roundtrips executed. ` + + "Files left unchanged so Java reads back its own bytes." + ); + process.exit(0); +} + +// --------------------------------------------------------------------------- +// Roundtrip helper: read file, deserialize, re-serialize, write back +// --------------------------------------------------------------------------- + +function fileRoundTrip( + envVar: string, + rootTypeId: number, + registerFn: (fory: any, type: any) => void, + foryOptions: { refTracking?: boolean | null } +): void { + const filePath = process.env[envVar]; + if (!filePath) { + return; + } + + console.log(`Processing ${envVar}: ${filePath}`); + + const fory = new Fory({ + compatible: false, + refTracking: foryOptions.refTracking ?? null, + }); + + registerFn(fory, Type); + + const serializer = fory.typeResolver.getSerializerByTypeInfo( + Type.struct(rootTypeId) + ); + + // Read binary data + const data = fs.readFileSync(filePath); + const bytes = new Uint8Array(data); + + // Deserialize + const obj = fory.deserialize(bytes, serializer); + + // Re-serialize + const result = fory.serialize(obj, serializer); + + // Write back + fs.writeFileSync(filePath, result); + console.log(` OK: roundtrip complete for ${envVar}`); +} + +// --------------------------------------------------------------------------- +// Process each data file type using generated code +// --------------------------------------------------------------------------- + + +fileRoundTrip("DATA_FILE", 103, registerAddressbookTypes, {}); + + +fileRoundTrip("DATA_FILE_AUTO_ID", 3022445236, registerAutoIdTypes, {}); + + +fileRoundTrip("DATA_FILE_PRIMITIVES", 200, registerComplexPbTypes, {}); + + +fileRoundTrip("DATA_FILE_COLLECTION", 210, registerCollectionTypes, {}); + +// DATA_FILE_COLLECTION_UNION: NumericCollectionUnion (IS a union) +// Union types are not yet supported in the Fory JS runtime. +if (process.env["DATA_FILE_COLLECTION_UNION"]) { + console.log( + "Processing DATA_FILE_COLLECTION_UNION: skipped (union type not yet supported)" + ); +} + +// DATA_FILE_COLLECTION_ARRAY: NumericCollectionsArray (type ID 212) +fileRoundTrip("DATA_FILE_COLLECTION_ARRAY", 212, registerCollectionTypes, {}); + +// DATA_FILE_COLLECTION_ARRAY_UNION: NumericCollectionArrayUnion (IS a union) +if (process.env["DATA_FILE_COLLECTION_ARRAY_UNION"]) { + console.log( + "Processing DATA_FILE_COLLECTION_ARRAY_UNION: skipped (union type not yet supported)" + ); +} + + +fileRoundTrip("DATA_FILE_OPTIONAL_TYPES", 122, registerOptionalTypesTypes, {}); + + +fileRoundTrip("DATA_FILE_TREE", 2251833438, registerTreeTypes, { + refTracking: true, +}); + + +fileRoundTrip("DATA_FILE_GRAPH", 2373163777, registerGraphTypes, { + refTracking: true, +}); + + +fileRoundTrip( + "DATA_FILE_FLATBUFFERS_MONSTER", + 438716985, + registerMonsterTypes, + {} +); + + +fileRoundTrip( + "DATA_FILE_FLATBUFFERS_TEST2", + 372413680, + registerComplexFbsTypes, + {} +); + +console.log("TypeScript roundtrip finished."); diff --git a/integration_tests/idl_tests/javascript/test/roundtrip.test.ts b/integration_tests/idl_tests/javascript/test/roundtrip.test.ts new file mode 100644 index 0000000000..b9cf861f82 --- /dev/null +++ b/integration_tests/idl_tests/javascript/test/roundtrip.test.ts @@ -0,0 +1,283 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * Integration tests for TypeScript IDL-generated code. + * + * These tests verify that: + * 1. Generated TypeScript types compile correctly + * 2. Objects can be constructed conforming to the generated interfaces + * 3. Roundtrip serialization works via the Fory JS runtime + */ + +import Fory, { Type } from '@fory/fory'; +import { + AddressBook, + Person, + Animal, + AnimalCase, + Dog, + Cat, + PhoneNumber, + PhoneType, + registerAddressbookTypes, +} from '../generated/addressbook'; +import { TreeNode } from '../generated/tree'; +import { + Envelope, + Payload, + Status, + DetailCase, + WrapperCase, + registerAutoIdTypes, +} from '../generated/auto_id'; + +// --------------------------------------------------------------------------- +// Helper: build test objects that conform to generated interfaces +// --------------------------------------------------------------------------- + +function buildDog(): Dog { + return { name: 'Rex', barkVolume: 5 }; +} + +function buildCat(): Cat { + return { name: 'Mimi', lives: 9 }; +} + +function buildPhoneNumber(num: string, pt: PhoneType): PhoneNumber { + return { number_: num, phoneType: pt }; +} + +function buildPerson(): Person { + return { + name: 'Alice', + id: 123, + email: 'alice@example.com', + tags: ['friend', 'colleague'], + scores: { math: 100, science: 98 }, + salary: 120000.5, + phones: [ + buildPhoneNumber('555-0100', PhoneType.MOBILE), + buildPhoneNumber('555-0111', PhoneType.WORK), + ], + pet: { case: AnimalCase.CAT, value: buildCat() }, + }; +} + +function buildAddressBook(): AddressBook { + const person = buildPerson(); + return { + people: [person], + peopleByName: { [person.name]: person }, + }; +} + +function buildTreeNode(): TreeNode { + const child1: TreeNode = { + id: 'child-1', + name: 'Child 1', + children: [], + parent: undefined, + }; + const child2: TreeNode = { + id: 'child-2', + name: 'Child 2', + children: [], + parent: undefined, + }; + return { + id: 'root', + name: 'Root', + children: [child1, child2], + parent: undefined, + }; +} + +function buildAutoIdEnvelope(): Envelope { + const payload: Payload = { value: 42 }; + return { + id: 'env-1', + payload, + detail: { case: DetailCase.PAYLOAD, value: payload }, + status: Status.OK, + }; +} + +// --------------------------------------------------------------------------- +// 1. Compilation & type-construction tests +// (If these tests run at all, the generated types compile correctly.) +// --------------------------------------------------------------------------- + +describe('Generated types compile and construct correctly', () => { + test('AddressBook type construction', () => { + const book = buildAddressBook(); + expect(book.people).toHaveLength(1); + expect(book.people[0].name).toBe('Alice'); + expect(book.people[0].id).toBe(123); + expect(book.people[0].email).toBe('alice@example.com'); + expect(book.people[0].tags).toEqual(['friend', 'colleague']); + expect(book.people[0].salary).toBe(120000.5); + expect(book.people[0].phones).toHaveLength(2); + expect(book.people[0].phones[0].phoneType).toBe(PhoneType.MOBILE); + expect(book.people[0].phones[1].phoneType).toBe(PhoneType.WORK); + expect(book.peopleByName['Alice']).toBe(book.people[0]); + }); + + test('Union (Animal) type construction', () => { + const dogAnimal: Animal = { + case: AnimalCase.DOG, + value: buildDog(), + }; + expect(dogAnimal.case).toBe(AnimalCase.DOG); + expect((dogAnimal.value as Dog).name).toBe('Rex'); + + const catAnimal: Animal = { + case: AnimalCase.CAT, + value: buildCat(), + }; + expect(catAnimal.case).toBe(AnimalCase.CAT); + expect((catAnimal.value as Cat).lives).toBe(9); + }); + + test('Enum values are correct', () => { + expect(PhoneType.MOBILE).toBe(0); + expect(PhoneType.HOME).toBe(1); + expect(PhoneType.WORK).toBe(2); + + expect(AnimalCase.DOG).toBe(1); + expect(AnimalCase.CAT).toBe(2); + }); + + test('TreeNode type construction with optional parent', () => { + const tree = buildTreeNode(); + expect(tree.id).toBe('root'); + expect(tree.children).toHaveLength(2); + expect(tree.parent).toBeUndefined(); + expect(tree.children[0].name).toBe('Child 1'); + }); + + test('AutoId types type construction', () => { + const envelope = buildAutoIdEnvelope(); + expect(envelope.id).toBe('env-1'); + expect(envelope.payload?.value).toBe(42); + expect(envelope.status).toBe(Status.OK); + + expect(Status.UNKNOWN).toBe(0); + expect(Status.OK).toBe(1); + + expect(WrapperCase.ENVELOPE).toBe(1); + expect(WrapperCase.RAW).toBe(2); + + expect(DetailCase.PAYLOAD).toBe(1); + expect(DetailCase.NOTE).toBe(2); + }); +}); + +// --------------------------------------------------------------------------- +// 2. Serialization roundtrip tests using the Fory JS runtime +// We manually build TypeInfo objects matching the generated interfaces. +// --------------------------------------------------------------------------- + +describe('Serialization roundtrip', () => { + test('Dog struct roundtrip', () => { + const fory = new Fory(); + registerAddressbookTypes(fory, Type); + const serializer = fory.typeResolver.getSerializerByTypeInfo( + Type.struct(104), + ); + + const dog: Dog = buildDog(); + const bytes = fory.serialize(dog, serializer); + const result = fory.deserialize(bytes, serializer) as Dog; + + expect(result).toEqual(dog); + }); + + test('Cat struct roundtrip', () => { + const fory = new Fory(); + registerAddressbookTypes(fory, Type); + const serializer = fory.typeResolver.getSerializerByTypeInfo( + Type.struct(105), + ); + + const cat: Cat = buildCat(); + const bytes = fory.serialize(cat, serializer); + const result = fory.deserialize(bytes, serializer) as Cat; + + expect(result).toEqual(cat); + }); + + test('PhoneNumber struct roundtrip', () => { + const fory = new Fory(); + registerAddressbookTypes(fory, Type); + const serializer = fory.typeResolver.getSerializerByTypeInfo( + Type.struct(102), + ); + + const phone: PhoneNumber = buildPhoneNumber('555-0100', PhoneType.MOBILE); + const bytes = fory.serialize(phone, serializer); + const result = fory.deserialize(bytes, serializer) as PhoneNumber; + + expect(result).toEqual(phone); + }); + + test('Payload (autoId) struct roundtrip', () => { + const fory = new Fory(); + registerAutoIdTypes(fory, Type); + const serializer = fory.typeResolver.getSerializerByTypeInfo( + Type.struct(2862577837), + ); + + const payload: Payload = { value: 42 }; + const bytes = fory.serialize(payload, serializer); + const result = fory.deserialize(bytes, serializer) as Payload; + + expect(result).toEqual(payload); + }); +}); + +// --------------------------------------------------------------------------- +// 3. Optional field tests +// --------------------------------------------------------------------------- + +describe('Optional field handling', () => { + test('struct with nullable string field', () => { + const fory = new Fory(); + const optType = Type.struct( + { typeName: 'test.OptionalStruct' }, + { + name: Type.string(), + nickname: Type.string().setNullable(true), + }, + ); + const { serialize, deserialize } = fory.registerSerializer(optType); + + // With value present + const withValue = { name: 'Alice', nickname: 'Ali' }; + const bytes1 = serialize(withValue); + const result1 = deserialize(bytes1); + expect(result1).toEqual(withValue); + + // With null value + const withNull = { name: 'Bob', nickname: null }; + const bytes2 = serialize(withNull); + const result2 = deserialize(bytes2); + expect(result2).toEqual(withNull); + }); +}); diff --git a/integration_tests/idl_tests/javascript/tsconfig.json b/integration_tests/idl_tests/javascript/tsconfig.json new file mode 100644 index 0000000000..b0cf9a0536 --- /dev/null +++ b/integration_tests/idl_tests/javascript/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "es2020", + "lib": ["es2020", "es2021.weakref"], + "module": "commonjs", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "outDir": "./dist", + "noUnusedLocals": false, + "noUnusedParameters": false + }, + "include": ["test/**/*", "generated/**/*", "roundtrip.ts"] +} diff --git a/integration_tests/idl_tests/run_javascript_tests.sh b/integration_tests/idl_tests/run_javascript_tests.sh new file mode 100755 index 0000000000..e0e6a67167 --- /dev/null +++ b/integration_tests/idl_tests/run_javascript_tests.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +python "${SCRIPT_DIR}/generate_idl.py" --lang javascript + +cd "${SCRIPT_DIR}/javascript" +npm install +ENABLE_FORY_DEBUG_OUTPUT=1 npx jest --ci + +IDL_PEER_LANG=javascript "${SCRIPT_DIR}/run_java_tests.sh" +