refactor fory compiler

Author: chaokunyang

compiler/fory_compiler/__init__.py

@@ -15,13 +15,13 @@
 # specific language governing permissions and limitations
 # under the License.
 
-"""FDL (Fory Definition Language) compiler for Apache Fory."""
+"""Fory IDL compiler for Apache Fory."""
 
 __version__ = "0.1.0"
 
-from fory_compiler.parser.ast import Schema, Message, Enum, Field, EnumValue, Import
-from fory_compiler.parser.parser import Parser
-from fory_compiler.parser.lexer import Lexer
+from fory_compiler.ir.ast import Schema, Message, Enum, Field, EnumValue, Import
+from fory_compiler.frontend.fdl import FDLFrontend
+from fory_compiler.frontend.proto import ProtoFrontend
 
 __all__ = [
     "Schema",
@@ -30,6 +30,6 @@
     "Field",
     "EnumValue",
     "Import",
-    "Parser",
-    "Lexer",
+    "FDLFrontend",
+    "ProtoFrontend",
 ]

compiler/fory_compiler/cli.py

@@ -15,16 +15,18 @@
 # specific language governing permissions and limitations
 # under the License.
 
-"""CLI entry point for the FDL compiler."""
+"""CLI entry point for the Fory IDL compiler."""
 
 import argparse
 import sys
 from pathlib import Path
 from typing import Dict, List, Optional, Set
 
-from fory_compiler.parser.lexer import Lexer, LexerError
-from fory_compiler.parser.parser import Parser, ParseError
-from fory_compiler.parser.ast import Schema
+from fory_compiler.frontend.base import FrontendError
+from fory_compiler.frontend.fdl import FDLFrontend
+from fory_compiler.frontend.proto import ProtoFrontend
+from fory_compiler.ir.ast import Schema
+from fory_compiler.ir.emitter import FDLEmitter
 from fory_compiler.generators.base import GeneratorOptions
 from fory_compiler.generators import GENERATORS
 
@@ -35,13 +37,19 @@ class ImportError(Exception):
     pass
 
 
-def parse_fdl_file(file_path: Path) -> Schema:
-    """Parse a single FDL file and return its schema."""
-    source = file_path.read_text()
-    lexer = Lexer(source, str(file_path))
-    tokens = lexer.tokenize()
-    parser = Parser(tokens)
-    return parser.parse()
+def get_frontend(file_path: Path):
+    """Select the correct frontend for a file."""
+    frontends = [FDLFrontend(), ProtoFrontend()]
+    for frontend in frontends:
+        if frontend.supports_file(file_path):
+            return frontend
+    raise ValueError(f"Unsupported file extension: {file_path.suffix}")
+
+
+def parse_idl_file(file_path: Path) -> Schema:
+    """Parse a single IDL file and return its schema."""
+    frontend = get_frontend(file_path)
+    return frontend.parse_file(file_path)
 
 
 def resolve_import_path(
@@ -117,7 +125,7 @@ def resolve_imports(
     visited.add(file_path)
 
     # Parse the file
-    schema = parse_fdl_file(file_path)
+    schema = parse_idl_file(file_path)
 
     # Process imports
     imported_enums = []
@@ -152,6 +160,9 @@ def resolve_imports(
         imports=schema.imports,
         enums=imported_enums + schema.enums,
         messages=imported_messages + schema.messages,
+        options=schema.options,
+        source_file=schema.source_file,
+        source_format=schema.source_format,
     )
 
     cache[file_path] = merged_schema
@@ -162,23 +173,23 @@ def parse_args(args: Optional[List[str]] = None) -> argparse.Namespace:
     """Parse command line arguments."""
     parser = argparse.ArgumentParser(
         prog="fory",
-        description="FDL (Fory Definition Language) compiler",
+        description="Fory IDL compiler",
     )
 
     subparsers = parser.add_subparsers(dest="command", help="Available commands")
 
     # compile command
     compile_parser = subparsers.add_parser(
         "compile",
-        help="Compile FDL files to language-specific code",
+        help="Compile IDL files (.fdl, .proto) to language-specific code",
     )
 
     compile_parser.add_argument(
         "files",
         nargs="+",
         type=Path,
         metavar="FILE",
-        help="FDL files to compile",
+        help="IDL files to compile",
     )
 
     compile_parser.add_argument(
@@ -264,6 +275,19 @@ def parse_args(args: Optional[List[str]] = None) -> argparse.Namespace:
         help="Go nested type naming style: camelcase or underscore (default)",
     )
 
+    compile_parser.add_argument(
+        "--emit-fdl",
+        action="store_true",
+        help="Emit translated FDL (for non-FDL inputs) for debugging",
+    )
+
+    compile_parser.add_argument(
+        "--emit-fdl-path",
+        type=Path,
+        default=None,
+        help="Write translated FDL to this path (file or directory)",
+    )
+
     return parser.parse_args(args)
 
 
@@ -290,11 +314,13 @@ def compile_file(
     package_override: Optional[str] = None,
     import_paths: Optional[List[Path]] = None,
     go_nested_type_style: Optional[str] = None,
+    emit_fdl: bool = False,
+    emit_fdl_path: Optional[Path] = None,
 ) -> bool:
-    """Compile a single FDL file with import resolution.
+    """Compile a single IDL file with import resolution.
 
     Args:
-        file_path: Path to the FDL file
+        file_path: Path to the IDL file
         lang_output_dirs: Dictionary mapping language name to output directory
         package_override: Optional package name override
         import_paths: List of import search paths
@@ -307,7 +333,7 @@ def compile_file(
     except OSError as e:
         print(f"Error reading {file_path}: {e}", file=sys.stderr)
         return False
-    except (LexerError, ParseError) as e:
+    except (FrontendError, ValueError) as e:
         print(f"Error: {e}", file=sys.stderr)
         return False
     except ImportError as e:
@@ -318,6 +344,24 @@ def compile_file(
     if schema.imports:
         print(f"  Resolved {len(schema.imports)} import(s)")
 
+    if emit_fdl:
+        emitter = FDLEmitter(schema)
+        fdl_content = emitter.emit()
+        if emit_fdl_path:
+            target = emit_fdl_path
+            if target.exists() and target.is_dir():
+                target = target / f"{file_path.stem}.fdl"
+            elif str(target).endswith("/") or str(target).endswith("\\"):
+                target.mkdir(parents=True, exist_ok=True)
+                target = target / f"{file_path.stem}.fdl"
+            target.parent.mkdir(parents=True, exist_ok=True)
+            target.write_text(fdl_content)
+            print(f"  Emitted FDL: {target}")
+        else:
+            print("=== Translated FDL ===")
+            print(fdl_content.rstrip())
+            print("======================")
+
     # Validate merged schema
     errors = schema.validate()
     if errors:
@@ -417,6 +461,8 @@ def cmd_compile(args: argparse.Namespace) -> int:
             args.package,
             import_paths,
             args.go_nested_type_style,
+            args.emit_fdl,
+            args.emit_fdl_path,
         ):
             success = False
 

compiler/fory_compiler/frontend/__init__.py

@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Frontends for Fory IDL compilation."""
+
+from fory_compiler.frontend.base import BaseFrontend, FrontendError
+
+__all__ = ["BaseFrontend", "FrontendError"]

compiler/fory_compiler/frontend/base.py

@@ -0,0 +1,53 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Frontend base classes for Fory IDL compilation."""
+
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import List
+
+from fory_compiler.ir.ast import Schema
+
+
+class FrontendError(Exception):
+    """Error during frontend parsing or translation."""
+
+    def __init__(self, message: str, file: str, line: int, column: int):
+        super().__init__(f"{file}:{line}:{column}: {message}")
+        self.message = message
+        self.file = file
+        self.line = line
+        self.column = column
+
+
+class BaseFrontend(ABC):
+    """Base class for all IDL frontends."""
+
+    extensions: List[str] = []
+
+    @abstractmethod
+    def parse(self, source: str, filename: str = "<input>") -> Schema:
+        """Parse source and return a Fory IR schema."""
+
+    def parse_file(self, path: Path) -> Schema:
+        """Parse a file and return a Fory IR schema."""
+        return self.parse(path.read_text(), str(path))
+
+    def supports_file(self, path: Path) -> bool:
+        """Return True if this frontend handles the file extension."""
+        return path.suffix.lower() in self.extensions

compiler/fory_compiler/frontend/fdl/__init__.py

@@ -0,0 +1,44 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""FDL frontend."""
+
+from fory_compiler.frontend.base import BaseFrontend, FrontendError
+from fory_compiler.ir.ast import Schema
+from fory_compiler.frontend.fdl.lexer import Lexer, LexerError
+from fory_compiler.frontend.fdl.parser import Parser, ParseError
+
+
+class FDLFrontend(BaseFrontend):
+    """Frontend for Fory Definition Language (.fdl)."""
+
+    extensions = [".fdl"]
+
+    def parse(self, source: str, filename: str = "<input>") -> Schema:
+        try:
+            lexer = Lexer(source, filename)
+            tokens = lexer.tokenize()
+            parser = Parser(tokens)
+            schema = parser.parse()
+        except (LexerError, ParseError) as exc:
+            raise FrontendError(exc.message, filename, exc.line, exc.column) from exc
+        schema.source_file = filename
+        schema.source_format = "fdl"
+        return schema
+
+
+__all__ = ["FDLFrontend"]